1package deployment_test
2
3import (
4 "context"
5 "testing"
6 "time"
7
8 "github.com/bnema/zerowrap"
9 "github.com/stretchr/testify/assert"
10 "github.com/stretchr/testify/mock"
11 "github.com/stretchr/testify/require"
12
13 "github.com/bnema/gordon/internal/adapters/out/appstate"
14 outmocks "github.com/bnema/gordon/internal/boundaries/out/mocks"
15 "github.com/bnema/gordon/internal/domain"
16 "github.com/bnema/gordon/internal/usecase/deployment"
17)
18
19// seedRevision materializes one revision of an app, so a later recovery can
20// rebuild that generation from its pinned digest.
21func seedRevision(t *testing.T, ctx context.Context, store *appstate.Store, intentID, supersedes string, rev domain.AppDesiredRevision) {
22 t.Helper()
23 require.NoError(t, store.StageApply(ctx, domain.AppApplyIntent{
24 Intent: intentID, App: rev.App, Revision: rev.Revision, Supersedes: supersedes,
25 SourceSHA256: "0f1e2d", Spec: rev.Spec, CreatedAt: time.Now().UTC(),
26 }))
27 require.NoError(t, store.CommitApply(ctx, rev.App, intentID))
28 require.NoError(t, store.MaterializeApply(ctx, rev.App, intentID))
29}
30
31// seedReplaceableApp seeds one app whose ACTIVE generation (rev-0) has a
32// materialized superseding revision (rev-1). The published container c-old is
33// gone from the runtime: the caller decides what the runtime reports for it.
34func seedReplaceableApp(t *testing.T, ctx context.Context, store *appstate.Store) (domain.AppDesiredRevision, domain.AppDesiredRevision) {
35 t.Helper()
36 spec := webService()
37 spec.Secrets = map[string]string{}
38 spec.Readiness = domain.AppReadiness{Type: domain.AppReadinessHTTP, Path: "/healthz", Timeout: time.Second}
39 activeRev := testRevision("blog", spec)
40 activeRev.Revision = "rev-0"
41 require.NoError(t, store.SaveIntent(ctx, domain.AppStopIntent{App: "blog"}))
42 require.NoError(t, store.SaveActive(ctx, domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
43 "web": {
44 Container: "c-old", EffectiveRevision: "rev-0", Image: spec.Image,
45 Digest: restartTestDigest, Spec: spec, BackendBinds: map[int]int{8080: 18080},
46 },
47 }}))
48 require.NoError(t, store.SaveOwnership(ctx, domain.AppOwnership{App: "blog", ID: "app-blog"}))
49 seedRevision(t, ctx, store, "intent-0", "", activeRev)
50 desired := testRevision("blog", spec)
51 seedRevision(t, ctx, store, "intent-1", "rev-0", desired)
52 return activeRev, desired
53}
54
55// TestDeploy_CandidateIsJournaledBeforeItStarts proves the crash window is
56// closed as far as a container ID allows: the candidate ID reaches the
57// durable journal before the container is started, and the journal is
58// finalized as a success only after publication.
59func TestDeploy_CandidateIsJournaledBeforeItStarts(t *testing.T) {
60 ctx := context.Background()
61 store := newTestStore(t)
62 runtime := outmocks.NewMockContainerRuntime(t)
63 images := outmocks.NewMockImageResolver(t)
64 secrets := outmocks.NewMockSecretProvider(t)
65 seedReplaceableApp(t, ctx, store)
66
67 images.EXPECT().ResolveDigest(mock.Anything, "docker.io/example/web:1.4.2").Return(restartTestDigest, nil).Once()
68 runtime.EXPECT().InspectImageVolumes(mock.Anything, "docker.io/example/web:1.4.2").Return(nil, nil).Once()
69 expectNetworkProvision(runtime, "app-blog", 1)
70 runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(nil).Once()
71 runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
72 runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-new", Name: "web"}, nil).Once()
73
74 journaledBeforeStart := false
75 runtime.EXPECT().StartContainer(mock.Anything, "c-new").RunAndReturn(func(context.Context, string) error {
76 op, ok, err := store.LoadLatestOperation(context.Background(), "blog")
77 require.NoError(t, err)
78 require.True(t, ok, "the replacement is journaled before it starts")
79 require.False(t, op.Terminal(), "the journal is still in flight while the candidate starts")
80 step, found := opStep(op, "service.web.replace")
81 journaledBeforeStart = found && step.After == "c-new" && step.State == domain.AppStepPending
82 return nil
83 }).Once()
84 runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return(
85 []domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18081, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
86
87 svc := deployment.NewService(deployment.Deps{
88 State: store, Runtime: runtime, Images: images, Secrets: secrets,
89 }, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
90 func(context.Context, string, string) (int, error) { return 200, nil },
91 func(context.Context, string) error { return nil },
92 ))
93
94 result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog", Op: "op-journal"})
95 require.NoError(t, err)
96 require.NotNil(t, result)
97 assert.Equal(t, "deployed", result.Services["web"].Result)
98 assert.True(t, journaledBeforeStart,
99 "the candidate ID is durable before the container is started")
100
101 op, err := store.LoadOperation(context.Background(), "blog", "op-journal")
102 require.NoError(t, err)
103 assert.Equal(t, domain.AppOutcomeSuccess, op.Outcome)
104 step, found := opStep(op, "service.web.replace")
105 require.True(t, found)
106 assert.Equal(t, domain.AppStepSucceeded, step.State)
107 assert.Equal(t, "c-new", step.After)
108}
109
110// crashDuringReplacement runs one replacement deploy and interrupts it
111// between creating the candidate and publishing it. The candidate is left
112// running, ACTIVE still names the superseded generation, and the operation
113// journal stays in flight: the state a process crash or a failed cleanup
114// leaves behind.
115func crashDuringReplacement(t *testing.T, store *appstate.Store, opKey string) {
116 t.Helper()
117 ctx, cancel := context.WithCancel(context.Background())
118 defer cancel()
119 runtime := outmocks.NewMockContainerRuntime(t)
120 images := outmocks.NewMockImageResolver(t)
121 secrets := outmocks.NewMockSecretProvider(t)
122
123 images.EXPECT().ResolveDigest(mock.Anything, "docker.io/example/web:1.4.2").Return(restartTestDigest, nil).Once()
124 runtime.EXPECT().InspectImageVolumes(mock.Anything, "docker.io/example/web:1.4.2").Return(nil, nil).Once()
125 expectNetworkProvision(runtime, "app-blog", 1)
126 runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(nil).Once()
127 runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
128 runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-orphan", Name: "web"}, nil).Once()
129 runtime.EXPECT().StartContainer(mock.Anything, "c-orphan").Return(nil).Once()
130 runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-orphan", mock.Anything).Return(
131 []domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18081, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
132 // The canceled context also refuses the candidate cleanup, so the
133 // candidate outlives the operation exactly as a crash would leave it.
134 runtime.EXPECT().RemoveContainer(mock.Anything, "c-orphan", true).RunAndReturn(
135 func(removeCtx context.Context, _ string, _ bool) error {
136 return removeCtx.Err()
137 }).Once()
138
139 probeStarted := make(chan struct{})
140 svc := deployment.NewService(deployment.Deps{
141 State: store, Runtime: runtime, Images: images, Secrets: secrets,
142 }, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
143 func(probeCtx context.Context, _, _ string) (int, error) {
144 select {
145 case <-probeStarted:
146 default:
147 close(probeStarted)
148 }
149 <-probeCtx.Done()
150 return 0, probeCtx.Err()
151 },
152 func(context.Context, string) error { return nil },
153 ))
154 done := make(chan error, 1)
155 go func() {
156 _, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog", Op: opKey})
157 done <- err
158 }()
159 select {
160 case <-probeStarted:
161 case <-time.After(5 * time.Second):
162 t.Fatal("timed out waiting for the readiness probe")
163 }
164 cancel()
165 require.Error(t, <-done)
166}
167
168// TestBootRecovery_RemovesUnpublishedCandidateBeforeRebuildingPublished
169// proves the crash window between creating a replacement and publishing it is
170// recoverable: an interruption leaves the candidate journaled but unpublished,
171// and boot recovery removes that candidate before it rebuilds the generation
172// recorded in ACTIVE, so two generations of one service never run together.
173func TestBootRecovery_RemovesUnpublishedCandidateBeforeRebuildingPublished(t *testing.T) {
174 ctx := context.Background()
175 store := newTestStore(t)
176 images := outmocks.NewMockImageResolver(t)
177 secrets := outmocks.NewMockSecretProvider(t)
178 seedReplaceableApp(t, ctx, store)
179 crashDuringReplacement(t, store, "op-crash")
180
181 // The crash left the candidate durable but unpublished, and ACTIVE still
182 // names the superseded generation.
183 interrupted, err := store.LoadOperation(context.Background(), "blog", "op-crash")
184 require.NoError(t, err)
185 assert.False(t, interrupted.Terminal(), "an interrupted operation stays in flight until recovery finalizes it")
186 crashStep, found := opStep(interrupted, "service.web.replace")
187 require.True(t, found)
188 assert.Equal(t, domain.AppStepPending, crashStep.State)
189 assert.Equal(t, "c-orphan", crashStep.After, "the created candidate is durably recorded")
190 active, ok, err := store.LoadActive(context.Background(), "blog")
191 require.NoError(t, err)
192 require.True(t, ok)
193 assert.Equal(t, "c-old", active.Services["web"].Container)
194
195 // Boot recovery: the orphan is removed before the published generation is
196 // rebuilt from its pinned revision.
197 var order []string
198 runtime2 := outmocks.NewMockContainerRuntime(t)
199 runtime2.EXPECT().RemoveContainer(mock.Anything, "c-orphan", true).RunAndReturn(
200 func(context.Context, string, bool) error {
201 order = append(order, "remove-orphan")
202 return nil
203 }).Once()
204 runtime2.EXPECT().IsContainerRunning(mock.Anything, "c-old").Return(false, nil).Once()
205 runtime2.EXPECT().StartContainer(mock.Anything, "c-old").Return(domain.ErrContainerNotFound).Once()
206 runtime2.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(domain.ErrContainerNotFound).Once()
207 runtime2.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(domain.ErrContainerNotFound).Once()
208 expectNetworkProvision(runtime2, "app-blog", 1)
209 runtime2.EXPECT().CreateContainer(mock.Anything, mock.Anything).RunAndReturn(
210 func(context.Context, *domain.ContainerConfig) (*domain.Container, error) {
211 order = append(order, "rebuild-published-generation")
212 return &domain.Container{ID: "c-rebuilt", Name: "web"}, nil
213 }).Once()
214 runtime2.EXPECT().StartContainer(mock.Anything, "c-rebuilt").Return(nil).Once()
215 runtime2.EXPECT().GetContainerBackendBinds(mock.Anything, "c-rebuilt", mock.Anything).Return(
216 []domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18082, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
217
218 rebooted := deployment.NewService(deployment.Deps{
219 State: store, Runtime: runtime2, Images: images, Secrets: secrets,
220 }, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime2,
221 func(context.Context, string, string) (int, error) { return 200, nil },
222 func(context.Context, string) error { return nil },
223 ))
224 // Boot recovery runs on a fresh process context: the crashed deploy's
225 // context is gone with it.
226 require.NoError(t, rebooted.ReconcileBoot(ctx))
227
228 assert.Equal(t, []string{"remove-orphan", "rebuild-published-generation"}, order,
229 "the unpublished candidate is gone before the published generation is rebuilt")
230 runtime2.AssertNotCalled(t, "StartContainer", mock.Anything, "c-orphan")
231 runtime2.AssertNotCalled(t, "RestartContainer", mock.Anything, "c-orphan", mock.Anything)
232
233 // The interrupted operation is finalized as failed, so a repeat of its key
234 // replays an explicit failure instead of a permanent in-flight claim.
235 finalized, err := store.LoadOperation(context.Background(), "blog", "op-crash")
236 require.NoError(t, err)
237 require.True(t, finalized.Terminal(), "boot recovery finalizes the interrupted operation")
238 assert.Equal(t, domain.AppOutcomeFailed, finalized.Outcome)
239 finalStep, found := opStep(finalized, "service.web.replace")
240 require.True(t, found)
241 assert.Equal(t, domain.AppStepFailed, finalStep.State)
242 assert.Contains(t, finalStep.Error, "unpublished candidate was removed")
243
244 // ACTIVE now names the rebuilt generation: recovery converged the app
245 // instead of leaving it pointing at a container that no longer exists.
246 converged, ok, err := store.LoadActive(context.Background(), "blog")
247 require.NoError(t, err)
248 require.True(t, ok)
249 assert.Equal(t, "c-rebuilt", converged.Services["web"].Container)
250 assert.Equal(t, "rev-0", converged.Services["web"].EffectiveRevision)
251}
252
253// TestDeploy_ReconcilesInterruptedPredecessorBeforeCreatingAReplacement
254// proves the live case: when a cleanup failed and Gordon kept running, the
255// next mutation of the app removes the leftover candidate before it creates a
256// new generation, so two generations of one service never run together even
257// without a restart. It also finalizes the interrupted journal.
258func TestDeploy_ReconcilesInterruptedPredecessorBeforeCreatingAReplacement(t *testing.T) {
259 ctx := context.Background()
260 store := newTestStore(t)
261 images := outmocks.NewMockImageResolver(t)
262 secrets := outmocks.NewMockSecretProvider(t)
263 seedReplaceableApp(t, ctx, store)
264 crashDuringReplacement(t, store, "op-crash")
265
266 var order []string
267 runtime := outmocks.NewMockContainerRuntime(t)
268 runtime.EXPECT().RemoveContainer(mock.Anything, "c-orphan", true).RunAndReturn(
269 func(context.Context, string, bool) error {
270 order = append(order, "remove-leftover-candidate")
271 return nil
272 }).Once()
273 images.EXPECT().ResolveDigest(mock.Anything, "docker.io/example/web:1.4.2").Return(restartTestDigest, nil).Once()
274 runtime.EXPECT().InspectImageVolumes(mock.Anything, "docker.io/example/web:1.4.2").Return(nil, nil).Once()
275 runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(domain.ErrContainerNotFound).Once()
276 runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(domain.ErrContainerNotFound).Once()
277 expectNetworkProvision(runtime, "app-blog", 1)
278 runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).RunAndReturn(
279 func(context.Context, *domain.ContainerConfig) (*domain.Container, error) {
280 order = append(order, "create-new-generation")
281 return &domain.Container{ID: "c-next", Name: "web"}, nil
282 }).Once()
283 runtime.EXPECT().StartContainer(mock.Anything, "c-next").Return(nil).Once()
284 runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-next", mock.Anything).Return(
285 []domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18082, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
286
287 svc := deployment.NewService(deployment.Deps{
288 State: store, Runtime: runtime, Images: images, Secrets: secrets,
289 }, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
290 func(context.Context, string, string) (int, error) { return 200, nil },
291 func(context.Context, string) error { return nil },
292 ))
293
294 result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog", Op: "op-next"})
295 require.NoError(t, err)
296 require.NotNil(t, result)
297 assert.Equal(t, "deployed", result.Services["web"].Result)
298 assert.Equal(t, []string{"remove-leftover-candidate", "create-new-generation"}, order)
299
300 active, ok, err := store.LoadActive(ctx, "blog")
301 require.NoError(t, err)
302 require.True(t, ok)
303 assert.Equal(t, "c-next", active.Services["web"].Container)
304
305 interrupted, err := store.LoadOperation(ctx, "blog", "op-crash")
306 require.NoError(t, err)
307 require.True(t, interrupted.Terminal(), "the interrupted operation is finalized")
308 assert.Equal(t, domain.AppOutcomeFailed, interrupted.Outcome)
309 step, found := opStep(interrupted, "service.web.replace")
310 require.True(t, found)
311 assert.Equal(t, domain.AppStepFailed, step.State)
312}
313
314// TestBootRecovery_RemovesCandidateOfAnInterruptedFirstDeployment proves the
315// same recovery works when the interrupted operation is the app's FIRST
316// deployment: there is no ACTIVE generation to converge, and the leftover
317// candidate is still removed and the journal finalized.
318func TestBootRecovery_RemovesCandidateOfAnInterruptedFirstDeployment(t *testing.T) {
319 ctx := context.Background()
320 store := newTestStore(t)
321 images := outmocks.NewMockImageResolver(t)
322 secrets := outmocks.NewMockSecretProvider(t)
323 require.NoError(t, store.SaveIntent(ctx, domain.AppStopIntent{App: "blog"}))
324 require.NoError(t, store.SaveOperation(ctx, domain.AppOperation{
325 Op: "op-seeded", Kind: "deploy", App: "blog", StartedAt: time.Now().UTC(),
326 Steps: []domain.AppOperationStep{
327 {ID: "preflight", State: domain.AppStepSucceeded},
328 {ID: "service.web.replace", State: domain.AppStepPending, Service: "web", Before: "c-old", After: "c-orphan"},
329 },
330 }))
331
332 runtime := outmocks.NewMockContainerRuntime(t)
333 runtime.EXPECT().RemoveContainer(mock.Anything, "c-orphan", true).Return(nil).Once()
334
335 svc := deployment.NewService(deployment.Deps{
336 State: store, Runtime: runtime, Images: images, Secrets: secrets,
337 }, zerowrap.Default())
338 require.NoError(t, svc.ReconcileBoot(ctx))
339
340 runtime.AssertNotCalled(t, "CreateContainer", mock.Anything, mock.Anything)
341 runtime.AssertNotCalled(t, "StartContainer", mock.Anything, "c-orphan")
342
343 finalized, err := store.LoadOperation(ctx, "blog", "op-seeded")
344 require.NoError(t, err)
345 require.True(t, finalized.Terminal())
346 assert.Equal(t, domain.AppOutcomeFailed, finalized.Outcome)
347 step, found := opStep(finalized, "service.web.replace")
348 require.True(t, found)
349 assert.Equal(t, domain.AppStepFailed, step.State)
350 assert.Contains(t, step.Error, "unpublished candidate was removed")
351}
352
353// TestRestart_FailedRebuildClearsTheInhibitionOfTheMissingContainer proves a
354// restart that falls back to a rebuild does not leave a durable inhibition for
355// a container that is already gone: nothing could ever revive it, and the
356// marker would refuse every later start, recovery pass, and restart.
357func TestRestart_FailedRebuildClearsTheInhibitionOfTheMissingContainer(t *testing.T) {
358 ctx := context.Background()
359 store := newTestStore(t)
360 images := outmocks.NewMockImageResolver(t)
361 secrets := outmocks.NewMockSecretProvider(t)
362 runtime := outmocks.NewMockContainerRuntime(t)
363
364 spec := webService()
365 spec.Secrets = map[string]string{}
366 spec.Volumes = []domain.AppVolume{{Name: "data", Path: "/data"}}
367 spec.Readiness = domain.AppReadiness{Type: domain.AppReadinessHTTP, Path: "/healthz", Timeout: 50 * time.Millisecond}
368 rev := testRevision("blog", spec)
369 rev.Revision = "rev-0"
370 require.NoError(t, store.SaveIntent(ctx, domain.AppStopIntent{App: "blog"}))
371 require.NoError(t, store.SaveActive(ctx, domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
372 "web": {
373 Container: "c-old", EffectiveRevision: "rev-0", Image: spec.Image,
374 Digest: restartTestDigest, Spec: spec, BackendBinds: map[int]int{8080: 18080},
375 },
376 }}))
377 require.NoError(t, store.SaveOwnership(ctx, domain.AppOwnership{App: "blog", ID: "app-blog"}))
378 seedRevision(t, ctx, store, "intent-0", "", rev)
379
380 runtime.EXPECT().RestartContainer(mock.Anything, "c-old", mock.Anything).Return(domain.ErrContainerNotFound).Once()
381 runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(domain.ErrContainerNotFound).Once()
382 runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(domain.ErrContainerNotFound).Once()
383 expectNetworkProvision(runtime, "app-blog", 1)
384 runtime.EXPECT().CreateVolume(mock.Anything, mock.Anything, mock.Anything).Return(nil).Once()
385 runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-rebuild", Name: "web"}, nil).Once()
386 runtime.EXPECT().StartContainer(mock.Anything, "c-rebuild").Return(nil).Once()
387 runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-rebuild", mock.Anything).Return(
388 []domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18081, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
389 runtime.EXPECT().GetContainerLogs(mock.Anything, "c-rebuild", false).Return(nil, assert.AnError).Once()
390 runtime.EXPECT().RemoveContainer(mock.Anything, "c-rebuild", true).Return(nil).Once()
391
392 svc := deployment.NewService(deployment.Deps{
393 State: store, Runtime: runtime, Images: images, Secrets: secrets,
394 }, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
395 func(context.Context, string, string) (int, error) { return 503, nil },
396 func(context.Context, string) error { return assert.AnError },
397 ))
398
399 result, err := svc.Restart(ctx, "blog", "web", "op-restart-failed")
400 require.Error(t, err)
401 require.NotNil(t, result)
402 assert.Equal(t, "failed", result.Services["web"].Result)
403
404 inhibitions, loadErr := store.LoadRecoveryInhibitions(ctx, "blog")
405 require.NoError(t, loadErr)
406 assert.Empty(t, inhibitions,
407 "the inhibition of a container proven gone must not survive a failed rebuild")
408}
409
410// TestBootRecovery_RetriesLeftoverOfATerminalFailedOperation proves a
411// leftover candidate recorded by an operation that already reached a terminal
412// failure is still converged: the journal keeps the candidate ID, and the next
413// recovery pass removes it.
414func TestBootRecovery_RetriesLeftoverOfATerminalFailedOperation(t *testing.T) {
415 ctx := context.Background()
416 store := newTestStore(t)
417 images := outmocks.NewMockImageResolver(t)
418 secrets := outmocks.NewMockSecretProvider(t)
419 require.NoError(t, store.SaveIntent(ctx, domain.AppStopIntent{App: "blog", Stopped: true}))
420 require.NoError(t, store.SaveActive(ctx, domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
421 "web": {Container: "c-old", EffectiveRevision: "rev-0", Spec: headlessSpec()},
422 }}))
423 require.NoError(t, store.SaveOperation(ctx, domain.AppOperation{
424 Op: "op-failed", Kind: "deploy", App: "blog", StartedAt: time.Now().UTC(),
425 Outcome: domain.AppOutcomeFailed,
426 Steps: []domain.AppOperationStep{
427 {ID: "preflight", State: domain.AppStepSucceeded},
428 {ID: "service.web.replace", State: domain.AppStepFailed, Service: "web", Before: "c-old", After: "c-orphan", Error: "readiness"},
429 },
430 }))
431
432 runtime := outmocks.NewMockContainerRuntime(t)
433 runtime.EXPECT().RemoveContainer(mock.Anything, "c-orphan", true).Return(nil).Once()
434 runtime.EXPECT().InspectContainer(mock.Anything, "c-old").Return(&domain.Container{ID: "c-old", Status: "running"}, nil).Once()
435 runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(nil).Once()
436
437 svc := deployment.NewService(deployment.Deps{
438 State: store, Runtime: runtime, Images: images, Secrets: secrets,
439 }, zerowrap.Default())
440 require.NoError(t, svc.ReconcileBoot(ctx))
441
442 runtime.AssertCalled(t, "RemoveContainer", mock.Anything, "c-orphan", true)
443
444 // The removal is recorded: the step no longer retries the candidate on a
445 // later pass, and the removed container id stays visible for audit.
446 convergedOp, err := store.LoadOperation(ctx, "blog", "op-failed")
447 require.NoError(t, err)
448 step, found := opStep(convergedOp, "service.web.replace")
449 require.True(t, found)
450 assert.Empty(t, step.After, "a converged candidate must not be retried on the next pass")
451 assert.Contains(t, step.Detail, "c-orphan", "the removed candidate stays in the audit trail")
452}
453
454// TestBootRecovery_InterruptionAfterTheLastStepIsASuccess proves an operation
455// whose effects all ran but whose final outcome write was interrupted is
456// classified as the success it was, and the published generation is kept.
457func TestBootRecovery_InterruptionAfterTheLastStepIsASuccess(t *testing.T) {
458 ctx := context.Background()
459 store := newTestStore(t)
460 images := outmocks.NewMockImageResolver(t)
461 secrets := outmocks.NewMockSecretProvider(t)
462 require.NoError(t, store.SaveIntent(ctx, domain.AppStopIntent{App: "blog", Stopped: true}))
463 require.NoError(t, store.SaveActive(ctx, domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
464 "web": {Container: "c-published", EffectiveRevision: "rev-0", Spec: headlessSpec()},
465 }}))
466 require.NoError(t, store.SaveOperation(ctx, domain.AppOperation{
467 Op: "op-unfinished", Kind: "deploy", App: "blog", StartedAt: time.Now().UTC(),
468 Steps: []domain.AppOperationStep{
469 {ID: "preflight", State: domain.AppStepSucceeded},
470 {ID: "service.web.replace", State: domain.AppStepSucceeded, Service: "web", Before: "c-old", After: "c-published"},
471 },
472 }))
473
474 runtime := outmocks.NewMockContainerRuntime(t)
475 runtime.EXPECT().InspectContainer(mock.Anything, "c-published").Return(&domain.Container{ID: "c-published", Status: "exited"}, nil).Once()
476
477 svc := deployment.NewService(deployment.Deps{
478 State: store, Runtime: runtime, Images: images, Secrets: secrets,
479 }, zerowrap.Default())
480 require.NoError(t, svc.ReconcileBoot(ctx))
481
482 reconciled, err := store.LoadOperation(ctx, "blog", "op-unfinished")
483 require.NoError(t, err)
484 require.True(t, reconciled.Terminal())
485 assert.Equal(t, domain.AppOutcomeSuccess, reconciled.Outcome)
486 step, found := opStep(reconciled, "service.web.replace")
487 require.True(t, found)
488 assert.Equal(t, domain.AppStepSucceeded, step.State, "a step that already succeeded is left as recorded")
489 runtime.AssertNotCalled(t, "RemoveContainer", mock.Anything, "c-published", mock.Anything)
490}
491
492// TestBootRecovery_FailsClosedWhenCandidateCannotBeRemoved proves
493// reconciliation never proceeds while an orphan candidate may still run: the
494// removal failure aborts the mutation, the interrupted journal is not
495// finalized, and no newer operation is created to mask the leftover.
496func TestBootRecovery_FailsClosedWhenCandidateCannotBeRemoved(t *testing.T) {
497 ctx := context.Background()
498 store := newTestStore(t)
499 images := outmocks.NewMockImageResolver(t)
500 secrets := outmocks.NewMockSecretProvider(t)
501 seedReplaceableApp(t, ctx, store)
502 require.NoError(t, store.SaveOperation(ctx, domain.AppOperation{
503 Op: "op-crash", Kind: "deploy", App: "blog", StartedAt: time.Now().UTC(),
504 Steps: []domain.AppOperationStep{
505 {ID: "service.web.replace", State: domain.AppStepPending, Service: "web", Before: "c-old", After: "c-orphan"},
506 },
507 }))
508
509 runtime := outmocks.NewMockContainerRuntime(t)
510 runtime.EXPECT().RemoveContainer(mock.Anything, "c-orphan", true).Return(assert.AnError)
511
512 svc := deployment.NewService(deployment.Deps{
513 State: store, Runtime: runtime, Images: images, Secrets: secrets,
514 }, zerowrap.Default())
515
516 err := svc.ReconcileBoot(ctx)
517 require.Error(t, err)
518 assert.Contains(t, err.Error(), "could not be removed")
519 runtime.AssertNotCalled(t, "CreateContainer", mock.Anything, mock.Anything)
520 runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
521
522 // The leftover stays journaled and non-terminal, so it is still the latest
523 // operation.
524 leftover, err := store.LoadOperation(ctx, "blog", "op-crash")
525 require.NoError(t, err)
526 assert.False(t, leftover.Terminal(), "a leftover that could not be removed is never finalized")
527
528 // A later mutation fails closed on the same leftover and opens no journal
529 // that could mask it.
530 _, err = svc.Deploy(ctx, deployment.DeployInput{App: "blog", Op: "op-next"})
531 require.Error(t, err)
532 assert.Contains(t, err.Error(), "could not be removed")
533 _, err = store.LoadOperation(ctx, "blog", "op-next")
534 require.ErrorIs(t, err, domain.ErrAppOperationNotFound)
535}
536
537// TestBootRecovery_ClearsReplacementInhibitionAndRebuildsFromActive proves
538// the crash window is fully recoverable for a volume-owning service: an
539// interruption after the superseded container was retired leaves the
540// replacement-pending inhibition behind, boot recovery removes the
541// never-published candidate, clears that stale inhibition, and rebuilds the
542// generation ACTIVE records from its pinned revision.
543func TestBootRecovery_ClearsReplacementInhibitionAndRebuildsFromActive(t *testing.T) {
544 ctx := context.Background()
545 store := newTestStore(t)
546 images := outmocks.NewMockImageResolver(t)
547 secrets := outmocks.NewMockSecretProvider(t)
548
549 spec := webService()
550 spec.Secrets = map[string]string{}
551 spec.Volumes = []domain.AppVolume{{Name: "data", Path: "/data"}}
552 spec.Readiness = domain.AppReadiness{Type: domain.AppReadinessHTTP, Path: "/healthz", Timeout: time.Second}
553 rev := testRevision("blog", spec)
554 rev.Revision = "rev-0"
555 require.NoError(t, store.SaveIntent(ctx, domain.AppStopIntent{App: "blog"}))
556 require.NoError(t, store.SaveActive(ctx, domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
557 "web": {
558 Container: "c-old", EffectiveRevision: "rev-0", Image: spec.Image,
559 Digest: restartTestDigest, Spec: spec, BackendBinds: map[int]int{8080: 18080},
560 },
561 }}))
562 require.NoError(t, store.SaveOwnership(ctx, domain.AppOwnership{App: "blog", ID: "app-blog"}))
563 seedRevision(t, ctx, store, "intent-0", "", rev)
564 // A replacement interrupted after it retired c-old and created its
565 // candidate: ACTIVE still names c-old, the candidate is journaled but
566 // unpublished, and c-old carries the replacement-pending inhibition.
567 require.NoError(t, store.SaveOperation(ctx, domain.AppOperation{
568 Op: "op-crash", Kind: "deploy", App: "blog", StartedAt: time.Now().UTC(),
569 Steps: []domain.AppOperationStep{
570 {ID: "preflight", State: domain.AppStepSucceeded},
571 {ID: "service.web.replace", State: domain.AppStepPending, Service: "web", Before: "c-old", After: "c-orphan"},
572 },
573 }))
574 require.NoError(t, store.SaveRecoveryInhibition(ctx, domain.AppRecoveryInhibition{
575 App: "blog", Service: "web", ContainerID: "c-old",
576 Reason: domain.AppInhibitReplacementPending, Operation: "op-crash",
577 }))
578
579 runtime := outmocks.NewMockContainerRuntime(t)
580 runtime.EXPECT().RemoveContainer(mock.Anything, "c-orphan", true).Return(nil).Once()
581 runtime.EXPECT().IsContainerRunning(mock.Anything, "c-old").Return(false, nil).Once()
582 runtime.EXPECT().StartContainer(mock.Anything, "c-old").Return(domain.ErrContainerNotFound).Once()
583 runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(domain.ErrContainerNotFound).Once()
584 runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(domain.ErrContainerNotFound).Once()
585 expectNetworkProvision(runtime, "app-blog", 1)
586 runtime.EXPECT().CreateVolume(mock.Anything, mock.Anything, mock.Anything).Return(nil).Once()
587 runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-rebuilt", Name: "web"}, nil).Once()
588 runtime.EXPECT().StartContainer(mock.Anything, "c-rebuilt").Return(nil).Once()
589 runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-rebuilt", mock.Anything).Return(
590 []domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18082, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
591
592 svc := deployment.NewService(deployment.Deps{
593 State: store, Runtime: runtime, Images: images, Secrets: secrets,
594 }, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
595 func(context.Context, string, string) (int, error) { return 200, nil },
596 func(context.Context, string) error { return nil },
597 ))
598 require.NoError(t, svc.ReconcileBoot(ctx))
599
600 inhibitions, err := store.LoadRecoveryInhibitions(ctx, "blog")
601 require.NoError(t, err)
602 assert.Empty(t, inhibitions, "the stale replacement inhibition must not survive recovery")
603
604 active, ok, err := store.LoadActive(ctx, "blog")
605 require.NoError(t, err)
606 require.True(t, ok)
607 assert.Equal(t, "c-rebuilt", active.Services["web"].Container)
608 assert.Equal(t, "rev-0", active.Services["web"].EffectiveRevision)
609
610 finalized, err := store.LoadOperation(ctx, "blog", "op-crash")
611 require.NoError(t, err)
612 require.True(t, finalized.Terminal())
613 assert.Equal(t, domain.AppOutcomeFailed, finalized.Outcome)
614}
615
616// TestRestart_RebuildsMissingActiveContainer proves restart is not a dead end
617// when the recorded container is gone: the service is rebuilt from the pinned
618// ACTIVE digest and published again.
619func TestRestart_RebuildsMissingActiveContainer(t *testing.T) {
620 ctx := context.Background()
621 store := newTestStore(t)
622 runtime := outmocks.NewMockContainerRuntime(t)
623 images := outmocks.NewMockImageResolver(t)
624 secrets := outmocks.NewMockSecretProvider(t)
625 seedReplaceableApp(t, ctx, store)
626
627 runtime.EXPECT().RestartContainer(mock.Anything, "c-old", mock.Anything).Return(domain.ErrContainerNotFound).Once()
628 runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(domain.ErrContainerNotFound).Once()
629 runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(domain.ErrContainerNotFound).Once()
630 expectNetworkProvision(runtime, "app-blog", 1)
631 runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-rebuilt", Name: "web"}, nil).Once()
632 runtime.EXPECT().StartContainer(mock.Anything, "c-rebuilt").Return(nil).Once()
633 runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-rebuilt", mock.Anything).Return(
634 []domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18082, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
635
636 svc := deployment.NewService(deployment.Deps{
637 State: store, Runtime: runtime, Images: images, Secrets: secrets,
638 }, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
639 func(context.Context, string, string) (int, error) { return 200, nil },
640 func(context.Context, string) error { return nil },
641 ))
642
643 result, err := svc.Restart(ctx, "blog", "web", "op-restart-missing")
644 require.NoError(t, err)
645 require.NotNil(t, result)
646 assert.Equal(t, "deployed", result.Services["web"].Result)
647 assert.Equal(t, "c-rebuilt", result.Services["web"].After,
648 "the missing generation is rebuilt from the pinned ACTIVE digest")
649
650 active, ok, err := store.LoadActive(context.Background(), "blog")
651 require.NoError(t, err)
652 require.True(t, ok)
653 assert.Equal(t, "c-rebuilt", active.Services["web"].Container)
654 assert.Equal(t, "rev-0", active.Services["web"].EffectiveRevision,
655 "the rebuild reuses the pinned revision")
656
657 op, err := store.LoadOperation(context.Background(), "blog", "op-restart-missing")
658 require.NoError(t, err)
659 assert.Equal(t, domain.AppOutcomeSuccess, op.Outcome)
660 step, found := opStep(op, "service.web.restart")
661 require.True(t, found)
662 assert.Equal(t, domain.AppStepSucceeded, step.State)
663 assert.Equal(t, "c-rebuilt", step.After)
664}