1package deployment_test
2
3import (
4 "context"
5 "testing"
6 "time"
7
8 "github.com/bnema/zerowrap"
9 "github.com/stretchr/testify/assert"
10 "github.com/stretchr/testify/mock"
11 "github.com/stretchr/testify/require"
12
13 "github.com/bnema/gordon/internal/adapters/out/appstate"
14 outmocks "github.com/bnema/gordon/internal/boundaries/out/mocks"
15 "github.com/bnema/gordon/internal/domain"
16 "github.com/bnema/gordon/internal/usecase/deployment"
17)
18
19// seedRevision materializes one revision of an app, so a later recovery can
20// rebuild that generation from its pinned digest.
21func seedRevision(t *testing.T, ctx context.Context, store *appstate.Store, intentID, supersedes string, rev domain.AppDesiredRevision) {
22 t.Helper()
23 require.NoError(t, store.StageApply(ctx, domain.AppApplyIntent{
24 Intent: intentID, App: rev.App, Revision: rev.Revision, Supersedes: supersedes,
25 SourceSHA256: "0f1e2d", Spec: rev.Spec, CreatedAt: time.Now().UTC(),
26 }))
27 require.NoError(t, store.CommitApply(ctx, rev.App, intentID))
28 require.NoError(t, store.MaterializeApply(ctx, rev.App, intentID))
29}
30
31// seedReplaceableApp seeds one app whose ACTIVE generation (rev-0) has a
32// materialized superseding revision (rev-1). The published container c-old is
33// gone from the runtime: the caller decides what the runtime reports for it.
34func seedReplaceableApp(t *testing.T, ctx context.Context, store *appstate.Store) (domain.AppDesiredRevision, domain.AppDesiredRevision) {
35 t.Helper()
36 spec := webService()
37 spec.Secrets = map[string]string{}
38 spec.Readiness = domain.AppReadiness{Type: domain.AppReadinessHTTP, Path: "/healthz", Timeout: time.Second}
39 activeRev := testRevision("blog", spec)
40 activeRev.Revision = "rev-0"
41 require.NoError(t, store.SaveIntent(ctx, domain.AppStopIntent{App: "blog"}))
42 require.NoError(t, store.SaveActive(ctx, domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
43 "web": {
44 Container: "c-old", EffectiveRevision: "rev-0", Image: spec.Image,
45 Digest: restartTestDigest, Spec: spec, BackendBinds: map[int]int{8080: 18080},
46 },
47 }}))
48 require.NoError(t, store.SaveOwnership(ctx, domain.AppOwnership{App: "blog", ID: "app-blog"}))
49 seedRevision(t, ctx, store, "intent-0", "", activeRev)
50 desired := testRevision("blog", spec)
51 seedRevision(t, ctx, store, "intent-1", "rev-0", desired)
52 return activeRev, desired
53}
54
55// TestDeploy_CandidateIsJournaledBeforeItStarts proves the crash window is
56// closed as far as a container ID allows: the candidate ID reaches the
57// durable journal before the container is started, and the journal is
58// finalized as a success only after publication.
59func TestDeploy_CandidateIsJournaledBeforeItStarts(t *testing.T) {
60 ctx := context.Background()
61 store := newTestStore(t)
62 runtime := outmocks.NewMockContainerRuntime(t)
63 images := outmocks.NewMockImageResolver(t)
64 secrets := outmocks.NewMockSecretProvider(t)
65 seedReplaceableApp(t, ctx, store)
66
67 images.EXPECT().ResolveDigest(mock.Anything, "docker.io/example/web:1.4.2").Return(restartTestDigest, nil).Once()
68 runtime.EXPECT().InspectImageVolumes(mock.Anything, "docker.io/example/web:1.4.2").Return(nil, nil).Once()
69 expectNetworkProvision(runtime, "app-blog", 1)
70 runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(nil).Once()
71 runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
72 runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-new", Name: "web"}, nil).Once()
73
74 journaledBeforeStart := false
75 runtime.EXPECT().StartContainer(mock.Anything, "c-new").RunAndReturn(func(context.Context, string) error {
76 op, ok, err := store.LoadLatestOperation(context.Background(), "blog")
77 require.NoError(t, err)
78 require.True(t, ok, "the replacement is journaled before it starts")
79 require.False(t, op.Terminal(), "the journal is still in flight while the candidate starts")
80 step, found := opStep(op, "service.web.replace")
81 journaledBeforeStart = found && step.After == "c-new" && step.State == domain.AppStepPending
82 return nil
83 }).Once()
84 runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
85 runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return(
86 []domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18081, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
87
88 svc := deployment.NewService(deployment.Deps{
89 State: store, Runtime: runtime, Images: images, Secrets: secrets,
90 }, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
91 func(context.Context, string, string) (int, error) { return 200, nil },
92 func(context.Context, string) error { return nil },
93 ))
94
95 result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog", Op: "op-journal"})
96 require.NoError(t, err)
97 require.NotNil(t, result)
98 assert.Equal(t, "deployed", result.Services["web"].Result)
99 assert.True(t, journaledBeforeStart,
100 "the candidate ID is durable before the container is started")
101
102 op, err := store.LoadOperation(context.Background(), "blog", "op-journal")
103 require.NoError(t, err)
104 assert.Equal(t, domain.AppOutcomeSuccess, op.Outcome)
105 step, found := opStep(op, "service.web.replace")
106 require.True(t, found)
107 assert.Equal(t, domain.AppStepSucceeded, step.State)
108 assert.Equal(t, "c-new", step.After)
109}
110
111// crashDuringReplacement runs one replacement deploy and interrupts it
112// between creating the candidate and publishing it. The candidate is left
113// running, ACTIVE still names the superseded generation (which keeps running,
114// as an overlap replacement leaves it), and the operation journal stays in flight: the state a process crash or a failed cleanup
115// leaves behind.
116func crashDuringReplacement(t *testing.T, store *appstate.Store, opKey string) {
117 t.Helper()
118 ctx, cancel := context.WithCancel(context.Background())
119 defer cancel()
120 runtime := outmocks.NewMockContainerRuntime(t)
121 images := outmocks.NewMockImageResolver(t)
122 secrets := outmocks.NewMockSecretProvider(t)
123
124 images.EXPECT().ResolveDigest(mock.Anything, "docker.io/example/web:1.4.2").Return(restartTestDigest, nil).Once()
125 runtime.EXPECT().InspectImageVolumes(mock.Anything, "docker.io/example/web:1.4.2").Return(nil, nil).Once()
126 expectNetworkProvision(runtime, "app-blog", 1)
127 // The overlap replacement leaves the superseded container untouched.
128 runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-orphan", Name: "web"}, nil).Once()
129 runtime.EXPECT().StartContainer(mock.Anything, "c-orphan").Return(nil).Once()
130 runtime.EXPECT().InspectContainer(mock.Anything, "c-orphan").Return(&domain.Container{ID: "c-orphan", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
131 runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-orphan", mock.Anything).Return(
132 []domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18081, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
133 // The canceled context also refuses the candidate cleanup, so the
134 // candidate outlives the operation exactly as a crash would leave it.
135 runtime.EXPECT().RemoveContainer(mock.Anything, "c-orphan", true).RunAndReturn(
136 func(removeCtx context.Context, _ string, _ bool) error {
137 return removeCtx.Err()
138 }).Once()
139
140 probeStarted := make(chan struct{})
141 svc := deployment.NewService(deployment.Deps{
142 State: store, Runtime: runtime, Images: images, Secrets: secrets,
143 }, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
144 func(probeCtx context.Context, _, _ string) (int, error) {
145 select {
146 case <-probeStarted:
147 default:
148 close(probeStarted)
149 }
150 <-probeCtx.Done()
151 return 0, probeCtx.Err()
152 },
153 func(context.Context, string) error { return nil },
154 ))
155 done := make(chan error, 1)
156 go func() {
157 _, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog", Op: opKey})
158 done <- err
159 }()
160 select {
161 case <-probeStarted:
162 case <-time.After(5 * time.Second):
163 t.Fatal("timed out waiting for the readiness probe")
164 }
165 cancel()
166 require.Error(t, <-done)
167}
168
169// TestBootRecovery_RemovesUnpublishedCandidateAndKeepsServingGeneration
170// proves the crash window between creating an overlap replacement and
171// publishing it is recoverable: an interruption leaves the candidate
172// journaled but unpublished, and boot recovery removes that candidate and
173// keeps the generation recorded in ACTIVE, which was never stopped.
174func TestBootRecovery_RemovesUnpublishedCandidateAndKeepsServingGeneration(t *testing.T) {
175 ctx := context.Background()
176 store := newTestStore(t)
177 images := outmocks.NewMockImageResolver(t)
178 secrets := outmocks.NewMockSecretProvider(t)
179 seedReplaceableApp(t, ctx, store)
180 crashDuringReplacement(t, store, "op-crash")
181
182 // The crash left the candidate durable but unpublished, and ACTIVE still
183 // names the superseded generation.
184 interrupted, err := store.LoadOperation(context.Background(), "blog", "op-crash")
185 require.NoError(t, err)
186 assert.False(t, interrupted.Terminal(), "an interrupted operation stays in flight until recovery finalizes it")
187 crashStep, found := opStep(interrupted, "service.web.replace")
188 require.True(t, found)
189 assert.Equal(t, domain.AppStepPending, crashStep.State)
190 assert.Equal(t, "c-orphan", crashStep.After, "the created candidate is durably recorded")
191 active, ok, err := store.LoadActive(context.Background(), "blog")
192 require.NoError(t, err)
193 require.True(t, ok)
194 assert.Equal(t, "c-old", active.Services["web"].Container)
195
196 // Boot recovery: the orphan is removed; the published generation is
197 // still running, so it is verified and kept, never rebuilt or stopped.
198 var order []string
199 runtime2 := outmocks.NewMockContainerRuntime(t)
200 runtime2.EXPECT().RemoveContainer(mock.Anything, "c-orphan", true).RunAndReturn(
201 func(context.Context, string, bool) error {
202 order = append(order, "remove-orphan")
203 return nil
204 }).Once()
205 runtime2.EXPECT().IsContainerRunning(mock.Anything, "c-old").RunAndReturn(
206 func(context.Context, string) (bool, error) {
207 order = append(order, "verify-published-generation")
208 return true, nil
209 }).Once()
210 runtime2.EXPECT().InspectContainer(mock.Anything, "c-old").Return(&domain.Container{ID: "c-old", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
211 runtime2.EXPECT().GetContainerBackendBinds(mock.Anything, "c-old", mock.Anything).Return(
212 []domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18080, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
213
214 rebooted := deployment.NewService(deployment.Deps{
215 State: store, Runtime: runtime2, Images: images, Secrets: secrets,
216 }, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime2,
217 func(context.Context, string, string) (int, error) { return 200, nil },
218 func(context.Context, string) error { return nil },
219 ))
220 // Boot recovery runs on a fresh process context: the crashed deploy's
221 // context is gone with it.
222 require.NoError(t, rebooted.ReconcileBoot(ctx))
223
224 assert.Equal(t, []string{"remove-orphan", "verify-published-generation"}, order,
225 "the unpublished candidate is gone and the serving generation is kept")
226 runtime2.AssertNotCalled(t, "StopContainer", mock.Anything, "c-old", mock.Anything)
227 runtime2.AssertNotCalled(t, "RemoveContainer", mock.Anything, "c-old", mock.Anything)
228 runtime2.AssertNotCalled(t, "CreateContainer", mock.Anything, mock.Anything)
229 runtime2.AssertNotCalled(t, "StartContainer", mock.Anything, "c-orphan")
230 runtime2.AssertNotCalled(t, "RestartContainer", mock.Anything, "c-orphan", mock.Anything)
231
232 // The interrupted operation is finalized as failed, so a repeat of its key
233 // replays an explicit failure instead of a permanent in-flight claim.
234 finalized, err := store.LoadOperation(context.Background(), "blog", "op-crash")
235 require.NoError(t, err)
236 require.True(t, finalized.Terminal(), "boot recovery finalizes the interrupted operation")
237 assert.Equal(t, domain.AppOutcomeFailed, finalized.Outcome)
238 finalStep, found := opStep(finalized, "service.web.replace")
239 require.True(t, found)
240 assert.Equal(t, domain.AppStepFailed, finalStep.State)
241 assert.Contains(t, finalStep.Error, "unpublished candidate was removed")
242
243 // ACTIVE still names the generation that kept serving.
244 converged, ok, err := store.LoadActive(context.Background(), "blog")
245 require.NoError(t, err)
246 require.True(t, ok)
247 assert.Equal(t, "c-old", converged.Services["web"].Container)
248 assert.Equal(t, "rev-0", converged.Services["web"].EffectiveRevision)
249}
250
251// TestDeploy_ReconcilesInterruptedPredecessorBeforeCreatingAReplacement
252// proves the live case: when a cleanup failed and Gordon kept running, the
253// next mutation of the app removes the leftover candidate before it creates a
254// new generation, so two generations of one service never run together even
255// without a restart. It also finalizes the interrupted journal.
256func TestDeploy_ReconcilesInterruptedPredecessorBeforeCreatingAReplacement(t *testing.T) {
257 ctx := context.Background()
258 store := newTestStore(t)
259 images := outmocks.NewMockImageResolver(t)
260 secrets := outmocks.NewMockSecretProvider(t)
261 seedReplaceableApp(t, ctx, store)
262 crashDuringReplacement(t, store, "op-crash")
263
264 var order []string
265 runtime := outmocks.NewMockContainerRuntime(t)
266 runtime.EXPECT().RemoveContainer(mock.Anything, "c-orphan", true).RunAndReturn(
267 func(context.Context, string, bool) error {
268 order = append(order, "remove-leftover-candidate")
269 return nil
270 }).Once()
271 images.EXPECT().ResolveDigest(mock.Anything, "docker.io/example/web:1.4.2").Return(restartTestDigest, nil).Once()
272 runtime.EXPECT().InspectImageVolumes(mock.Anything, "docker.io/example/web:1.4.2").Return(nil, nil).Once()
273 expectNetworkProvision(runtime, "app-blog", 1)
274 runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).RunAndReturn(
275 func(context.Context, *domain.ContainerConfig) (*domain.Container, error) {
276 order = append(order, "create-new-generation")
277 return &domain.Container{ID: "c-next", Name: "web"}, nil
278 }).Once()
279 // The overlap replacement retires the superseded container only after
280 // the new generation is published.
281 runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).RunAndReturn(
282 func(context.Context, string, time.Duration) error {
283 order = append(order, "retire-superseded")
284 return nil
285 }).Once()
286 runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
287 runtime.EXPECT().StartContainer(mock.Anything, "c-next").Return(nil).Once()
288 runtime.EXPECT().InspectContainer(mock.Anything, "c-next").Return(&domain.Container{ID: "c-next", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
289 runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-next", mock.Anything).Return(
290 []domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18082, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
291
292 svc := deployment.NewService(deployment.Deps{
293 State: store, Runtime: runtime, Images: images, Secrets: secrets,
294 }, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
295 func(context.Context, string, string) (int, error) { return 200, nil },
296 func(context.Context, string) error { return nil },
297 ))
298
299 result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog", Op: "op-next"})
300 require.NoError(t, err)
301 require.NotNil(t, result)
302 assert.Equal(t, "deployed", result.Services["web"].Result)
303 assert.Equal(t, []string{"remove-leftover-candidate", "create-new-generation", "retire-superseded"}, order)
304
305 active, ok, err := store.LoadActive(ctx, "blog")
306 require.NoError(t, err)
307 require.True(t, ok)
308 assert.Equal(t, "c-next", active.Services["web"].Container)
309
310 interrupted, err := store.LoadOperation(ctx, "blog", "op-crash")
311 require.NoError(t, err)
312 require.True(t, interrupted.Terminal(), "the interrupted operation is finalized")
313 assert.Equal(t, domain.AppOutcomeFailed, interrupted.Outcome)
314 step, found := opStep(interrupted, "service.web.replace")
315 require.True(t, found)
316 assert.Equal(t, domain.AppStepFailed, step.State)
317}
318
319// TestBootRecovery_RemovesCandidateOfAnInterruptedFirstDeployment proves the
320// same recovery works when the interrupted operation is the app's FIRST
321// deployment: there is no ACTIVE generation to converge, and the leftover
322// candidate is still removed and the journal finalized.
323func TestBootRecovery_RemovesCandidateOfAnInterruptedFirstDeployment(t *testing.T) {
324 ctx := context.Background()
325 store := newTestStore(t)
326 images := outmocks.NewMockImageResolver(t)
327 secrets := outmocks.NewMockSecretProvider(t)
328 require.NoError(t, store.SaveIntent(ctx, domain.AppStopIntent{App: "blog"}))
329 require.NoError(t, store.SaveOperation(ctx, domain.AppOperation{
330 Op: "op-seeded", Kind: "deploy", App: "blog", StartedAt: time.Now().UTC(),
331 Steps: []domain.AppOperationStep{
332 {ID: "preflight", State: domain.AppStepSucceeded},
333 {ID: "service.web.replace", State: domain.AppStepPending, Service: "web", Before: "c-old", After: "c-orphan"},
334 },
335 }))
336
337 runtime := outmocks.NewMockContainerRuntime(t)
338 runtime.EXPECT().RemoveContainer(mock.Anything, "c-orphan", true).Return(nil).Once()
339
340 svc := deployment.NewService(deployment.Deps{
341 State: store, Runtime: runtime, Images: images, Secrets: secrets,
342 }, zerowrap.Default())
343 require.NoError(t, svc.ReconcileBoot(ctx))
344
345 runtime.AssertNotCalled(t, "CreateContainer", mock.Anything, mock.Anything)
346 runtime.AssertNotCalled(t, "StartContainer", mock.Anything, "c-orphan")
347
348 finalized, err := store.LoadOperation(ctx, "blog", "op-seeded")
349 require.NoError(t, err)
350 require.True(t, finalized.Terminal())
351 assert.Equal(t, domain.AppOutcomeFailed, finalized.Outcome)
352 step, found := opStep(finalized, "service.web.replace")
353 require.True(t, found)
354 assert.Equal(t, domain.AppStepFailed, step.State)
355 assert.Contains(t, step.Error, "unpublished candidate was removed")
356}
357
358// TestRestart_FailedRebuildClearsTheInhibitionOfTheMissingContainer proves a
359// restart that falls back to a rebuild does not leave a durable inhibition for
360// a container that is already gone: nothing could ever revive it, and the
361// marker would refuse every later start, recovery pass, and restart.
362func TestRestart_FailedRebuildClearsTheInhibitionOfTheMissingContainer(t *testing.T) {
363 ctx := context.Background()
364 store := newTestStore(t)
365 images := outmocks.NewMockImageResolver(t)
366 secrets := outmocks.NewMockSecretProvider(t)
367 runtime := outmocks.NewMockContainerRuntime(t)
368
369 spec := webService()
370 spec.Secrets = map[string]string{}
371 spec.Volumes = []domain.AppVolume{{Name: "data", Path: "/data"}}
372 spec.Readiness = domain.AppReadiness{Type: domain.AppReadinessHTTP, Path: "/healthz", Timeout: 50 * time.Millisecond}
373 rev := testRevision("blog", spec)
374 rev.Revision = "rev-0"
375 require.NoError(t, store.SaveIntent(ctx, domain.AppStopIntent{App: "blog"}))
376 require.NoError(t, store.SaveActive(ctx, domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
377 "web": {
378 Container: "c-old", EffectiveRevision: "rev-0", Image: spec.Image,
379 Digest: restartTestDigest, Spec: spec, BackendBinds: map[int]int{8080: 18080},
380 },
381 }}))
382 require.NoError(t, store.SaveOwnership(ctx, domain.AppOwnership{App: "blog", ID: "app-blog"}))
383 seedRevision(t, ctx, store, "intent-0", "", rev)
384
385 runtime.EXPECT().RestartContainer(mock.Anything, "c-old", mock.Anything).Return(domain.ErrContainerNotFound).Once()
386 runtime.EXPECT().InspectContainer(mock.Anything, "c-old").Return(&domain.Container{ID: "c-old", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
387 runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(domain.ErrContainerNotFound).Once()
388 runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(domain.ErrContainerNotFound).Once()
389 expectNetworkProvision(runtime, "app-blog", 1)
390 runtime.EXPECT().CreateVolume(mock.Anything, mock.Anything, mock.Anything).Return(nil).Once()
391 runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-rebuild", Name: "web"}, nil).Once()
392 runtime.EXPECT().StartContainer(mock.Anything, "c-rebuild").Return(nil).Once()
393 runtime.EXPECT().InspectContainer(mock.Anything, "c-rebuild").Return(&domain.Container{ID: "c-rebuild", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
394 runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-rebuild", mock.Anything).Return(
395 []domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18081, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
396 runtime.EXPECT().GetContainerLogs(mock.Anything, "c-rebuild", false).Return(nil, assert.AnError).Once()
397 runtime.EXPECT().RemoveContainer(mock.Anything, "c-rebuild", true).Return(nil).Once()
398
399 svc := deployment.NewService(deployment.Deps{
400 State: store, Runtime: runtime, Images: images, Secrets: secrets,
401 }, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
402 func(context.Context, string, string) (int, error) { return 503, nil },
403 func(context.Context, string) error { return assert.AnError },
404 ))
405
406 result, err := svc.Restart(ctx, "blog", "web", "op-restart-failed")
407 require.Error(t, err)
408 require.NotNil(t, result)
409 assert.Equal(t, "failed", result.Services["web"].Result)
410
411 inhibitions, loadErr := store.LoadRecoveryInhibitions(ctx, "blog")
412 require.NoError(t, loadErr)
413 assert.Empty(t, inhibitions,
414 "the inhibition of a container proven gone must not survive a failed rebuild")
415}
416
417// TestBootRecovery_RetriesLeftoverOfATerminalFailedOperation proves a
418// leftover candidate recorded by an operation that already reached a terminal
419// failure is still converged: the journal keeps the candidate ID, and the next
420// recovery pass removes it.
421func TestBootRecovery_RetriesLeftoverOfATerminalFailedOperation(t *testing.T) {
422 ctx := context.Background()
423 store := newTestStore(t)
424 images := outmocks.NewMockImageResolver(t)
425 secrets := outmocks.NewMockSecretProvider(t)
426 require.NoError(t, store.SaveIntent(ctx, domain.AppStopIntent{App: "blog", Stopped: true}))
427 require.NoError(t, store.SaveActive(ctx, domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
428 "web": {Container: "c-old", EffectiveRevision: "rev-0", Spec: headlessSpec()},
429 }}))
430 require.NoError(t, store.SaveOperation(ctx, domain.AppOperation{
431 Op: "op-failed", Kind: "deploy", App: "blog", StartedAt: time.Now().UTC(),
432 Outcome: domain.AppOutcomeFailed,
433 Steps: []domain.AppOperationStep{
434 {ID: "preflight", State: domain.AppStepSucceeded},
435 {ID: "service.web.replace", State: domain.AppStepFailed, Service: "web", Before: "c-old", After: "c-orphan", Error: "readiness"},
436 },
437 }))
438
439 runtime := outmocks.NewMockContainerRuntime(t)
440 runtime.EXPECT().RemoveContainer(mock.Anything, "c-orphan", true).Return(nil).Once()
441 runtime.EXPECT().InspectContainer(mock.Anything, "c-old").Return(&domain.Container{ID: "c-old", Status: "running"}, nil).Once()
442 runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(nil).Once()
443
444 svc := deployment.NewService(deployment.Deps{
445 State: store, Runtime: runtime, Images: images, Secrets: secrets,
446 }, zerowrap.Default())
447 require.NoError(t, svc.ReconcileBoot(ctx))
448
449 runtime.AssertCalled(t, "RemoveContainer", mock.Anything, "c-orphan", true)
450
451 // The removal is recorded: the step no longer retries the candidate on a
452 // later pass, and the removed container id stays visible for audit.
453 convergedOp, err := store.LoadOperation(ctx, "blog", "op-failed")
454 require.NoError(t, err)
455 step, found := opStep(convergedOp, "service.web.replace")
456 require.True(t, found)
457 assert.Empty(t, step.After, "a converged candidate must not be retried on the next pass")
458 assert.Contains(t, step.Detail, "c-orphan", "the removed candidate stays in the audit trail")
459}
460
461// TestBootRecovery_InterruptionAfterTheLastStepIsASuccess proves an operation
462// whose effects all ran but whose final outcome write was interrupted is
463// classified as the success it was, and the published generation is kept.
464func TestBootRecovery_InterruptionAfterTheLastStepIsASuccess(t *testing.T) {
465 ctx := context.Background()
466 store := newTestStore(t)
467 images := outmocks.NewMockImageResolver(t)
468 secrets := outmocks.NewMockSecretProvider(t)
469 require.NoError(t, store.SaveIntent(ctx, domain.AppStopIntent{App: "blog", Stopped: true}))
470 require.NoError(t, store.SaveActive(ctx, domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
471 "web": {Container: "c-published", EffectiveRevision: "rev-0", Spec: headlessSpec()},
472 }}))
473 require.NoError(t, store.SaveOperation(ctx, domain.AppOperation{
474 Op: "op-unfinished", Kind: "deploy", App: "blog", StartedAt: time.Now().UTC(),
475 Steps: []domain.AppOperationStep{
476 {ID: "preflight", State: domain.AppStepSucceeded},
477 {ID: "service.web.replace", State: domain.AppStepSucceeded, Service: "web", Before: "c-old", After: "c-published"},
478 },
479 }))
480
481 runtime := outmocks.NewMockContainerRuntime(t)
482 runtime.EXPECT().InspectContainer(mock.Anything, "c-published").Return(&domain.Container{ID: "c-published", Status: "exited"}, nil).Once()
483
484 svc := deployment.NewService(deployment.Deps{
485 State: store, Runtime: runtime, Images: images, Secrets: secrets,
486 }, zerowrap.Default())
487 require.NoError(t, svc.ReconcileBoot(ctx))
488
489 reconciled, err := store.LoadOperation(ctx, "blog", "op-unfinished")
490 require.NoError(t, err)
491 require.True(t, reconciled.Terminal())
492 assert.Equal(t, domain.AppOutcomeSuccess, reconciled.Outcome)
493 step, found := opStep(reconciled, "service.web.replace")
494 require.True(t, found)
495 assert.Equal(t, domain.AppStepSucceeded, step.State, "a step that already succeeded is left as recorded")
496 runtime.AssertNotCalled(t, "RemoveContainer", mock.Anything, "c-published", mock.Anything)
497}
498
499// TestBootRecovery_FailsClosedWhenCandidateCannotBeRemoved proves
500// reconciliation never proceeds while an orphan candidate may still run: the
501// removal failure aborts the mutation, the interrupted journal is not
502// finalized, and no newer operation is created to mask the leftover.
503func TestBootRecovery_FailsClosedWhenCandidateCannotBeRemoved(t *testing.T) {
504 ctx := context.Background()
505 store := newTestStore(t)
506 images := outmocks.NewMockImageResolver(t)
507 secrets := outmocks.NewMockSecretProvider(t)
508 seedReplaceableApp(t, ctx, store)
509 require.NoError(t, store.SaveOperation(ctx, domain.AppOperation{
510 Op: "op-crash", Kind: "deploy", App: "blog", StartedAt: time.Now().UTC(),
511 Steps: []domain.AppOperationStep{
512 {ID: "service.web.replace", State: domain.AppStepPending, Service: "web", Before: "c-old", After: "c-orphan"},
513 },
514 }))
515
516 runtime := outmocks.NewMockContainerRuntime(t)
517 runtime.EXPECT().RemoveContainer(mock.Anything, "c-orphan", true).Return(assert.AnError)
518
519 svc := deployment.NewService(deployment.Deps{
520 State: store, Runtime: runtime, Images: images, Secrets: secrets,
521 }, zerowrap.Default())
522
523 err := svc.ReconcileBoot(ctx)
524 require.Error(t, err)
525 assert.Contains(t, err.Error(), "could not be removed")
526 runtime.AssertNotCalled(t, "CreateContainer", mock.Anything, mock.Anything)
527 runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
528
529 // The leftover stays journaled and non-terminal, so it is still the latest
530 // operation.
531 leftover, err := store.LoadOperation(ctx, "blog", "op-crash")
532 require.NoError(t, err)
533 assert.False(t, leftover.Terminal(), "a leftover that could not be removed is never finalized")
534
535 // A later mutation fails closed on the same leftover and opens no journal
536 // that could mask it.
537 _, err = svc.Deploy(ctx, deployment.DeployInput{App: "blog", Op: "op-next"})
538 require.Error(t, err)
539 assert.Contains(t, err.Error(), "could not be removed")
540 _, err = store.LoadOperation(ctx, "blog", "op-next")
541 require.ErrorIs(t, err, domain.ErrAppOperationNotFound)
542}
543
544// TestBootRecovery_ClearsReplacementInhibitionAndRebuildsFromActive proves
545// the crash window is fully recoverable for a volume-owning service: an
546// interruption after the superseded container was retired leaves the
547// replacement-pending inhibition behind, boot recovery removes the
548// never-published candidate, clears that stale inhibition, and rebuilds the
549// generation ACTIVE records from its pinned revision.
550func TestBootRecovery_ClearsReplacementInhibitionAndRebuildsFromActive(t *testing.T) {
551 ctx := context.Background()
552 store := newTestStore(t)
553 images := outmocks.NewMockImageResolver(t)
554 secrets := outmocks.NewMockSecretProvider(t)
555
556 spec := webService()
557 spec.Secrets = map[string]string{}
558 spec.Volumes = []domain.AppVolume{{Name: "data", Path: "/data"}}
559 spec.Readiness = domain.AppReadiness{Type: domain.AppReadinessHTTP, Path: "/healthz", Timeout: time.Second}
560 rev := testRevision("blog", spec)
561 rev.Revision = "rev-0"
562 require.NoError(t, store.SaveIntent(ctx, domain.AppStopIntent{App: "blog"}))
563 require.NoError(t, store.SaveActive(ctx, domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
564 "web": {
565 Container: "c-old", EffectiveRevision: "rev-0", Image: spec.Image,
566 Digest: restartTestDigest, Spec: spec, BackendBinds: map[int]int{8080: 18080},
567 },
568 }}))
569 require.NoError(t, store.SaveOwnership(ctx, domain.AppOwnership{App: "blog", ID: "app-blog"}))
570 seedRevision(t, ctx, store, "intent-0", "", rev)
571 // A replacement interrupted after it retired c-old and created its
572 // candidate: ACTIVE still names c-old, the candidate is journaled but
573 // unpublished, and c-old carries the replacement-pending inhibition.
574 require.NoError(t, store.SaveOperation(ctx, domain.AppOperation{
575 Op: "op-crash", Kind: "deploy", App: "blog", StartedAt: time.Now().UTC(),
576 Steps: []domain.AppOperationStep{
577 {ID: "preflight", State: domain.AppStepSucceeded},
578 {ID: "service.web.replace", State: domain.AppStepPending, Service: "web", Before: "c-old", After: "c-orphan"},
579 },
580 }))
581 require.NoError(t, store.SaveRecoveryInhibition(ctx, domain.AppRecoveryInhibition{
582 App: "blog", Service: "web", ContainerID: "c-old",
583 Reason: domain.AppInhibitReplacementPending, Operation: "op-crash",
584 }))
585
586 runtime := outmocks.NewMockContainerRuntime(t)
587 runtime.EXPECT().RemoveContainer(mock.Anything, "c-orphan", true).Return(nil).Once()
588 runtime.EXPECT().IsContainerRunning(mock.Anything, "c-old").Return(false, nil).Once()
589 runtime.EXPECT().StartContainer(mock.Anything, "c-old").Return(domain.ErrContainerNotFound).Once()
590 runtime.EXPECT().InspectContainer(mock.Anything, "c-old").Return(&domain.Container{ID: "c-old", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
591 runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(domain.ErrContainerNotFound).Once()
592 runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(domain.ErrContainerNotFound).Once()
593 expectNetworkProvision(runtime, "app-blog", 1)
594 runtime.EXPECT().CreateVolume(mock.Anything, mock.Anything, mock.Anything).Return(nil).Once()
595 runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-rebuilt", Name: "web"}, nil).Once()
596 runtime.EXPECT().StartContainer(mock.Anything, "c-rebuilt").Return(nil).Once()
597 runtime.EXPECT().InspectContainer(mock.Anything, "c-rebuilt").Return(&domain.Container{ID: "c-rebuilt", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
598 runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-rebuilt", mock.Anything).Return(
599 []domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18082, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
600
601 svc := deployment.NewService(deployment.Deps{
602 State: store, Runtime: runtime, Images: images, Secrets: secrets,
603 }, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
604 func(context.Context, string, string) (int, error) { return 200, nil },
605 func(context.Context, string) error { return nil },
606 ))
607 require.NoError(t, svc.ReconcileBoot(ctx))
608
609 inhibitions, err := store.LoadRecoveryInhibitions(ctx, "blog")
610 require.NoError(t, err)
611 assert.Empty(t, inhibitions, "the stale replacement inhibition must not survive recovery")
612
613 active, ok, err := store.LoadActive(ctx, "blog")
614 require.NoError(t, err)
615 require.True(t, ok)
616 assert.Equal(t, "c-rebuilt", active.Services["web"].Container)
617 assert.Equal(t, "rev-0", active.Services["web"].EffectiveRevision)
618
619 finalized, err := store.LoadOperation(ctx, "blog", "op-crash")
620 require.NoError(t, err)
621 require.True(t, finalized.Terminal())
622 assert.Equal(t, domain.AppOutcomeFailed, finalized.Outcome)
623}
624
625// TestRestart_RebuildsMissingActiveContainer proves restart is not a dead end
626// when the recorded container is gone: the service is rebuilt from the pinned
627// ACTIVE digest and published again.
628func TestRestart_RebuildsMissingActiveContainer(t *testing.T) {
629 ctx := context.Background()
630 store := newTestStore(t)
631 runtime := outmocks.NewMockContainerRuntime(t)
632 images := outmocks.NewMockImageResolver(t)
633 secrets := outmocks.NewMockSecretProvider(t)
634 seedReplaceableApp(t, ctx, store)
635
636 runtime.EXPECT().RestartContainer(mock.Anything, "c-old", mock.Anything).Return(domain.ErrContainerNotFound).Once()
637 runtime.EXPECT().InspectContainer(mock.Anything, "c-old").Return(&domain.Container{ID: "c-old", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
638 runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(domain.ErrContainerNotFound).Once()
639 runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(domain.ErrContainerNotFound).Once()
640 expectNetworkProvision(runtime, "app-blog", 1)
641 runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-rebuilt", Name: "web"}, nil).Once()
642 runtime.EXPECT().StartContainer(mock.Anything, "c-rebuilt").Return(nil).Once()
643 runtime.EXPECT().InspectContainer(mock.Anything, "c-rebuilt").Return(&domain.Container{ID: "c-rebuilt", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
644 runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-rebuilt", mock.Anything).Return(
645 []domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18082, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
646
647 svc := deployment.NewService(deployment.Deps{
648 State: store, Runtime: runtime, Images: images, Secrets: secrets,
649 }, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
650 func(context.Context, string, string) (int, error) { return 200, nil },
651 func(context.Context, string) error { return nil },
652 ))
653
654 result, err := svc.Restart(ctx, "blog", "web", "op-restart-missing")
655 require.NoError(t, err)
656 require.NotNil(t, result)
657 assert.Equal(t, "deployed", result.Services["web"].Result)
658 assert.Equal(t, "c-rebuilt", result.Services["web"].After,
659 "the missing generation is rebuilt from the pinned ACTIVE digest")
660
661 active, ok, err := store.LoadActive(context.Background(), "blog")
662 require.NoError(t, err)
663 require.True(t, ok)
664 assert.Equal(t, "c-rebuilt", active.Services["web"].Container)
665 assert.Equal(t, "rev-0", active.Services["web"].EffectiveRevision,
666 "the rebuild reuses the pinned revision")
667
668 op, err := store.LoadOperation(context.Background(), "blog", "op-restart-missing")
669 require.NoError(t, err)
670 assert.Equal(t, domain.AppOutcomeSuccess, op.Outcome)
671 step, found := opStep(op, "service.web.restart")
672 require.True(t, found)
673 assert.Equal(t, domain.AppStepSucceeded, step.State)
674 assert.Equal(t, "c-rebuilt", step.After)
675}