1package deployment_test
2
3import (
4 "context"
5 "maps"
6 "os"
7 "path/filepath"
8 "sync"
9 "testing"
10 "time"
11
12 "github.com/bnema/zerowrap"
13 "github.com/stretchr/testify/assert"
14 "github.com/stretchr/testify/mock"
15 "github.com/stretchr/testify/require"
16
17 "github.com/bnema/gordon/internal/adapters/out/appstate"
18 outmocks "github.com/bnema/gordon/internal/boundaries/out/mocks"
19 "github.com/bnema/gordon/internal/domain"
20 "github.com/bnema/gordon/internal/usecase/deployment"
21)
22
23// TestDeploy_NonOverlapEligibleServicesKeepSequentialOrder proves a service
24// with a volume, a bind, or a device is never run beside its predecessor: the
25// service is withdrawn, the superseded container is stopped and removed, and
26// only then is the replacement created.
27func TestDeploy_NonOverlapEligibleServicesKeepSequentialOrder(t *testing.T) {
28 root := t.TempDir()
29 source := filepath.Join(root, "binds", "config")
30 require.NoError(t, os.MkdirAll(source, 0o755))
31
32 cases := []struct {
33 name string
34 mutate func(*domain.AppService)
35 inhibits bool
36 cdiSupported bool
37 }{
38 {"volume", func(s *domain.AppService) { s.Volumes = []domain.AppVolume{{Name: "data", Path: "/data"}} }, true, false},
39 {"bind", func(s *domain.AppService) {
40 s.Binds = []domain.AppBind{{Name: "config", Path: "/etc/app.conf", ReadOnly: true}}
41 }, true, false},
42 {"device", func(s *domain.AppService) { s.Devices = []string{"test_gpu"} }, false, true},
43 {"udp", func(s *domain.AppService) {
44 s.UDP = []domain.AppUDPInterface{{Entrypoint: "game", Port: 28015, Publish: "0.0.0.0:28015"}}
45 }, false, false},
46 }
47 for _, tc := range cases {
48 t.Run(tc.name, func(t *testing.T) {
49 ctx := context.Background()
50 state, runtime, images, secrets := mockDeps(t)
51 rev := mockRevision()
52 tc.mutate(&rev.Spec.Services[0])
53
54 expectDeployPreflight(state, runtime, images, secrets, rev)
55 runtime.EXPECT().VolumeExists(mock.Anything, mock.Anything).Return(false, nil).Maybe()
56 runtime.EXPECT().CreateVolume(mock.Anything, mock.Anything, mock.Anything).Return(nil).Maybe()
57 if tc.cdiSupported {
58 runtime.EXPECT().SupportsCDIDevices(mock.Anything).Return(nil)
59 }
60 expectNetworkProvision(runtime, "app-blog", 1)
61 state.EXPECT().LoadActive(mock.Anything, "blog").Return(replaceableActive(), true, nil)
62 state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
63 if tc.inhibits {
64 state.EXPECT().SaveRecoveryInhibition(mock.Anything, mock.Anything).Return(nil).Once()
65 }
66
67 var order []string
68 expectRetireOld(state, runtime, &order, "c-old")
69 expectSequentialCreate(state, runtime, &order, "c-new", 18080)
70 state.EXPECT().SaveActive(mock.Anything, mock.Anything).RunAndReturn(
71 func(context.Context, domain.AppActive) error {
72 recordEvent(&order, "publish-active")
73 return nil
74 }).Once()
75 state.EXPECT().SaveOwnership(mock.Anything, mock.Anything).Return(nil)
76 state.EXPECT().ClearRecoveryInhibition(mock.Anything, "blog", "web", "c-old").Return(nil).Once()
77
78 svc := deployment.NewService(deployment.Deps{
79 State: state, Runtime: runtime, Images: images, Secrets: secrets,
80 Traffic: &orderedTraffic{events: &order},
81 }, zerowrap.Default()).
82 WithBindPolicies(map[string]domain.AppBindPolicy{"config": {
83 Name: "config", Source: source, Root: root, AllowedApps: []string{"blog"}, AllowedServices: []string{"web"},
84 }}).
85 WithDevicePolicies(map[string]domain.AppDevicePolicy{"test_gpu": {
86 Name: "test_gpu", CDI: []string{"example.com/gpu=GPU-test-uuid"},
87 AllowedApps: []string{"blog"}, AllowedServices: []string{"web"},
88 }}).
89 WithProbeDeps(deployment.NewTestProbeDeps(runtime,
90 func(context.Context, string, string) (int, error) { return 200, nil },
91 func(context.Context, string) error { return nil },
92 ))
93
94 result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
95 require.NoError(t, err)
96 assert.Equal(t, "deployed", result.Services["web"].Result)
97 assert.Equal(t,
98 []string{"withdraw", "stop-old", "remove-old", "create-new", "start-new", "publish-active", "traffic"},
99 order, "the superseded container is gone before the replacement is created")
100 })
101 }
102}
103
104// TestDeploy_OverlapPublicationFailureKeepsOldContainer proves a failed ACTIVE
105// publication of an overlap replacement never stops the superseded container,
106// which still serves because ACTIVE and routing were not switched. The
107// candidate stays journaled as the step's After, so reconciliation removes it,
108// and the failure is reported with a leftover warning on the old container.
109func TestDeploy_OverlapPublicationFailureKeepsOldContainer(t *testing.T) {
110 ctx := context.Background()
111 state, runtime, images, secrets := mockDeps(t)
112 rev := mockRevision()
113
114 expectDeployPreflight(state, runtime, images, secrets, rev)
115 expectNetworkProvision(runtime, "app-blog", 1)
116 state.EXPECT().LoadActive(mock.Anything, "blog").Return(replaceableActive(), true, nil)
117 state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
118 expectSequentialCreate(state, runtime, nil, "c-new", 18080)
119 expectRetirementMarker(state, "c-old")
120 state.EXPECT().SaveActive(mock.Anything, mock.Anything).Return(assert.AnError).Once()
121
122 var events []string
123 svc := deployment.NewService(deployment.Deps{
124 State: state, Runtime: runtime, Images: images, Secrets: secrets,
125 Traffic: &orderedTraffic{events: &events},
126 }, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
127 func(context.Context, string, string) (int, error) { return 200, nil },
128 func(context.Context, string) error { return nil },
129 ))
130
131 result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
132 require.Error(t, err)
133 assert.Equal(t, "failed", result.Services["web"].Result)
134 assert.Equal(t, "c-new", result.Services["web"].After)
135 require.Len(t, result.Services["web"].CleanupWarnings, 1)
136 assert.Equal(t, "c-old", result.Services["web"].CleanupWarnings[0].Leftover)
137 assert.Empty(t, events, "the proxy graph was neither withdrawn nor rebuilt")
138 runtime.AssertNotCalled(t, "StopContainer", mock.Anything, mock.Anything, mock.Anything)
139 runtime.AssertNotCalled(t, "RemoveContainer", mock.Anything, mock.Anything, mock.Anything)
140 state.AssertNotCalled(t, "ReleaseBackendBinds", mock.Anything, mock.Anything, mock.Anything)
141}
142
143// storeTraffic is a traffic publisher that behaves like the real one against
144// a real app-state store: a withdrawal drops the service's recorded binds from
145// ACTIVE (and can then report a failed graph apply), and a rebuild routes
146// exactly the binds ACTIVE records at that moment. Tests assert on what is
147// routed, not on call order.
148type storeTraffic struct {
149 mu sync.Mutex
150 store *appstate.Store
151 failWithdraw int
152 failRebuild int
153 withdrawn []string
154 routed []map[string]map[int]int
155}
156
157func (t *storeTraffic) WithdrawService(ctx context.Context, app, service string) error {
158 t.mu.Lock()
159 defer t.mu.Unlock()
160 t.withdrawn = append(t.withdrawn, app+"/"+service)
161 active, ok, err := t.store.LoadActive(ctx, app)
162 if err != nil || !ok {
163 return err
164 }
165 if eff, found := active.Services[service]; found {
166 eff.BackendBinds, eff.UDPBackendBinds = nil, nil
167 active.Services[service] = eff
168 if err := t.store.SaveActive(ctx, active); err != nil {
169 return err
170 }
171 }
172 if t.failWithdraw > 0 {
173 t.failWithdraw--
174 return assert.AnError
175 }
176 return nil
177}
178
179func (t *storeTraffic) WithdrawServiceState(ctx context.Context, app, service string) error {
180 return t.WithdrawService(ctx, app, service)
181}
182
183func (t *storeTraffic) RebuildTraffic(ctx context.Context) error {
184 t.mu.Lock()
185 defer t.mu.Unlock()
186 if t.failRebuild > 0 {
187 t.failRebuild--
188 return assert.AnError
189 }
190 active, _, err := t.store.LoadActive(ctx, "blog")
191 if err != nil {
192 return err
193 }
194 routes := map[string]map[int]int{}
195 for name, eff := range active.Services {
196 routes[name] = maps.Clone(eff.BackendBinds)
197 }
198 t.routed = append(t.routed, routes)
199 return nil
200}
201
202// lastRoute returns the binds the most recent graph routed for a service.
203func (t *storeTraffic) lastRoute(service string) map[int]int {
204 t.mu.Lock()
205 defer t.mu.Unlock()
206 if len(t.routed) == 0 {
207 return nil
208 }
209 return t.routed[len(t.routed)-1][service]
210}
211
212// allRoutes returns the binds of every graph routed for a service.
213func (t *storeTraffic) allRoutes(service string) []map[int]int {
214 t.mu.Lock()
215 defer t.mu.Unlock()
216 var routes []map[int]int
217 for _, routed := range t.routed {
218 routes = append(routes, routed[service])
219 }
220 return routes
221}
222
223// withdrawals returns the services withdrawn so far, in order.
224func (t *storeTraffic) withdrawals() []string {
225 t.mu.Lock()
226 defer t.mu.Unlock()
227 return append([]string(nil), t.withdrawn...)
228}
229
230func (t *storeTraffic) rebuildCount() int {
231 t.mu.Lock()
232 defer t.mu.Unlock()
233 return len(t.routed)
234}
235
236const overlapImage = "registry.example.com/blog/web:1.4.2"
237
238// overlapWebSpec is a stateless public HTTP service.
239func overlapWebSpec(grace time.Duration) domain.AppService {
240 return domain.AppService{
241 Name: "web", Image: overlapImage, Secrets: map[string]string{}, StopGrace: grace,
242 HTTP: []domain.AppHTTPInterface{{Host: "blog.example.com", Port: 8080, TLS: domain.AppTLSAuto}},
243 Readiness: domain.AppReadiness{
244 Type: domain.AppReadinessHTTP, Path: "/healthz", Timeout: time.Second,
245 },
246 }
247}
248
249func overlapOldBinds() map[int]int { return map[int]int{8080: 17000} }
250
251// seedServingApp seeds an app whose container c-old serves on 17000, with a
252// materialized revision to deploy over it.
253func seedServingApp(t *testing.T, ctx context.Context, store *appstate.Store, spec domain.AppService) {
254 t.Helper()
255 require.NoError(t, store.SaveIntent(ctx, domain.AppStopIntent{App: "blog"}))
256 require.NoError(t, store.SaveOwnership(ctx, domain.AppOwnership{App: "blog", ID: "app-blog"}))
257 require.NoError(t, store.SaveActive(ctx, domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
258 "web": {Container: "c-old", EffectiveRevision: "rev-0", Spec: spec, BackendBinds: overlapOldBinds()},
259 }}))
260 seedDesired(t, ctx, store, testRevision("blog", spec))
261}
262
263// overlapDeployDeps wires the preflight and candidate creation of one overlap
264// deploy that creates c-new on 18081 and expects readiness to pass.
265func overlapDeployMocks(runtime *outmocks.MockContainerRuntime, images *outmocks.MockImageResolver) {
266 images.EXPECT().ResolveDigest(mock.Anything, overlapImage).Return(restartTestDigest, nil).Once()
267 runtime.EXPECT().InspectImageVolumes(mock.Anything, overlapImage).Return(nil, nil).Once()
268 expectNetworkProvision(runtime, "app-blog", 1)
269 runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-new", Name: "web"}, nil).Once()
270 runtime.EXPECT().StartContainer(mock.Anything, "c-new").Return(nil).Once()
271 runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
272 runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return(
273 []domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18081, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
274}
275
276func overlapService(store *appstate.Store, runtime *outmocks.MockContainerRuntime, images *outmocks.MockImageResolver, secrets *outmocks.MockSecretProvider, traffic *storeTraffic) *deployment.Service {
277 return deployment.NewService(deployment.Deps{
278 State: store, Runtime: runtime, Images: images, Secrets: secrets, Traffic: traffic,
279 ImagePolicy: domain.ImageSourcePolicy{AllowedRegistries: []string{"registry.example.com"}},
280 }, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
281 func(context.Context, string, string) (int, error) { return 200, nil },
282 func(context.Context, string) error { return nil },
283 ))
284}
285
286// expectVerifiedRunning wires one recovery verification of a running
287// container that publishes the given loopback port.
288func expectVerifiedRunning(runtime *outmocks.MockContainerRuntime, id string, port int) {
289 started := time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC)
290 runtime.EXPECT().InspectContainer(mock.Anything, id).
291 Return(&domain.Container{ID: id, Status: "running", StartedAt: started}, nil)
292 runtime.EXPECT().GetContainerHealthStatus(mock.Anything, id).Return("", false, nil)
293 runtime.EXPECT().GetContainerBackendBinds(mock.Anything, id, mock.Anything).Return(
294 []domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: port, Protocol: domain.NetworkProtocolTCP}}, nil)
295}
296
297// TestDeploy_OverlapSwitchIsNotErasedByAPendingWithdrawal proves that a
298// publication withdrawal left pending by an earlier failed recovery does not
299// unroute the new generation: the deploy resolves it before publishing, so
300// the graph the switch applies routes the candidate's binds. The traffic fake
301// mutates ACTIVE like the real publisher, so a stray withdrawal after the
302// switch would leave the service unrouted.
303func TestDeploy_OverlapSwitchIsNotErasedByAPendingWithdrawal(t *testing.T) {
304 ctx := context.Background()
305 store := newTestStore(t)
306 runtime := outmocks.NewMockContainerRuntime(t)
307 images := outmocks.NewMockImageResolver(t)
308 secrets := outmocks.NewMockSecretProvider(t)
309 seedServingApp(t, ctx, store, overlapWebSpec(0))
310 traffic := &storeTraffic{store: store, failWithdraw: 1}
311 svc := overlapService(store, runtime, images, secrets, traffic)
312
313 // A recovery pass finds c-old down and cannot prove its withdrawal: the
314 // service is marked as having a pending withdrawal.
315 runtime.EXPECT().InspectContainer(mock.Anything, "c-old").
316 Return(&domain.Container{ID: "c-old", Status: "exited"}, nil).Once()
317 require.Error(t, svc.ReconcileRunning(ctx))
318
319 overlapDeployMocks(runtime, images)
320 runtime.EXPECT().StopContainer(mock.Anything, "c-old", domain.AppDefaultStopGrace).Return(nil).Once()
321 runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
322
323 result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
324 require.NoError(t, err)
325 assert.Equal(t, "deployed", result.Services["web"].Result)
326
327 assert.Equal(t, map[int]int{8080: 18081}, traffic.lastRoute("web"), "the service routes the new generation")
328 active, _, err := store.LoadActive(ctx, "blog")
329 require.NoError(t, err)
330 assert.Equal(t, "c-new", active.Services["web"].Container)
331 assert.Equal(t, map[int]int{8080: 18081}, active.Services["web"].BackendBinds,
332 "ACTIVE keeps only the new generation's binds")
333}
334
335// TestDeploy_OverlapRetirementSurvivesANewerOperation proves the cleanup of a
336// superseded container is owned by the durable retirement marker, not by the
337// operation journal: after the first retirement fails and a newer operation
338// becomes the latest journal, the periodic reconciliation still retires it,
339// with the predecessor's own stop grace, and only once its successor is
340// verified and routed.
341func TestDeploy_OverlapRetirementSurvivesANewerOperation(t *testing.T) {
342 ctx := context.Background()
343 store := newTestStore(t)
344 runtime := outmocks.NewMockContainerRuntime(t)
345 images := outmocks.NewMockImageResolver(t)
346 secrets := outmocks.NewMockSecretProvider(t)
347 seedServingApp(t, ctx, store, overlapWebSpec(45*time.Second))
348 traffic := &storeTraffic{store: store}
349 svc := overlapService(store, runtime, images, secrets, traffic)
350
351 overlapDeployMocks(runtime, images)
352 runtime.EXPECT().StopContainer(mock.Anything, "c-old", 45*time.Second).Return(nil).Twice()
353 runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(assert.AnError).Once()
354
355 result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
356 require.NoError(t, err, "a retirement leftover does not fail a deploy whose new generation serves")
357 require.Len(t, result.CleanupWarnings, 1)
358 assert.Equal(t, "c-old", result.CleanupWarnings[0].Leftover)
359
360 markers, err := store.LoadRecoveryInhibitions(ctx, "blog")
361 require.NoError(t, err)
362 require.Len(t, markers, 1, "the failed retirement leaves its durable marker")
363 assert.Equal(t, "c-old", markers[0].ContainerID)
364 assert.Equal(t, domain.AppInhibitRetirementPending, markers[0].Reason)
365 assert.Equal(t, 45*time.Second, markers[0].StopGrace, "the predecessor's grace travels with the marker")
366
367 // A newer operation becomes the latest journal and masks the deploy's.
368 require.NoError(t, store.SaveOperation(ctx, domain.AppOperation{
369 Op: "op-newer", Kind: "deploy", App: "blog", StartedAt: time.Now().Add(time.Minute).UTC(),
370 Outcome: domain.AppOutcomeSuccess,
371 }))
372
373 expectVerifiedRunning(runtime, "c-new", 18082)
374 runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
375 require.NoError(t, svc.ReconcileRunning(ctx))
376
377 markers, err = store.LoadRecoveryInhibitions(ctx, "blog")
378 require.NoError(t, err)
379 assert.Empty(t, markers, "the marker is cleared once the container is confirmed gone")
380 assert.Equal(t, map[int]int{8080: 18082}, traffic.lastRoute("web"), "the verified successor is routed")
381 active, _, err := store.LoadActive(ctx, "blog")
382 require.NoError(t, err)
383 assert.Equal(t, "c-new", active.Services["web"].Container)
384}
385
386// TestReconcileRunning_RetiresSupersededOnlyAfterSuccessorIsVerified proves a
387// superseded container is kept while its successor cannot be verified: the
388// working generation is never stopped in favour of one that is not serving.
389func TestReconcileRunning_RetiresSupersededOnlyAfterSuccessorIsVerified(t *testing.T) {
390 ctx := context.Background()
391 store := newTestStore(t)
392 runtime := outmocks.NewMockContainerRuntime(t)
393 images := outmocks.NewMockImageResolver(t)
394 secrets := outmocks.NewMockSecretProvider(t)
395 seedServingApp(t, ctx, store, overlapWebSpec(0))
396 require.NoError(t, store.SaveActive(ctx, domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
397 "web": {Container: "c-new", EffectiveRevision: "rev-1", Spec: overlapWebSpec(0), BackendBinds: map[int]int{8080: 18081}},
398 }}))
399 require.NoError(t, store.SaveRecoveryInhibition(ctx, domain.AppRecoveryInhibition{
400 App: "blog", Service: "web", ContainerID: "c-old", Reason: domain.AppInhibitRetirementPending,
401 }))
402 traffic := &storeTraffic{store: store}
403 svc := overlapService(store, runtime, images, secrets, traffic)
404
405 runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(nil, assert.AnError).Once()
406 require.Error(t, svc.ReconcileRunning(ctx))
407 runtime.AssertNotCalled(t, "StopContainer", mock.Anything, "c-old", mock.Anything)
408 markers, err := store.LoadRecoveryInhibitions(ctx, "blog")
409 require.NoError(t, err)
410 assert.Len(t, markers, 1, "the marker waits for a verified successor")
411
412 // A record written before the grace was persisted falls back to the default.
413 expectVerifiedRunning(runtime, "c-new", 18081)
414 runtime.EXPECT().StopContainer(mock.Anything, "c-old", domain.AppDefaultStopGrace).Return(nil).Once()
415 runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
416 require.NoError(t, svc.ReconcileRunning(ctx))
417 markers, err = store.LoadRecoveryInhibitions(ctx, "blog")
418 require.NoError(t, err)
419 assert.Empty(t, markers)
420}
421
422// TestReconcileRunning_NeverRetiresAPendingContainerThatActiveNamesAgain
423// proves a rollback to the superseded container is safe: when ACTIVE names the
424// container again, its stale retirement marker is dropped and it is kept.
425func TestReconcileRunning_NeverRetiresAPendingContainerThatActiveNamesAgain(t *testing.T) {
426 ctx := context.Background()
427 store := newTestStore(t)
428 runtime := outmocks.NewMockContainerRuntime(t)
429 images := outmocks.NewMockImageResolver(t)
430 secrets := outmocks.NewMockSecretProvider(t)
431 seedServingApp(t, ctx, store, overlapWebSpec(0))
432 require.NoError(t, store.SaveRecoveryInhibition(ctx, domain.AppRecoveryInhibition{
433 App: "blog", Service: "web", ContainerID: "c-old", Reason: domain.AppInhibitRetirementPending,
434 }))
435 svc := overlapService(store, runtime, images, secrets, &storeTraffic{store: store})
436
437 expectVerifiedRunning(runtime, "c-old", 17000)
438 require.NoError(t, svc.ReconcileRunning(ctx))
439
440 runtime.AssertNotCalled(t, "StopContainer", mock.Anything, mock.Anything, mock.Anything)
441 runtime.AssertNotCalled(t, "RemoveContainer", mock.Anything, mock.Anything, mock.Anything)
442 markers, err := store.LoadRecoveryInhibitions(ctx, "blog")
443 require.NoError(t, err)
444 assert.Empty(t, markers)
445 active, _, err := store.LoadActive(ctx, "blog")
446 require.NoError(t, err)
447 assert.Equal(t, "c-old", active.Services["web"].Container)
448}
449
450// TestBootRecovery_DoesNotRepublishUnverifiedBindsOfAPublishedCandidate proves
451// interrupted-operation reconciliation never rebuilds traffic from the binds
452// ACTIVE persisted for a published candidate: a restarted candidate's
453// ephemeral port may belong to another workload. Only the verified start path
454// routes, and the superseded container is retired afterwards through its
455// marker.
456func TestBootRecovery_DoesNotRepublishUnverifiedBindsOfAPublishedCandidate(t *testing.T) {
457 ctx := context.Background()
458 store := newTestStore(t)
459 runtime := outmocks.NewMockContainerRuntime(t)
460 images := outmocks.NewMockImageResolver(t)
461 secrets := outmocks.NewMockSecretProvider(t)
462 spec := overlapWebSpec(0)
463 require.NoError(t, store.SaveIntent(ctx, domain.AppStopIntent{App: "blog"}))
464 require.NoError(t, store.SaveActive(ctx, domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
465 "web": {Container: "c-new", EffectiveRevision: "rev-1", Spec: spec, BackendBinds: map[int]int{8080: 18081}},
466 }}))
467 require.NoError(t, store.SaveOperation(ctx, domain.AppOperation{
468 Op: "op-switch", Kind: "deploy", App: "blog", StartedAt: time.Now().UTC(),
469 Steps: []domain.AppOperationStep{
470 {ID: "preflight", State: domain.AppStepSucceeded},
471 {ID: "service.web.replace", State: domain.AppStepPending, Service: "web", Before: "c-old", After: "c-new"},
472 },
473 }))
474 require.NoError(t, store.SaveRecoveryInhibition(ctx, domain.AppRecoveryInhibition{
475 App: "blog", Service: "web", ContainerID: "c-old", Reason: domain.AppInhibitRetirementPending,
476 }))
477 traffic := &storeTraffic{store: store}
478 svc := overlapService(store, runtime, images, secrets, traffic)
479
480 // The candidate restarted and now publishes a different port.
481 runtime.EXPECT().IsContainerRunning(mock.Anything, "c-new").Return(true, nil).Once()
482 runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
483 runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return(
484 []domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18090, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
485 require.NoError(t, svc.ReconcileBoot(ctx))
486
487 require.Positive(t, traffic.rebuildCount())
488 for _, routed := range traffic.allRoutes("web") {
489 assert.NotEqual(t, map[int]int{8080: 18081}, routed, "the unverified persisted port is never routed")
490 }
491 assert.Equal(t, map[int]int{8080: 18090}, traffic.lastRoute("web"), "the verified bind is routed")
492 active, _, err := store.LoadActive(ctx, "blog")
493 require.NoError(t, err)
494 assert.Equal(t, map[int]int{8080: 18090}, active.Services["web"].BackendBinds)
495
496 // Boot keeps the superseded container and its marker: the periodic pass
497 // retires it once the successor is verified.
498 runtime.AssertNotCalled(t, "StopContainer", mock.Anything, "c-old", mock.Anything)
499 markers, err := store.LoadRecoveryInhibitions(ctx, "blog")
500 require.NoError(t, err)
501 require.Len(t, markers, 1)
502 assert.Equal(t, "c-old", markers[0].ContainerID)
503}
504
505// TestDeploy_OverlapTrafficFailureIsRecoveredByThePeriodicPass proves the
506// failed switch is recoverable and traceable: the graph apply is rejected after
507// ACTIVE named the new container, the superseded container keeps running with
508// its marker, the next deploy-side reconciliation keeps that marker, and the
509// periodic pass retires the old container once the successor is routed.
510func TestDeploy_OverlapTrafficFailureIsRecoveredByThePeriodicPass(t *testing.T) {
511 ctx := context.Background()
512 store := newTestStore(t)
513 runtime := outmocks.NewMockContainerRuntime(t)
514 images := outmocks.NewMockImageResolver(t)
515 secrets := outmocks.NewMockSecretProvider(t)
516 seedServingApp(t, ctx, store, overlapWebSpec(0))
517 traffic := &storeTraffic{store: store, failRebuild: 1}
518 svc := overlapService(store, runtime, images, secrets, traffic)
519
520 overlapDeployMocks(runtime, images)
521 _, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
522 require.Error(t, err, "a rejected graph is a failed deploy")
523 runtime.AssertNotCalled(t, "StopContainer", mock.Anything, mock.Anything, mock.Anything)
524
525 active, _, err := store.LoadActive(ctx, "blog")
526 require.NoError(t, err)
527 assert.Equal(t, "c-new", active.Services["web"].Container, "ACTIVE names the new container")
528 markers, err := store.LoadRecoveryInhibitions(ctx, "blog")
529 require.NoError(t, err)
530 require.Len(t, markers, 1)
531 assert.Equal(t, "c-old", markers[0].ContainerID)
532
533 // The next mutation reconciles the failed deploy: the published candidate
534 // is kept, and so is the retirement marker.
535 require.NoError(t, svc.AbandonDeploy(ctx, deployment.DeployClaim{App: "blog", Op: "op-none"}))
536 markers, err = store.LoadRecoveryInhibitions(ctx, "blog")
537 require.NoError(t, err)
538 require.Len(t, markers, 1, "a published candidate's reconciliation keeps the retirement marker")
539
540 expectVerifiedRunning(runtime, "c-new", 18081)
541 runtime.EXPECT().StopContainer(mock.Anything, "c-old", domain.AppDefaultStopGrace).Return(nil).Once()
542 runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
543 require.NoError(t, svc.ReconcileRunning(ctx))
544 assert.Equal(t, map[int]int{8080: 18081}, traffic.lastRoute("web"))
545 markers, err = store.LoadRecoveryInhibitions(ctx, "blog")
546 require.NoError(t, err)
547 assert.Empty(t, markers)
548}
549
550// TestDeploy_PredecessorWithOwnedResourcesNeverOverlaps proves eligibility
551// looks at the predecessor's effective spec too: a replacement that drops the
552// UDP interface or the volume of the generation it supersedes still replaces
553// it sequentially, because the old generation's UDP sessions or writes
554// outlive the switch.
555func TestDeploy_PredecessorWithOwnedResourcesNeverOverlaps(t *testing.T) {
556 cases := map[string]func(*domain.AppService){
557 "udp": func(s *domain.AppService) {
558 s.UDP = []domain.AppUDPInterface{{Entrypoint: "game", Port: 28015, Publish: "0.0.0.0:28015"}}
559 },
560 "volume": func(s *domain.AppService) { s.Volumes = []domain.AppVolume{{Name: "data", Path: "/data"}} },
561 }
562 for name, mutate := range cases {
563 t.Run(name, func(t *testing.T) {
564 ctx := context.Background()
565 state, runtime, images, secrets := mockDeps(t)
566 rev := mockRevision() // HTTP-only, no volume: eligible on its own
567 previous := rev.Spec.Services[0]
568 mutate(&previous)
569 active := domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
570 "web": {EffectiveRevision: "rev-0", Image: "img:0", Container: "c-old", Spec: previous},
571 }}
572
573 expectDeployPreflight(state, runtime, images, secrets, rev)
574 expectNetworkProvision(runtime, "app-blog", 1)
575 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil)
576 state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
577 var order []string
578 expectRetireOld(state, runtime, &order, "c-old")
579 expectSequentialCreate(state, runtime, &order, "c-new", 18080)
580 state.EXPECT().SaveActive(mock.Anything, mock.Anything).Return(nil).Once()
581 state.EXPECT().SaveOwnership(mock.Anything, mock.Anything).Return(nil)
582 state.EXPECT().ClearRecoveryInhibition(mock.Anything, "blog", "web", "c-old").Return(nil).Once()
583
584 svc := deployment.NewService(deployment.Deps{
585 State: state, Runtime: runtime, Images: images, Secrets: secrets,
586 Traffic: &orderedTraffic{events: &order},
587 }, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
588 func(context.Context, string, string) (int, error) { return 200, nil },
589 func(context.Context, string) error { return nil },
590 ))
591 _, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
592 require.NoError(t, err)
593 assert.Equal(t, []string{"withdraw", "stop-old", "remove-old", "create-new", "start-new", "traffic"}, order[:6],
594 "the predecessor is gone before the replacement is created")
595 state.AssertNotCalled(t, "SaveRecoveryInhibition", mock.Anything, mock.Anything)
596 })
597 }
598}
599
600// seedPendingRetirement seeds ACTIVE naming c-new and one retirement-pending
601// marker per given superseded container.
602func seedPendingRetirement(t *testing.T, ctx context.Context, store *appstate.Store, superseded ...string) {
603 t.Helper()
604 require.NoError(t, store.SaveIntent(ctx, domain.AppStopIntent{App: "blog"}))
605 require.NoError(t, store.SaveActive(ctx, domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
606 "web": {Container: "c-new", EffectiveRevision: "rev-1", Spec: overlapWebSpec(0), BackendBinds: map[int]int{8080: 18081}},
607 }}))
608 for _, id := range superseded {
609 require.NoError(t, store.SaveRecoveryInhibition(ctx, domain.AppRecoveryInhibition{
610 App: "blog", Service: "web", ContainerID: id, Reason: domain.AppInhibitRetirementPending,
611 }))
612 }
613}
614
615func markerIDs(t *testing.T, ctx context.Context, store *appstate.Store) []string {
616 t.Helper()
617 markers, err := store.LoadRecoveryInhibitions(ctx, "blog")
618 require.NoError(t, err)
619 ids := make([]string, 0, len(markers))
620 for _, marker := range markers {
621 ids = append(ids, marker.ContainerID)
622 }
623 return ids
624}
625
626// TestReconcileRunning_DoesNotRetireWhenSuccessorIsNotVerifiedByThisPass is the
627// regression for a successor that was verified by an earlier pass but is
628// restarting in this one: it is withdrawn, not verified, and a final
629// inspection that happens to see it running must not retire the predecessor.
630func TestReconcileRunning_DoesNotRetireWhenSuccessorIsNotVerifiedByThisPass(t *testing.T) {
631 ctx := context.Background()
632 store := newTestStore(t)
633 runtime := outmocks.NewMockContainerRuntime(t)
634 images := outmocks.NewMockImageResolver(t)
635 secrets := outmocks.NewMockSecretProvider(t)
636 seedPendingRetirement(t, ctx, store, "c-old")
637 traffic := &storeTraffic{store: store}
638 svc := overlapService(store, runtime, images, secrets, traffic)
639
640 // Earlier pass: the successor is verified and routed, retirement fails.
641 expectVerifiedRunning(runtime, "c-new", 18081)
642 runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(nil).Once()
643 runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(assert.AnError).Once()
644 require.Error(t, svc.ReconcileRunning(ctx))
645
646 // This pass: the successor is restarting, so it is withdrawn and not
647 // verified. Any later inspection would see it running.
648 runtime.ExpectedCalls, runtime.Calls = nil, nil
649 runtime.EXPECT().InspectContainer(mock.Anything, "c-new").
650 Return(&domain.Container{ID: "c-new", Status: "restarting"}, nil).Once()
651 runtime.EXPECT().InspectContainer(mock.Anything, "c-new").
652 Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: time.Now()}, nil).Maybe()
653 require.NoError(t, svc.ReconcileRunning(ctx))
654
655 runtime.AssertNotCalled(t, "StopContainer", mock.Anything, "c-old", mock.Anything)
656 assert.Equal(t, []string{"c-old"}, markerIDs(t, ctx, store), "the predecessor stays until a pass verifies the successor")
657}
658
659// TestReconcileRunning_DoesNotRetireWhenSuccessorRestartedAfterVerification
660// covers the other half: a successor verified at one start time and seen with
661// another by the final inspection is a different execution, so unverified.
662func TestReconcileRunning_DoesNotRetireWhenSuccessorRestartedAfterVerification(t *testing.T) {
663 ctx := context.Background()
664 store := newTestStore(t)
665 runtime := outmocks.NewMockContainerRuntime(t)
666 images := outmocks.NewMockImageResolver(t)
667 secrets := outmocks.NewMockSecretProvider(t)
668 seedPendingRetirement(t, ctx, store, "c-old")
669 svc := overlapService(store, runtime, images, secrets, &storeTraffic{store: store})
670
671 first := time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC)
672 restarted := first.Add(time.Minute)
673 runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: first}, nil).Twice()
674 runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: restarted}, nil).Once()
675 runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "c-new").Return("", false, nil)
676 runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return(
677 []domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18081, Protocol: domain.NetworkProtocolTCP}}, nil)
678 require.NoError(t, svc.ReconcileRunning(ctx))
679
680 runtime.AssertNotCalled(t, "StopContainer", mock.Anything, "c-old", mock.Anything)
681 assert.Equal(t, []string{"c-old"}, markerIDs(t, ctx, store))
682}
683
684// TestReconcileRunning_RetiresStackedPredecessors proves several superseded
685// containers of one service (deploys that each left one behind) are all
686// retired once the successor is verified.
687func TestReconcileRunning_RetiresStackedPredecessors(t *testing.T) {
688 ctx := context.Background()
689 store := newTestStore(t)
690 runtime := outmocks.NewMockContainerRuntime(t)
691 images := outmocks.NewMockImageResolver(t)
692 secrets := outmocks.NewMockSecretProvider(t)
693 seedPendingRetirement(t, ctx, store, "c-old-1", "c-old-2")
694 svc := overlapService(store, runtime, images, secrets, &storeTraffic{store: store})
695
696 expectVerifiedRunning(runtime, "c-new", 18081)
697 for _, id := range []string{"c-old-1", "c-old-2"} {
698 runtime.EXPECT().StopContainer(mock.Anything, id, domain.AppDefaultStopGrace).Return(nil).Once()
699 runtime.EXPECT().RemoveContainer(mock.Anything, id, false).Return(nil).Once()
700 }
701 require.NoError(t, svc.ReconcileRunning(ctx))
702 assert.Empty(t, markerIDs(t, ctx, store))
703}
704
705// releaseFailingState wraps the real store and fails the first backend claim
706// releases, like a store write error would.
707type releaseFailingState struct {
708 *appstate.Store
709 failures int
710}
711
712func (s *releaseFailingState) ReleaseBackendBinds(ctx context.Context, app, containerID string) error {
713 if s.failures > 0 {
714 s.failures--
715 return assert.AnError
716 }
717 return s.Store.ReleaseBackendBinds(ctx, app, containerID)
718}
719
720// TestReconcileRunning_KeepsMarkerUntilBackendClaimsAreReleased proves a
721// retirement whose claim release failed does not drop the durable marker: the
722// leak is reported by the pass and retried (retiring a gone container is
723// idempotent) until the claims are released.
724func TestReconcileRunning_KeepsMarkerUntilBackendClaimsAreReleased(t *testing.T) {
725 ctx := context.Background()
726 store := newTestStore(t)
727 runtime := outmocks.NewMockContainerRuntime(t)
728 images := outmocks.NewMockImageResolver(t)
729 secrets := outmocks.NewMockSecretProvider(t)
730 seedPendingRetirement(t, ctx, store, "c-old")
731 state := &releaseFailingState{Store: store, failures: 1}
732 traffic := &storeTraffic{store: store}
733 svc := deployment.NewService(deployment.Deps{
734 State: state, Runtime: runtime, Images: images, Secrets: secrets, Traffic: traffic,
735 ImagePolicy: domain.ImageSourcePolicy{AllowedRegistries: []string{"registry.example.com"}},
736 }, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
737 func(context.Context, string, string) (int, error) { return 200, nil },
738 func(context.Context, string) error { return nil },
739 ))
740
741 expectVerifiedRunning(runtime, "c-new", 18081)
742 runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(nil).Twice()
743 runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
744 runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(domain.ErrContainerNotFound).Once()
745
746 err := svc.ReconcileRunning(ctx)
747 require.Error(t, err, "the leaked claims are reported, not discarded")
748 assert.Contains(t, err.Error(), "release backend claims")
749 assert.Equal(t, []string{"c-old"}, markerIDs(t, ctx, store), "the marker stays until the claims are released")
750
751 require.NoError(t, svc.ReconcileRunning(ctx), "not-found counts as gone, so the retry settles it")
752 assert.Empty(t, markerIDs(t, ctx, store))
753}
754
755// TestStop_RetiresPendingSupersededContainers proves a stopped app does not
756// keep a superseded predecessor running, and that one which cannot be retired
757// fails the stop and keeps its marker.
758func TestStop_RetiresPendingSupersededContainers(t *testing.T) {
759 ctx := context.Background()
760 store := newTestStore(t)
761 runtime := outmocks.NewMockContainerRuntime(t)
762 images := outmocks.NewMockImageResolver(t)
763 secrets := outmocks.NewMockSecretProvider(t)
764 seedPendingRetirement(t, ctx, store, "c-old")
765 svc := overlapService(store, runtime, images, secrets, &storeTraffic{store: store})
766
767 runtime.EXPECT().StopContainer(mock.Anything, "c-new", mock.Anything).Return(nil).Twice()
768 runtime.EXPECT().RemoveContainer(mock.Anything, "c-new", false).Return(nil).Twice()
769 runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(nil).Twice()
770 runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(assert.AnError).Once()
771
772 result, err := svc.Stop(ctx, "blog", "")
773 require.Error(t, err, "a predecessor that is still running fails the stop")
774 assert.Equal(t, "failed", result.Services["web"].Result)
775 assert.Equal(t, []string{"c-old"}, markerIDs(t, ctx, store), "its marker stays for the next attempt")
776
777 runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
778 _, err = svc.Stop(ctx, "blog", "")
779 require.NoError(t, err)
780 assert.Empty(t, markerIDs(t, ctx, store))
781}
782
783// TestRemove_FailsAndKeepsStateWhenAPendingPredecessorCannotBeRetired proves
784// Remove does not end the incarnation (which deletes the markers) while a
785// superseded container is still running.
786func TestRemove_FailsAndKeepsStateWhenAPendingPredecessorCannotBeRetired(t *testing.T) {
787 ctx := context.Background()
788 store := newTestStore(t)
789 runtime := outmocks.NewMockContainerRuntime(t)
790 images := outmocks.NewMockImageResolver(t)
791 secrets := outmocks.NewMockSecretProvider(t)
792 seedPendingRetirement(t, ctx, store, "c-old")
793 svc := overlapService(store, runtime, images, secrets, &storeTraffic{store: store})
794
795 runtime.EXPECT().StopContainer(mock.Anything, "c-new", mock.Anything).Return(nil).Once()
796 runtime.EXPECT().RemoveContainer(mock.Anything, "c-new", false).Return(nil).Once()
797 runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(nil).Once()
798 runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(assert.AnError).Once()
799
800 _, err := svc.Remove(ctx, "blog", "")
801 require.Error(t, err)
802 assert.Equal(t, []string{"c-old"}, markerIDs(t, ctx, store), "the marker survives so the container stays tracked")
803 _, ok, loadErr := store.LoadActive(ctx, "blog")
804 require.NoError(t, loadErr)
805 assert.True(t, ok, "the incarnation is not ended")
806}
807
808// inhibitionsFailingState wraps the real store and fails the load of the
809// recovery inhibitions, which is how Stop learns about pending predecessors.
810type inhibitionsFailingState struct {
811 *appstate.Store
812}
813
814func (s *inhibitionsFailingState) LoadRecoveryInhibitions(context.Context, string) ([]domain.AppRecoveryInhibition, error) {
815 return nil, assert.AnError
816}
817
818// TestStop_PendingRetirementLoadFailureIsAJournaledFailure proves an app-level
819// cleanup failure (the pending predecessors cannot even be listed) is not lost
820// in the per-service results: the operation is journaled as failed, so Stop
821// errors and a replay of the same key reads the same failure instead of a
822// success.
823func TestStop_PendingRetirementLoadFailureIsAJournaledFailure(t *testing.T) {
824 ctx := context.Background()
825 store := newTestStore(t)
826 runtime := outmocks.NewMockContainerRuntime(t)
827 seedPendingRetirement(t, ctx, store, "c-old")
828 // The ACTIVE container is retired successfully before the load fails.
829 runtime.EXPECT().StopContainer(mock.Anything, "c-new", mock.Anything).Return(nil).Once()
830 runtime.EXPECT().RemoveContainer(mock.Anything, "c-new", false).Return(nil).Once()
831 svc := deployment.NewService(deployment.Deps{
832 State: &inhibitionsFailingState{Store: store}, Runtime: runtime, Traffic: &storeTraffic{store: store},
833 }, zerowrap.Default())
834
835 result, err := svc.Stop(ctx, "blog", "key-stop")
836 require.Error(t, err)
837 assert.Equal(t, "deployed", result.Services["web"].Result, "the ACTIVE container really was stopped")
838 require.NotEmpty(t, result.Warnings, "the app-level failure is surfaced on the result")
839
840 op, loadErr := store.LoadOperation(ctx, "blog", "key-stop")
841 require.NoError(t, loadErr)
842 assert.Equal(t, domain.AppOutcomeFailed, op.Outcome, "the journal records the failure, not the per-service success")
843 step, found := opStep(op, "pending.retire")
844 require.True(t, found)
845 assert.Equal(t, domain.AppStepFailed, step.State)
846 assert.NotEmpty(t, step.Error)
847 require.NotEmpty(t, op.Warnings)
848 assert.Contains(t, op.Warnings[0].Detail, "pending retirements unknown")
849
850 // Replaying the same key returns the same failure and runs nothing again.
851 _, replayErr := svc.Stop(ctx, "blog", "key-stop")
852 require.Error(t, replayErr)
853 assert.ErrorIs(t, replayErr, domain.ErrAppStateConflict)
854 assert.Contains(t, replayErr.Error(), domain.AppOutcomeFailed)
855 runtime.AssertNumberOfCalls(t, "StopContainer", 1)
856}
857
858// executionStartedAt is the start time the fakes report for a freshly started
859// container: deploy and restart observe it right after the start.
860var executionStartedAt = time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC)
861
862// dropInspectExpectations removes the InspectContainer expectations so a test
863// can report a different execution after a deploy.
864func dropInspectExpectations(runtime *outmocks.MockContainerRuntime) {
865 kept := runtime.ExpectedCalls[:0]
866 for _, call := range runtime.ExpectedCalls {
867 if call.Method != "InspectContainer" {
868 kept = append(kept, call)
869 }
870 }
871 runtime.ExpectedCalls = kept
872}
873
874// TestReconcileRunning_DoesNotWithdrawAFreshlyDeployedGeneration is the
875// regression for a deployed generation being withdrawn by the first periodic
876// pass: the deploy verified and routed that execution, so the pass must treat
877// it as already seen. The traffic fake mutates ACTIVE like the real publisher,
878// so a withdrawal would drop the service's routed binds.
879func TestReconcileRunning_DoesNotWithdrawAFreshlyDeployedGeneration(t *testing.T) {
880 ctx := context.Background()
881 store := newTestStore(t)
882 runtime := outmocks.NewMockContainerRuntime(t)
883 images := outmocks.NewMockImageResolver(t)
884 secrets := outmocks.NewMockSecretProvider(t)
885 seedServingApp(t, ctx, store, overlapWebSpec(0))
886 traffic := &storeTraffic{store: store}
887 svc := overlapService(store, runtime, images, secrets, traffic)
888
889 overlapDeployMocks(runtime, images)
890 runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(nil).Once()
891 runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
892 _, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
893 require.NoError(t, err)
894 require.Equal(t, map[int]int{8080: 18081}, traffic.lastRoute("web"))
895
896 // The first periodic pass sees the same execution the deploy verified.
897 runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "c-new").Return("", false, nil)
898 runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return(
899 []domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18081, Protocol: domain.NetworkProtocolTCP}}, nil)
900 require.NoError(t, svc.ReconcileRunning(ctx))
901
902 assert.Empty(t, traffic.withdrawals(), "the verified generation is never withdrawn")
903 for i, routed := range traffic.allRoutes("web") {
904 assert.Equal(t, map[int]int{8080: 18081}, routed, "graph %d kept routing the service", i)
905 }
906 active, _, err := store.LoadActive(ctx, "blog")
907 require.NoError(t, err)
908 assert.Equal(t, map[int]int{8080: 18081}, active.Services["web"].BackendBinds)
909}
910
911// TestReconcileRunning_TreatsARestartedExecutionAfterDeployAsFresh proves the
912// recorded execution is the one observed before readiness: if the container
913// restarted natively afterwards, its start time differs and the pass withdraws
914// and verifies it again.
915func TestReconcileRunning_TreatsARestartedExecutionAfterDeployAsFresh(t *testing.T) {
916 ctx := context.Background()
917 store := newTestStore(t)
918 runtime := outmocks.NewMockContainerRuntime(t)
919 images := outmocks.NewMockImageResolver(t)
920 secrets := outmocks.NewMockSecretProvider(t)
921 seedServingApp(t, ctx, store, overlapWebSpec(0))
922 traffic := &storeTraffic{store: store}
923 svc := overlapService(store, runtime, images, secrets, traffic)
924
925 overlapDeployMocks(runtime, images)
926 runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(nil).Once()
927 runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
928 _, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
929 require.NoError(t, err)
930
931 dropInspectExpectations(runtime)
932 runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(
933 &domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt.Add(time.Minute)}, nil)
934 runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "c-new").Return("", false, nil)
935 runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return(
936 []domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18090, Protocol: domain.NetworkProtocolTCP}}, nil)
937 require.NoError(t, svc.ReconcileRunning(ctx))
938
939 assert.Equal(t, []string{"blog/web"}, traffic.withdrawals(), "a different execution is withdrawn before it is verified")
940 assert.Equal(t, map[int]int{8080: 18090}, traffic.lastRoute("web"), "and routed again on its verified binds")
941}
942
943// TestReconcileRunning_DoesNotWithdrawARestartedGeneration covers the restart
944// path: an in-place restart verified and routed the new execution, so the
945// first periodic pass keeps it.
946func TestReconcileRunning_DoesNotWithdrawARestartedGeneration(t *testing.T) {
947 ctx := context.Background()
948 store := newTestStore(t)
949 runtime := outmocks.NewMockContainerRuntime(t)
950 images := outmocks.NewMockImageResolver(t)
951 secrets := outmocks.NewMockSecretProvider(t)
952 seedServingApp(t, ctx, store, overlapWebSpec(0))
953 traffic := &storeTraffic{store: store}
954 svc := overlapService(store, runtime, images, secrets, traffic)
955
956 restartedAt := executionStartedAt.Add(time.Hour)
957 runtime.EXPECT().RestartContainer(mock.Anything, "c-old", mock.Anything).Return(nil).Once()
958 runtime.EXPECT().InspectContainer(mock.Anything, "c-old").Return(
959 &domain.Container{ID: "c-old", Status: "running", StartedAt: restartedAt}, nil)
960 runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-old", mock.Anything).Return(
961 []domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 17002, Protocol: domain.NetworkProtocolTCP}}, nil)
962 runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "c-old").Return("", false, nil)
963 _, err := svc.Restart(ctx, "blog", "web", "")
964 require.NoError(t, err)
965 withdrawnByRestart := len(traffic.withdrawals())
966
967 require.NoError(t, svc.ReconcileRunning(ctx))
968 assert.Len(t, traffic.withdrawals(), withdrawnByRestart, "the periodic pass withdraws nothing more")
969 assert.Equal(t, map[int]int{8080: 17002}, traffic.lastRoute("web"))
970}
971
972// TestReconcileRunning_RestoresARebuiltGenerationWhoseTrafficWasRejected is the
973// regression for a missing-container rebuild whose graph apply was rejected:
974// the rebuilt execution must not be recorded as verified, or the next periodic
975// pass sees an already verified generation whose binds match ACTIVE, skips the
976// publication, and leaves the service unrouted indefinitely.
977func TestReconcileRunning_RestoresARebuiltGenerationWhoseTrafficWasRejected(t *testing.T) {
978 ctx := context.Background()
979 store := newTestStore(t)
980 runtime := outmocks.NewMockContainerRuntime(t)
981 images := outmocks.NewMockImageResolver(t)
982 secrets := outmocks.NewMockSecretProvider(t)
983 seedReplaceableApp(t, ctx, store)
984 traffic := &storeTraffic{store: store, failRebuild: 1}
985 svc := deployment.NewService(deployment.Deps{
986 State: store, Runtime: runtime, Images: images, Secrets: secrets, Traffic: traffic,
987 }, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
988 func(context.Context, string, string) (int, error) { return 200, nil },
989 func(context.Context, string) error { return nil },
990 ))
991
992 // The recorded container is gone: restart rebuilds it from the pinned
993 // ACTIVE digest, and the graph apply that follows is rejected.
994 runtime.EXPECT().RestartContainer(mock.Anything, "c-old", mock.Anything).Return(domain.ErrContainerNotFound).Once()
995 runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(domain.ErrContainerNotFound).Once()
996 runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(domain.ErrContainerNotFound).Once()
997 expectNetworkProvision(runtime, "app-blog", 1)
998 runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-rebuilt", Name: "web"}, nil).Once()
999 runtime.EXPECT().StartContainer(mock.Anything, "c-rebuilt").Return(nil).Once()
1000 runtime.EXPECT().InspectContainer(mock.Anything, "c-rebuilt").Return(
1001 &domain.Container{ID: "c-rebuilt", Status: "running", StartedAt: executionStartedAt}, nil)
1002 runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-rebuilt", mock.Anything).Return(
1003 []domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18082, Protocol: domain.NetworkProtocolTCP}}, nil)
1004 runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "c-rebuilt").Return("", false, nil)
1005
1006 _, err := svc.Restart(ctx, "blog", "web", "op-restart")
1007 require.Error(t, err, "a rejected graph is a failed restart")
1008 require.Zero(t, traffic.rebuildCount(), "no graph was accepted")
1009 active, _, loadErr := store.LoadActive(ctx, "blog")
1010 require.NoError(t, loadErr)
1011 require.Equal(t, "c-rebuilt", active.Services["web"].Container)
1012
1013 // The next periodic pass must verify and route the generation itself.
1014 require.NoError(t, svc.ReconcileRunning(ctx))
1015 assert.Equal(t, map[int]int{8080: 18082}, traffic.lastRoute("web"), "the rebuilt generation is routed again")
1016}