1package deployment_test
2
3import (
4 "context"
5 "strings"
6 "sync"
7 "testing"
8 "time"
9
10 "github.com/bnema/zerowrap"
11 "github.com/stretchr/testify/assert"
12 "github.com/stretchr/testify/mock"
13 "github.com/stretchr/testify/require"
14
15 outmocks "github.com/bnema/gordon/internal/boundaries/out/mocks"
16 "github.com/bnema/gordon/internal/domain"
17 "github.com/bnema/gordon/internal/usecase/deployment"
18)
19
20// orderedTraffic records traffic boundary calls into a shared event log and
21// can fail one withdrawal or one graph apply.
22type orderedTraffic struct {
23 mu sync.Mutex
24 events *[]string
25 failWithdraw bool
26 failRebuild bool
27}
28
29func (t *orderedTraffic) record(event string) {
30 t.mu.Lock()
31 defer t.mu.Unlock()
32 if t.events != nil {
33 *t.events = append(*t.events, event)
34 }
35}
36
37func (t *orderedTraffic) RebuildTraffic(context.Context) error {
38 t.record("traffic")
39 if t.failRebuild {
40 return assert.AnError
41 }
42 return nil
43}
44
45func (t *orderedTraffic) WithdrawService(context.Context, string, string) error {
46 t.record("withdraw")
47 if t.failWithdraw {
48 return assert.AnError
49 }
50 return nil
51}
52
53func (t *orderedTraffic) WithdrawServiceState(context.Context, string, string) error {
54 t.record("withdraw-state")
55 if t.failWithdraw {
56 return assert.AnError
57 }
58 return nil
59}
60
61// replaceableActive is one ACTIVE record whose web service is superseded.
62func replaceableActive() domain.AppActive {
63 return domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
64 "web": {EffectiveRevision: "rev-0", Image: "img:0", Container: "c-old"},
65 }}
66}
67
68// recordEvent appends one step to a shared event log. Test doubles and mock
69// hooks share the log, so the sequence a deployment performs is observable.
70func recordEvent(events *[]string, event string) {
71 if events != nil {
72 *events = append(*events, event)
73 }
74}
75
76// expectReplacementPreflight wires the preflight calls of one replacement
77// without pre-registering an operation-journal write, so a test can observe
78// the terminal journal write itself.
79func expectReplacementPreflight(
80 state *outmocks.MockAppState,
81 runtime *outmocks.MockContainerRuntime,
82 images *outmocks.MockImageResolver,
83 secrets *outmocks.MockSecretProvider,
84 rev domain.AppDesiredRevision,
85) {
86 state.EXPECT().Recover(mock.Anything).Return(nil)
87 state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
88 images.EXPECT().ResolveDigest(mock.Anything, rev.Spec.Services[0].Image).Return(
89 "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", nil)
90 secrets.EXPECT().GetSecret(mock.Anything, "gordon/apps/app-blog/web/database-url").Return("x", nil)
91 runtime.EXPECT().InspectImageVolumes(mock.Anything, rev.Spec.Services[0].Image).Return(nil, nil)
92 state.EXPECT().LoadCheckpoint(mock.Anything).Return(domain.AppStoreCheckpoint{}, nil)
93 state.EXPECT().LoadOwnership(mock.Anything, "blog").Return(domain.AppOwnership{App: "blog", ID: "app-blog"}, nil)
94}
95
96// expectSequentialCreate wires one replacement's candidate creation and loopback
97// publication. It records "create-new" and "start-new" in order.
98func expectSequentialCreate(
99 state *outmocks.MockAppState,
100 runtime *outmocks.MockContainerRuntime,
101 events *[]string,
102 containerID string,
103 hostPort int,
104) {
105 runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).RunAndReturn(
106 func(context.Context, *domain.ContainerConfig) (*domain.Container, error) {
107 recordEvent(events, "create-new")
108 return &domain.Container{ID: containerID, Name: "web"}, nil
109 }).Once()
110 runtime.EXPECT().StartContainer(mock.Anything, containerID).RunAndReturn(
111 func(context.Context, string) error {
112 recordEvent(events, "start-new")
113 return nil
114 }).Once()
115 // Deploy observes the execution start right after the start.
116 runtime.EXPECT().InspectContainer(mock.Anything, containerID).Return(
117 &domain.Container{ID: containerID, Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
118 runtime.EXPECT().GetContainerBackendBinds(mock.Anything, containerID, mock.Anything).Return(
119 []domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: hostPort, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
120 state.EXPECT().RegisterBackendBinds(mock.Anything, mock.Anything).Return(nil).Once()
121}
122
123// expectRetirementMarker wires the durable retirement-pending marker an
124// overlap replacement records for the superseded container before the switch.
125func expectRetirementMarker(state *outmocks.MockAppState, containerID string) {
126 state.EXPECT().SaveRecoveryInhibition(mock.Anything, mock.MatchedBy(func(i domain.AppRecoveryInhibition) bool {
127 return i.ContainerID == containerID && i.Reason == domain.AppInhibitRetirementPending
128 })).Return(nil).Once()
129}
130
131// expectRetireOld wires the confirmed retirement of the superseded container.
132func expectRetireOld(
133 state *outmocks.MockAppState,
134 runtime *outmocks.MockContainerRuntime,
135 events *[]string,
136 containerID string,
137) {
138 runtime.EXPECT().StopContainer(mock.Anything, containerID, mock.Anything).RunAndReturn(
139 func(context.Context, string, time.Duration) error {
140 recordEvent(events, "stop-old")
141 return nil
142 }).Once()
143 runtime.EXPECT().RemoveContainer(mock.Anything, containerID, false).RunAndReturn(
144 func(context.Context, string, bool) error {
145 recordEvent(events, "remove-old")
146 return nil
147 }).Once()
148 state.EXPECT().ReleaseBackendBinds(mock.Anything, "blog", containerID).Return(nil).Once()
149}
150
151// TestDeploy_StatelessReplacementOverlapsAndSwitchesBeforeRetiring proves the
152// replacement order for a public HTTP service without volumes, binds, or
153// devices: the candidate is created and ready while the superseded container
154// keeps serving, ACTIVE and the proxy graph are switched to it, and only then
155// is the superseded container stopped and removed. The old container is never
156// withdrawn from traffic and never inhibited.
157func TestDeploy_StatelessReplacementOverlapsAndSwitchesBeforeRetiring(t *testing.T) {
158 ctx := context.Background()
159 state, runtime, images, secrets := mockDeps(t)
160 rev := mockRevision()
161
162 expectDeployPreflight(state, runtime, images, secrets, rev)
163 expectNetworkProvision(runtime, "app-blog", 1)
164 state.EXPECT().LoadActive(mock.Anything, "blog").Return(replaceableActive(), true, nil)
165 state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
166
167 var order []string
168 expectSequentialCreate(state, runtime, &order, "c-new", 18080)
169 state.EXPECT().LoadActive(mock.Anything, "blog").Return(replaceableActive(), true, nil)
170 state.EXPECT().SaveActive(mock.Anything, mock.MatchedBy(func(active domain.AppActive) bool {
171 return active.Services["web"].Container == "c-new" && active.Services["web"].BackendBinds[8080] == 18080
172 })).RunAndReturn(
173 func(context.Context, domain.AppActive) error {
174 recordEvent(&order, "publish-active")
175 return nil
176 }).Once()
177 state.EXPECT().SaveOwnership(mock.Anything, mock.Anything).Return(nil)
178 expectRetirementMarker(state, "c-old")
179 // Confirmed retirement drops the marker.
180 state.EXPECT().ClearRecoveryInhibition(mock.Anything, "blog", "web", "c-old").Return(nil).Once()
181 expectRetireOld(state, runtime, &order, "c-old")
182
183 var probedURL string
184 traffic := &orderedTraffic{events: &order}
185 svc := deployment.NewService(deployment.Deps{
186 State: state, Runtime: runtime, Images: images, Secrets: secrets, Traffic: traffic,
187 }, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
188 func(_ context.Context, url, _ string) (int, error) {
189 probedURL = url
190 recordEvent(&order, "ready")
191 return 200, nil
192 },
193 func(context.Context, string) error { return nil },
194 ))
195
196 result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
197 require.NoError(t, err)
198 require.NotNil(t, result)
199 assert.Equal(t, "deployed", result.Services["web"].Result)
200 assert.Contains(t, probedURL, "127.0.0.1:18080", "readiness dials the replacement's fresh loopback bind")
201 assert.Equal(t,
202 []string{"create-new", "start-new", "ready", "publish-active", "traffic", "stop-old", "remove-old"},
203 order, "the old container serves until the new one is published and routed; it is never withdrawn")
204 assert.Empty(t, result.CleanupWarnings)
205}
206
207// TestDeploy_OverlapRetiresOldGenerationWithItsOwnStopGrace proves the
208// superseded container is stopped with the grace of the generation being
209// stopped (stop_grace from ACTIVE), not the grace of its replacement.
210func TestDeploy_OverlapRetiresOldGenerationWithItsOwnStopGrace(t *testing.T) {
211 ctx := context.Background()
212 state, runtime, images, secrets := mockDeps(t)
213 rev := mockRevision()
214 rev.Spec.Services[0].StopGrace = 3 * time.Second
215
216 oldActive := replaceableActive()
217 oldActive.Services["web"] = domain.AppEffectiveService{
218 EffectiveRevision: "rev-0", Image: "img:0", Container: "c-old", Spec: graceSpec(45 * time.Second),
219 }
220 expectDeployPreflight(state, runtime, images, secrets, rev)
221 expectNetworkProvision(runtime, "app-blog", 1)
222 state.EXPECT().LoadActive(mock.Anything, "blog").Return(oldActive, true, nil)
223 state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
224 expectSequentialCreate(state, runtime, nil, "c-new", 18080)
225 state.EXPECT().SaveActive(mock.Anything, mock.Anything).Return(nil).Once()
226 state.EXPECT().SaveOwnership(mock.Anything, mock.Anything).Return(nil)
227 expectRetirementMarker(state, "c-old")
228 state.EXPECT().ClearRecoveryInhibition(mock.Anything, "blog", "web", "c-old").Return(nil).Once()
229 runtime.EXPECT().StopContainer(mock.Anything, "c-old", 45*time.Second).Return(nil).Once()
230 runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(nil).Once()
231 state.EXPECT().ReleaseBackendBinds(mock.Anything, "blog", "c-old").Return(nil).Once()
232
233 svc := deployment.NewService(deployment.Deps{
234 State: state, Runtime: runtime, Images: images, Secrets: secrets, Traffic: &orderedTraffic{},
235 }, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
236 func(context.Context, string, string) (int, error) { return 200, nil },
237 func(context.Context, string) error { return nil },
238 ))
239
240 _, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
241 require.NoError(t, err)
242}
243
244// TestDeploy_OverlapOldRetirementFailureIsACleanupWarning proves a superseded
245// container that cannot be removed does not fail a deploy whose new generation
246// already serves: the leftover is a cleanup warning on the service and the
247// journal, and the step stays eligible for reconciliation.
248func TestDeploy_OverlapOldRetirementFailureIsACleanupWarning(t *testing.T) {
249 ctx := context.Background()
250 state, runtime, images, secrets := mockDeps(t)
251 rev := mockRevision()
252
253 expectDeployPreflight(state, runtime, images, secrets, rev)
254 expectNetworkProvision(runtime, "app-blog", 1)
255 state.EXPECT().LoadActive(mock.Anything, "blog").Return(replaceableActive(), true, nil)
256 state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
257 expectSequentialCreate(state, runtime, nil, "c-new", 18080)
258 state.EXPECT().SaveActive(mock.Anything, mock.Anything).Return(nil).Once()
259 state.EXPECT().SaveOwnership(mock.Anything, mock.Anything).Return(nil)
260 expectRetirementMarker(state, "c-old")
261 runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(nil).Once()
262 runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(assert.AnError).Once()
263
264 svc := deployment.NewService(deployment.Deps{
265 State: state, Runtime: runtime, Images: images, Secrets: secrets, Traffic: &orderedTraffic{},
266 }, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
267 func(context.Context, string, string) (int, error) { return 200, nil },
268 func(context.Context, string) error { return nil },
269 ))
270
271 result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
272 require.NoError(t, err, "the new generation serves: a retirement leftover is not a deploy failure")
273 assert.Equal(t, "deployed", result.Services["web"].Result)
274 require.Len(t, result.CleanupWarnings, 1)
275 assert.Equal(t, "c-old", result.CleanupWarnings[0].Leftover)
276 assert.Equal(t, "web", result.CleanupWarnings[0].Service)
277 assert.Contains(t, result.CleanupWarnings[0].Detail, "remove")
278 state.AssertNotCalled(t, "ReleaseBackendBinds", mock.Anything, "blog", "c-old")
279 state.AssertNotCalled(t, "ClearRecoveryInhibition", mock.Anything, mock.Anything, mock.Anything, mock.Anything)
280}
281
282// TestDeploy_PreflightFailureLeavesTheWorkloadUntouched proves preflight runs
283// to completion before anything is disrupted: an unresolvable image fails the
284// deploy without withdrawing traffic or stopping, removing, creating, or
285// starting any container.
286func TestDeploy_PreflightFailureLeavesTheWorkloadUntouched(t *testing.T) {
287 ctx := context.Background()
288 state, runtime, images, secrets := mockDeps(t)
289 rev := mockRevision()
290
291 state.EXPECT().Recover(mock.Anything).Return(nil).Once()
292 state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil).Once()
293 state.EXPECT().LoadOwnership(mock.Anything, "blog").Return(domain.AppOwnership{App: "blog", ID: "app-blog"}, nil)
294 images.EXPECT().ResolveDigest(mock.Anything, mock.Anything).Return("", assert.AnError).Once()
295 state.EXPECT().SaveOperation(mock.Anything, mock.Anything).Return(nil)
296
297 var events []string
298 svc := deployment.NewService(deployment.Deps{
299 State: state, Runtime: runtime, Images: images, Secrets: secrets,
300 Traffic: &orderedTraffic{events: &events},
301 }, zerowrap.Default())
302
303 _, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
304 require.ErrorIs(t, err, domain.ErrAppImageUnresolvable)
305 assert.Empty(t, events, "preflight must not touch traffic before it succeeds")
306 runtime.AssertNotCalled(t, "StopContainer", mock.Anything, mock.Anything, mock.Anything)
307 runtime.AssertNotCalled(t, "RemoveContainer", mock.Anything, mock.Anything, mock.Anything)
308 runtime.AssertNotCalled(t, "CreateContainer", mock.Anything, mock.Anything)
309 runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
310 runtime.AssertNotCalled(t, "ListNetworks", mock.Anything)
311 state.AssertNotCalled(t, "SaveActive", mock.Anything, mock.Anything)
312}
313
314// statefulMockRevision is mockRevision with a persistent volume, so the
315// service keeps the sequential replacement.
316func statefulMockRevision() domain.AppDesiredRevision {
317 rev := mockRevision()
318 rev.Spec.Services[0].Volumes = []domain.AppVolume{{Name: "data", Path: "/data"}}
319 return rev
320}
321
322// TestDeploy_WithdrawalFailureBlocksMutation proves a traffic withdrawal that
323// cannot be applied leaves the running generation of a single-writer service
324// untouched: no container is stopped, removed, created, or started, and no
325// ACTIVE state is published.
326func TestDeploy_WithdrawalFailureBlocksMutation(t *testing.T) {
327 ctx := context.Background()
328 state, runtime, images, secrets := mockDeps(t)
329 rev := statefulMockRevision()
330
331 expectDeployPreflight(state, runtime, images, secrets, rev)
332 runtime.EXPECT().VolumeExists(mock.Anything, mock.Anything).Return(false, nil)
333 state.EXPECT().LoadActive(mock.Anything, "blog").Return(replaceableActive(), true, nil)
334 state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
335
336 svc := deployment.NewService(deployment.Deps{
337 State: state, Runtime: runtime, Images: images, Secrets: secrets,
338 Traffic: &orderedTraffic{failWithdraw: true},
339 }, zerowrap.Default())
340
341 result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
342 require.Error(t, err)
343 require.NotNil(t, result)
344 assert.Equal(t, "failed", result.Services["web"].Result)
345 assert.Contains(t, result.Services["web"].Error, "withdraw")
346 runtime.AssertNotCalled(t, "StopContainer", mock.Anything, mock.Anything, mock.Anything)
347 runtime.AssertNotCalled(t, "RemoveContainer", mock.Anything, mock.Anything, mock.Anything)
348 runtime.AssertNotCalled(t, "CreateContainer", mock.Anything, mock.Anything)
349 runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
350 runtime.AssertNotCalled(t, "CreateVolume", mock.Anything, mock.Anything, mock.Anything)
351 runtime.AssertNotCalled(t, "ListNetworks", mock.Anything)
352 state.AssertNotCalled(t, "SaveActive", mock.Anything, mock.Anything)
353 state.AssertNotCalled(t, "ReleaseBackendBinds", mock.Anything, mock.Anything, mock.Anything)
354}
355
356// TestDeploy_FirstDeploymentRetiresNothing proves a service with no previous
357// ACTIVE generation is created, verified, and published without any
358// withdrawal or retirement step.
359func TestDeploy_FirstDeploymentRetiresNothing(t *testing.T) {
360 ctx := context.Background()
361 state, runtime, images, secrets := mockDeps(t)
362 rev := mockRevision()
363
364 expectDeployPreflight(state, runtime, images, secrets, rev)
365 expectNetworkProvision(runtime, "app-blog", 1)
366 state.EXPECT().LoadActive(mock.Anything, "blog").Return(domain.AppActive{
367 App: "blog", Services: map[string]domain.AppEffectiveService{},
368 }, true, nil)
369 state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
370
371 var order []string
372 expectSequentialCreate(state, runtime, &order, "c-new", 18080)
373 state.EXPECT().LoadActive(mock.Anything, "blog").Return(domain.AppActive{
374 App: "blog", Services: map[string]domain.AppEffectiveService{},
375 }, true, nil)
376 state.EXPECT().SaveActive(mock.Anything, mock.Anything).RunAndReturn(
377 func(context.Context, domain.AppActive) error {
378 recordEvent(&order, "publish-active")
379 return nil
380 }).Once()
381 state.EXPECT().SaveOwnership(mock.Anything, mock.Anything).Return(nil)
382
383 traffic := &orderedTraffic{events: &order}
384 svc := deployment.NewService(deployment.Deps{
385 State: state, Runtime: runtime, Images: images, Secrets: secrets, Traffic: traffic,
386 }, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
387 func(context.Context, string, string) (int, error) {
388 recordEvent(&order, "ready")
389 return 200, nil
390 },
391 func(context.Context, string) error { return nil },
392 ))
393
394 result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
395 require.NoError(t, err)
396 require.NotNil(t, result)
397 assert.Equal(t, "deployed", result.Services["web"].Result)
398 assert.Equal(t,
399 []string{"create-new", "start-new", "ready", "publish-active", "traffic"},
400 order,
401 "a first deployment withdraws nothing and retires nothing")
402 runtime.AssertNotCalled(t, "StopContainer", mock.Anything, mock.Anything, mock.Anything)
403 runtime.AssertNotCalled(t, "RemoveContainer", mock.Anything, mock.Anything, mock.Anything)
404}
405
406// TestDeploy_ReadinessFailureJournalsFailureAndRemovesCandidate proves a
407// replacement that is not ready is journaled as failed, that ACTIVE is never
408// published for it, and that the unready candidate is removed. The superseded
409// container keeps serving: it is never withdrawn, stopped, removed, or
410// inhibited.
411func TestDeploy_ReadinessFailureJournalsFailureAndRemovesCandidate(t *testing.T) {
412 ctx := context.Background()
413 state, runtime, images, secrets := mockDeps(t)
414 rev := mockRevision()
415 rev.Spec.Services[0].Readiness.Timeout = 50 * time.Millisecond
416
417 expectReplacementPreflight(state, runtime, images, secrets, rev)
418 expectNetworkProvision(runtime, "app-blog", 1)
419 state.EXPECT().LoadActive(mock.Anything, "blog").Return(replaceableActive(), true, nil)
420 state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
421
422 var order []string
423 expectSequentialCreate(state, runtime, &order, "c-new", 18080)
424 runtime.EXPECT().GetContainerLogs(mock.Anything, "c-new", false).Return(nil, assert.AnError).Once()
425 runtime.EXPECT().RemoveContainer(mock.Anything, "c-new", true).RunAndReturn(
426 func(context.Context, string, bool) error {
427 recordEvent(&order, "remove-candidate")
428 return nil
429 }).Once()
430 state.EXPECT().ReleaseBackendBinds(mock.Anything, "blog", "c-new").Return(nil).Once()
431
432 var saved []domain.AppOperation
433 state.EXPECT().SaveOperation(mock.Anything, mock.Anything).RunAndReturn(
434 func(_ context.Context, op domain.AppOperation) error {
435 saved = append(saved, op)
436 return nil
437 })
438
439 svc := deployment.NewService(deployment.Deps{
440 State: state, Runtime: runtime, Images: images, Secrets: secrets,
441 Traffic: &orderedTraffic{events: &order},
442 }, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
443 func(context.Context, string, string) (int, error) { return 503, nil },
444 func(context.Context, string) error { return assert.AnError },
445 ))
446
447 result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
448 require.Error(t, err)
449 require.NotNil(t, result)
450 assert.Equal(t, "failed", result.Services["web"].Result)
451 assert.Contains(t, result.Services["web"].Error, "readiness", "the failure names the readiness probe")
452 assert.NotContains(t, result.Services["web"].Error, "logs:", "raw logs must never be embedded in the public error")
453 assert.Equal(t, "c-new", result.Services["web"].After)
454
455 // ACTIVE was never repointed and the old container was never touched.
456 state.AssertNotCalled(t, "SaveActive", mock.Anything, mock.Anything)
457 state.AssertNotCalled(t, "ClearRecoveryInhibition", mock.Anything, mock.Anything, mock.Anything, mock.Anything)
458 state.AssertNotCalled(t, "SaveRecoveryInhibition", mock.Anything, mock.Anything)
459
460 require.NotEmpty(t, saved)
461 last := saved[len(saved)-1]
462 assert.Equal(t, domain.AppOutcomeFailed, last.Outcome, "a failed replacement must not be journaled as success")
463 step, ok := opStep(last, "service.web.replace")
464 require.True(t, ok)
465 assert.Equal(t, domain.AppStepFailed, step.State)
466
467 assert.Equal(t, []string{"create-new", "start-new", "remove-candidate"}, order,
468 "the old generation is never withdrawn: no withdraw or traffic event, the unready candidate is removed")
469 runtime.AssertNotCalled(t, "StopContainer", mock.Anything, mock.Anything, mock.Anything)
470 runtime.AssertNotCalled(t, "RemoveContainer", mock.Anything, "c-old", mock.Anything)
471 state.AssertNotCalled(t, "ReleaseBackendBinds", mock.Anything, "blog", "c-old")
472}
473
474// TestDeploy_CandidateCleanupFailureStaysVisible proves a candidate that
475// cannot be removed does not turn a failed replacement into a success: the
476// leftover is reported as a cleanup warning on the failed service.
477func TestDeploy_CandidateCleanupFailureStaysVisible(t *testing.T) {
478 ctx := context.Background()
479 state, runtime, images, secrets := mockDeps(t)
480 rev := mockRevision()
481 rev.Spec.Services[0].Readiness.Timeout = 50 * time.Millisecond
482
483 expectDeployPreflight(state, runtime, images, secrets, rev)
484 expectNetworkProvision(runtime, "app-blog", 1)
485 state.EXPECT().LoadActive(mock.Anything, "blog").Return(replaceableActive(), true, nil)
486 state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
487
488 expectSequentialCreate(state, runtime, nil, "c-new", 18080)
489 runtime.EXPECT().GetContainerLogs(mock.Anything, "c-new", false).Return(nil, assert.AnError).Once()
490 runtime.EXPECT().RemoveContainer(mock.Anything, "c-new", true).Return(assert.AnError).Once()
491
492 svc := deployment.NewService(deployment.Deps{
493 State: state, Runtime: runtime, Images: images, Secrets: secrets,
494 }, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
495 func(context.Context, string, string) (int, error) { return 503, nil },
496 func(context.Context, string) error { return assert.AnError },
497 ))
498
499 result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
500 require.Error(t, err)
501 require.NotNil(t, result)
502 svcResult := result.Services["web"]
503 assert.Equal(t, "failed", svcResult.Result)
504 require.Len(t, svcResult.CleanupWarnings, 1, "an unremovable candidate is reported, never hidden")
505 assert.Equal(t, "web", svcResult.CleanupWarnings[0].Service)
506 assert.Equal(t, "c-new", svcResult.CleanupWarnings[0].Leftover)
507 assert.Contains(t, svcResult.CleanupWarnings[0].Detail, "remove")
508 state.AssertNotCalled(t, "ReleaseBackendBinds", mock.Anything, "blog", "c-new")
509 state.AssertNotCalled(t, "SaveActive", mock.Anything, mock.Anything)
510}
511
512// TestDeploy_CancellationDuringReadinessIsNotReportedAsSuccess proves a
513// cancellation during the readiness wait never becomes a success: ACTIVE is
514// not published, no terminal outcome is written, and the candidate cleanup
515// that the canceled context could not complete is reported as a leftover.
516// The durable store and the runtime refuse work on a canceled context, so
517// this test models both.
518func TestDeploy_CancellationDuringReadinessIsNotReportedAsSuccess(t *testing.T) {
519 ctx, cancel := context.WithCancel(context.Background())
520 defer cancel()
521 state, runtime, images, secrets := mockDeps(t)
522 rev := mockRevision()
523 rev.Spec.Services[0].Readiness.Timeout = 30 * time.Second
524
525 expectReplacementPreflight(state, runtime, images, secrets, rev)
526 expectNetworkProvision(runtime, "app-blog", 1)
527 state.EXPECT().LoadActive(mock.Anything, "blog").Return(replaceableActive(), true, nil)
528 state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
529
530 expectSequentialCreate(state, runtime, nil, "c-new", 18080)
531 // The runtime refuses to remove the candidate with a canceled context.
532 runtime.EXPECT().RemoveContainer(mock.Anything, "c-new", true).Return(context.Canceled).Once()
533
534 var saved []domain.AppOperation
535 state.EXPECT().SaveOperation(mock.Anything, mock.Anything).RunAndReturn(
536 func(opCtx context.Context, op domain.AppOperation) error {
537 if opCtx.Err() != nil {
538 // The durable store refuses writes on a canceled context, so
539 // the operation stays in flight instead of becoming terminal.
540 return opCtx.Err()
541 }
542 saved = append(saved, op)
543 return nil
544 })
545
546 probeStarted := make(chan struct{})
547 svc := deployment.NewService(deployment.Deps{
548 State: state, Runtime: runtime, Images: images, Secrets: secrets,
549 }, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
550 func(probeCtx context.Context, _, _ string) (int, error) {
551 select {
552 case <-probeStarted:
553 default:
554 close(probeStarted)
555 }
556 <-probeCtx.Done()
557 return 0, probeCtx.Err()
558 },
559 func(context.Context, string) error { return nil },
560 ))
561
562 done := make(chan error, 1)
563 var deployResult *deployment.DeployResult
564 go func() {
565 result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
566 deployResult = result
567 done <- err
568 }()
569 select {
570 case <-probeStarted:
571 case <-time.After(5 * time.Second):
572 t.Fatal("timed out waiting for the readiness probe")
573 }
574 cancel()
575 err := <-done
576 require.Error(t, err)
577 assert.Contains(t, err.Error(), context.Canceled.Error(), "the canceled probe is reported")
578
579 // No success is claimed anywhere: ACTIVE is untouched, and no journaled
580 // outcome records the replacement as succeeded. The durable store refuses
581 // the terminal write on a canceled context, so the operation stays in
582 // flight rather than becoming a false success.
583 state.AssertNotCalled(t, "SaveActive", mock.Anything, mock.Anything)
584 require.NotEmpty(t, saved)
585 for _, op := range saved {
586 assert.NotContains(t, []string{domain.AppOutcomeSuccess, domain.AppOutcomePartial}, op.Outcome,
587 "a canceled replacement is never journaled as a success")
588 if step, ok := opStep(op, "service.web.replace"); ok {
589 assert.NotEqual(t, domain.AppStepSucceeded, step.State,
590 "a canceled replacement step is never recorded as succeeded")
591 }
592 }
593
594 // The canceled cleanup is best effort, and what it could not finish is
595 // reported instead of being hidden.
596 require.NotNil(t, deployResult)
597 require.Len(t, deployResult.Services["web"].CleanupWarnings, 1,
598 "a candidate the canceled cleanup could not remove is reported")
599 assert.Equal(t, "c-new", deployResult.Services["web"].CleanupWarnings[0].Leftover)
600 runtime.AssertCalled(t, "RemoveContainer", mock.Anything, "c-new", true)
601 state.AssertNotCalled(t, "ReleaseBackendBinds", mock.Anything, "blog", "c-new")
602}
603
604// TestDeploy_MultiServiceStopsAtFirstFailure proves services are deployed in
605// sorted order and the first failure stops the run: the failing service is
606// cleaned up, later services are untouched, and the services already deployed
607// are kept without any rollback.
608func TestDeploy_MultiServiceStopsAtFirstFailure(t *testing.T) {
609 ctx := context.Background()
610 state, runtime, images, secrets := mockDeps(t)
611
612 httpService := func(name, host, image string, timeout time.Duration) domain.AppService {
613 return domain.AppService{
614 Name: name, Image: image,
615 Readiness: domain.AppReadiness{Type: domain.AppReadinessHTTP, Path: "/healthz", Timeout: timeout},
616 HTTP: []domain.AppHTTPInterface{{Host: host, Port: 8080, TLS: domain.AppTLSAuto}},
617 }
618 }
619 rev := testRevision("blog",
620 httpService("alpha", "alpha.example.com", "docker.io/example/alpha:1.4.2", time.Second),
621 httpService("beta", "beta.example.com", "docker.io/example/beta:1.4.2", 50*time.Millisecond),
622 httpService("gamma", "gamma.example.com", "docker.io/example/gamma:1.4.2", time.Second),
623 )
624
625 state.EXPECT().Recover(mock.Anything).Return(nil)
626 state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
627 images.EXPECT().ResolveDigest(mock.Anything, "docker.io/example/alpha:1.4.2").Return("sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", nil).Once()
628 images.EXPECT().ResolveDigest(mock.Anything, "docker.io/example/beta:1.4.2").Return("sha256:"+strings.Repeat("b", 64), nil).Once()
629 images.EXPECT().ResolveDigest(mock.Anything, "docker.io/example/gamma:1.4.2").Return("sha256:"+strings.Repeat("c", 64), nil).Once()
630 runtime.EXPECT().InspectImageVolumes(mock.Anything, "docker.io/example/alpha:1.4.2").Return(nil, nil).Once()
631 runtime.EXPECT().InspectImageVolumes(mock.Anything, "docker.io/example/beta:1.4.2").Return(nil, nil).Once()
632 runtime.EXPECT().InspectImageVolumes(mock.Anything, "docker.io/example/gamma:1.4.2").Return(nil, nil).Once()
633 state.EXPECT().LoadCheckpoint(mock.Anything).Return(domain.AppStoreCheckpoint{}, nil)
634 state.EXPECT().LoadOwnership(mock.Anything, "blog").Return(domain.AppOwnership{App: "blog", ID: "app-blog"}, nil)
635 state.EXPECT().LoadActive(mock.Anything, "blog").Return(domain.AppActive{
636 App: "blog", Services: map[string]domain.AppEffectiveService{},
637 }, true, nil)
638 state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
639 expectNetworkProvision(runtime, "app-blog", 2)
640
641 expectSequentialCreate(state, runtime, nil, "c-alpha", 18080)
642 state.EXPECT().SaveActive(mock.Anything, mock.MatchedBy(func(active domain.AppActive) bool {
643 return active.Services["alpha"].Container == "c-alpha"
644 })).Return(nil).Once()
645 state.EXPECT().SaveOwnership(mock.Anything, mock.Anything).Return(nil)
646
647 runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-beta", mock.Anything).Return(
648 []domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18081, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
649 state.EXPECT().RegisterBackendBinds(mock.Anything, mock.Anything).Return(nil).Once()
650 runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-beta", Name: "beta"}, nil).Once()
651 runtime.EXPECT().StartContainer(mock.Anything, "c-beta").Return(nil).Once()
652 runtime.EXPECT().InspectContainer(mock.Anything, "c-beta").Return(&domain.Container{ID: "c-beta", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
653 runtime.EXPECT().GetContainerLogs(mock.Anything, "c-beta", false).Return(nil, assert.AnError).Once()
654 runtime.EXPECT().RemoveContainer(mock.Anything, "c-beta", true).Return(nil).Once()
655 state.EXPECT().ReleaseBackendBinds(mock.Anything, "blog", "c-beta").Return(nil).Once()
656
657 var saved []domain.AppOperation
658 state.EXPECT().SaveOperation(mock.Anything, mock.Anything).RunAndReturn(
659 func(_ context.Context, op domain.AppOperation) error {
660 saved = append(saved, op)
661 return nil
662 })
663
664 traffic := &orderedTraffic{}
665 svc := deployment.NewService(deployment.Deps{
666 State: state, Runtime: runtime, Images: images, Secrets: secrets, Traffic: traffic,
667 }, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
668 func(_ context.Context, url, _ string) (int, error) {
669 if strings.Contains(url, "18081") {
670 return 503, nil
671 }
672 return 200, nil
673 },
674 func(context.Context, string) error { return nil },
675 ))
676
677 result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
678 require.Error(t, err)
679 require.NotNil(t, result)
680 assert.Equal(t, "deployed", result.Services["alpha"].Result, "an already deployed service is kept")
681 assert.Equal(t, "failed", result.Services["beta"].Result)
682 assert.NotContains(t, result.Services, "gamma", "a later service is never attempted after a failure")
683
684 runtime.AssertNumberOfCalls(t, "CreateContainer", 2)
685 runtime.AssertNotCalled(t, "CreateContainer", mock.Anything, mock.MatchedBy(func(cfg *domain.ContainerConfig) bool {
686 return cfg.Name == "gamma"
687 }))
688 // No rollback of the already deployed service.
689 runtime.AssertNotCalled(t, "StopContainer", mock.Anything, mock.Anything, mock.Anything)
690 runtime.AssertNotCalled(t, "RemoveContainer", mock.Anything, "c-alpha", mock.Anything)
691 state.AssertNumberOfCalls(t, "SaveActive", 1)
692
693 require.NotEmpty(t, saved)
694 last := saved[len(saved)-1]
695 assert.Equal(t, domain.AppOutcomePartial, last.Outcome,
696 "the run stops at the first failure; the earlier success is kept, not rolled back")
697}
698
699// TestDeploy_VolumeReplacementPreventsOverlappingWriters proves a stateful
700// replacement durably inhibits the superseded generation before it is
701// retired, stops and removes it before the candidate starts, and keeps the
702// inhibition while the candidate's removal cannot be confirmed. Exclusion
703// holds because the previous process is gone before the replacement can write.
704func TestDeploy_VolumeReplacementPreventsOverlappingWriters(t *testing.T) {
705 ctx := context.Background()
706 store := newTestStore(t)
707 runtime := outmocks.NewMockContainerRuntime(t)
708 images := outmocks.NewMockImageResolver(t)
709 secrets := outmocks.NewMockSecretProvider(t)
710
711 image := "registry.example.com/blog/web:1.4.2"
712 spec := domain.AppService{
713 Name: "web",
714 Image: image,
715 Secrets: map[string]string{},
716 Volumes: []domain.AppVolume{{Name: "data", Path: "/data"}},
717 HTTP: []domain.AppHTTPInterface{{Host: "blog.example.com", Port: 8080, TLS: domain.AppTLSAuto}},
718 Readiness: domain.AppReadiness{
719 Type: domain.AppReadinessHTTP, Path: "/healthz", Timeout: 50 * time.Millisecond,
720 },
721 }
722 rev := testRevision("blog", spec)
723
724 require.NoError(t, store.SaveIntent(ctx, domain.AppStopIntent{App: "blog"}))
725 require.NoError(t, store.SaveActive(ctx, domain.AppActive{
726 App: "blog",
727 Services: map[string]domain.AppEffectiveService{
728 "web": {
729 Container: "c-old", EffectiveRevision: "rev-0", Spec: spec,
730 BackendBinds: map[int]int{8080: 18080},
731 },
732 },
733 }))
734 seedDesired(t, ctx, store, rev)
735 // The app keeps one incarnation: the private network every service joins
736 // is derived from it.
737 require.NoError(t, store.SaveOwnership(ctx, domain.AppOwnership{App: "blog", ID: "app-blog"}))
738 expectNetworkProvision(runtime, "app-blog", 1)
739
740 images.EXPECT().ResolveDigest(mock.Anything, image).Return("sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", nil).Once()
741 runtime.EXPECT().InspectImageVolumes(mock.Anything, image).Return(nil, nil).Once()
742 runtime.EXPECT().VolumeExists(mock.Anything, mock.Anything).Return(false, nil).Once()
743
744 var order []string
745 inhibitedBeforeStop := false
746 runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).RunAndReturn(
747 func(context.Context, string, time.Duration) error {
748 recordEvent(&order, "stop-old")
749 // The durable marker must already exist before the superseded
750 // writer is stopped: recovery must never race the destructive step.
751 inhibitions, loadErr := store.LoadRecoveryInhibitions(context.Background(), "blog")
752 require.NoError(t, loadErr)
753 inhibitedBeforeStop = len(inhibitions) == 1 &&
754 inhibitions[0].ContainerID == "c-old" &&
755 inhibitions[0].Reason == domain.AppInhibitReplacementPending
756 return nil
757 }).Once()
758 runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).RunAndReturn(
759 func(context.Context, string, bool) error {
760 recordEvent(&order, "remove-old")
761 return nil
762 }).Once()
763 runtime.EXPECT().CreateVolume(mock.Anything, mock.Anything, mock.Anything).RunAndReturn(
764 func(context.Context, string, map[string]string) error {
765 recordEvent(&order, "create-volume")
766 return nil
767 }).Once()
768 runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).RunAndReturn(
769 func(context.Context, *domain.ContainerConfig) (*domain.Container, error) {
770 recordEvent(&order, "create-new")
771 return &domain.Container{ID: "c-new", Name: "web"}, nil
772 }).Once()
773 runtime.EXPECT().StartContainer(mock.Anything, "c-new").RunAndReturn(
774 func(context.Context, string) error {
775 recordEvent(&order, "start-new")
776 return nil
777 }).Once()
778 runtime.EXPECT().InspectContainer(mock.Anything, "c-new").Return(&domain.Container{ID: "c-new", Status: "running", StartedAt: executionStartedAt}, nil).Maybe()
779 runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return(
780 []domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18081, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
781 runtime.EXPECT().GetContainerLogs(mock.Anything, "c-new", false).Return(nil, assert.AnError).Once()
782 // The failed replacement removes its candidate once. Boot reconciliation
783 // retries it because a terminal failed operation cannot know the earlier
784 // removal succeeded without re-checking the runtime; that retry fails, so
785 // recovery fails closed and never revives the superseded writer whose
786 // volume the orphan may still touch.
787 runtime.EXPECT().RemoveContainer(mock.Anything, "c-new", true).Return(nil).Once()
788 runtime.EXPECT().RemoveContainer(mock.Anything, "c-new", true).Return(assert.AnError).Once()
789
790 svc := deployment.NewService(deployment.Deps{
791 State: store, Runtime: runtime, Images: images, Secrets: secrets,
792 ImagePolicy: domain.ImageSourcePolicy{AllowedRegistries: []string{"registry.example.com"}},
793 }, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
794 func(context.Context, string, string) (int, error) { return 503, nil },
795 func(context.Context, string) error { return assert.AnError },
796 ))
797
798 _, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
799 require.Error(t, err)
800
801 assert.Equal(t, []string{"stop-old", "remove-old", "create-volume", "create-new", "start-new"}, order,
802 "the superseded writer is gone before the replacement can write")
803 assert.True(t, inhibitedBeforeStop,
804 "the durable recovery inhibition exists before the superseded writer is stopped")
805
806 // The failed replacement never cleared the durable inhibition: a later
807 // boot must not revive the old writer on top of data the candidate may
808 // already have touched.
809 inhibitions, loadErr := store.LoadRecoveryInhibitions(ctx, "blog")
810 require.NoError(t, loadErr)
811 require.Len(t, inhibitions, 1)
812 assert.Equal(t, "c-old", inhibitions[0].ContainerID)
813 assert.Equal(t, domain.AppInhibitReplacementPending, inhibitions[0].Reason)
814
815 // ACTIVE still names the superseded generation: the failure is explicit
816 // and no availability was restored on its behalf.
817 active, ok, loadErr := store.LoadActive(ctx, "blog")
818 require.NoError(t, loadErr)
819 require.True(t, ok)
820 assert.Equal(t, "c-old", active.Services["web"].Container)
821
822 // The failed replacement keeps its candidate in the journal: a leftover
823 // must stay traceable so reconciliation can converge it.
824 failedOp, hasOp, loadErr := store.LoadLatestOperation(ctx, "blog")
825 require.NoError(t, loadErr)
826 require.True(t, hasOp)
827 failedStep, found := opStep(failedOp, "service.web.replace")
828 require.True(t, found)
829 assert.Equal(t, domain.AppStepFailed, failedStep.State)
830 assert.Equal(t, "c-new", failedStep.After, "the created candidate stays recorded")
831
832 rebooted := deployment.NewService(deployment.Deps{
833 State: store, Runtime: runtime, Images: images, Secrets: secrets,
834 }, zerowrap.Default())
835 bootErr := rebooted.ReconcileBoot(ctx)
836 require.Error(t, bootErr)
837 assert.Contains(t, bootErr.Error(), "could not be removed")
838 runtime.AssertNotCalled(t, "StartContainer", mock.Anything, "c-old")
839 runtime.AssertNotCalled(t, "RestartContainer", mock.Anything, "c-old", mock.Anything)
840
841 // The orphan that could not be removed keeps the inhibition: the
842 // superseded writer is never revived on top of data the candidate may
843 // already have touched.
844 keptInhibitions, keptErr := store.LoadRecoveryInhibitions(ctx, "blog")
845 require.NoError(t, keptErr)
846 require.Len(t, keptInhibitions, 1)
847 assert.Equal(t, "c-old", keptInhibitions[0].ContainerID)
848 assert.Equal(t, domain.AppInhibitReplacementPending, keptInhibitions[0].Reason)
849}