1package deployment_test
2
3import (
4 "context"
5 "strings"
6 "sync"
7 "testing"
8 "time"
9
10 "github.com/bnema/zerowrap"
11 "github.com/stretchr/testify/assert"
12 "github.com/stretchr/testify/mock"
13 "github.com/stretchr/testify/require"
14
15 outmocks "github.com/bnema/gordon/internal/boundaries/out/mocks"
16 "github.com/bnema/gordon/internal/domain"
17 "github.com/bnema/gordon/internal/usecase/deployment"
18)
19
20// orderedTraffic records traffic boundary calls into a shared event log and
21// can fail one withdrawal or one graph apply.
22type orderedTraffic struct {
23 mu sync.Mutex
24 events *[]string
25 failWithdraw bool
26 failRebuild bool
27}
28
29func (t *orderedTraffic) record(event string) {
30 t.mu.Lock()
31 defer t.mu.Unlock()
32 if t.events != nil {
33 *t.events = append(*t.events, event)
34 }
35}
36
37func (t *orderedTraffic) RebuildTraffic(context.Context) error {
38 t.record("traffic")
39 if t.failRebuild {
40 return assert.AnError
41 }
42 return nil
43}
44
45func (t *orderedTraffic) WithdrawService(context.Context, string, string) error {
46 t.record("withdraw")
47 if t.failWithdraw {
48 return assert.AnError
49 }
50 return nil
51}
52
53func (t *orderedTraffic) WithdrawServiceState(context.Context, string, string) error {
54 t.record("withdraw-state")
55 if t.failWithdraw {
56 return assert.AnError
57 }
58 return nil
59}
60
61// replaceableActive is one ACTIVE record whose web service is superseded.
62func replaceableActive() domain.AppActive {
63 return domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
64 "web": {EffectiveRevision: "rev-0", Image: "img:0", Container: "c-old"},
65 }}
66}
67
68// recordEvent appends one step to a shared event log. Test doubles and mock
69// hooks share the log, so the sequence a deployment performs is observable.
70func recordEvent(events *[]string, event string) {
71 if events != nil {
72 *events = append(*events, event)
73 }
74}
75
76// expectReplacementPreflight wires the preflight calls of one replacement
77// without pre-registering an operation-journal write, so a test can observe
78// the terminal journal write itself.
79func expectReplacementPreflight(
80 state *outmocks.MockAppState,
81 runtime *outmocks.MockContainerRuntime,
82 images *outmocks.MockImageResolver,
83 secrets *outmocks.MockSecretProvider,
84 rev domain.AppDesiredRevision,
85) {
86 state.EXPECT().Recover(mock.Anything).Return(nil)
87 state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
88 images.EXPECT().ResolveDigest(mock.Anything, rev.Spec.Services[0].Image).Return(
89 "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", nil)
90 secrets.EXPECT().GetSecret(mock.Anything, "gordon/apps/app-blog/web/database-url").Return("x", nil)
91 runtime.EXPECT().InspectImageVolumes(mock.Anything, rev.Spec.Services[0].Image).Return(nil, nil)
92 state.EXPECT().LoadCheckpoint(mock.Anything).Return(domain.AppStoreCheckpoint{}, nil)
93 state.EXPECT().LoadOwnership(mock.Anything, "blog").Return(domain.AppOwnership{App: "blog", ID: "app-blog"}, nil)
94}
95
96// expectSequentialCreate wires one replacement's candidate creation and loopback
97// publication. It records "create-new" and "start-new" in order.
98func expectSequentialCreate(
99 state *outmocks.MockAppState,
100 runtime *outmocks.MockContainerRuntime,
101 events *[]string,
102 containerID string,
103 hostPort int,
104) {
105 runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).RunAndReturn(
106 func(context.Context, *domain.ContainerConfig) (*domain.Container, error) {
107 recordEvent(events, "create-new")
108 return &domain.Container{ID: containerID, Name: "web"}, nil
109 }).Once()
110 runtime.EXPECT().StartContainer(mock.Anything, containerID).RunAndReturn(
111 func(context.Context, string) error {
112 recordEvent(events, "start-new")
113 return nil
114 }).Once()
115 runtime.EXPECT().GetContainerBackendBinds(mock.Anything, containerID, mock.Anything).Return(
116 []domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: hostPort, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
117 state.EXPECT().RegisterBackendBinds(mock.Anything, mock.Anything).Return(nil).Once()
118}
119
120// expectRetireOld wires the confirmed retirement of the superseded container.
121func expectRetireOld(
122 state *outmocks.MockAppState,
123 runtime *outmocks.MockContainerRuntime,
124 events *[]string,
125 containerID string,
126) {
127 runtime.EXPECT().StopContainer(mock.Anything, containerID, mock.Anything).RunAndReturn(
128 func(context.Context, string, time.Duration) error {
129 recordEvent(events, "stop-old")
130 return nil
131 }).Once()
132 runtime.EXPECT().RemoveContainer(mock.Anything, containerID, false).RunAndReturn(
133 func(context.Context, string, bool) error {
134 recordEvent(events, "remove-old")
135 return nil
136 }).Once()
137 state.EXPECT().ReleaseBackendBinds(mock.Anything, "blog", containerID).Return(nil).Once()
138}
139
140// TestDeploy_SequentialReplacementNeverOverlapsGenerations proves the
141// replacement order for a public HTTP service without volumes: withdrawal is
142// confirmed first, the superseded container is stopped and removed before the
143// replacement is created, readiness runs before ACTIVE is published, and the
144// graph is applied last.
145func TestDeploy_SequentialReplacementNeverOverlapsGenerations(t *testing.T) {
146 ctx := context.Background()
147 state, runtime, images, secrets := mockDeps(t)
148 rev := mockRevision()
149
150 expectDeployPreflight(state, runtime, images, secrets, rev)
151 expectNetworkProvision(runtime, "app-blog", 1)
152 state.EXPECT().LoadActive(mock.Anything, "blog").Return(replaceableActive(), true, nil)
153 state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
154
155 var order []string
156 expectRetireOld(state, runtime, &order, "c-old")
157 expectSequentialCreate(state, runtime, &order, "c-new", 18080)
158 state.EXPECT().LoadActive(mock.Anything, "blog").Return(replaceableActive(), true, nil)
159 state.EXPECT().SaveActive(mock.Anything, mock.Anything).RunAndReturn(
160 func(context.Context, domain.AppActive) error {
161 recordEvent(&order, "publish-active")
162 return nil
163 }).Once()
164 state.EXPECT().SaveOwnership(mock.Anything, mock.Anything).Return(nil)
165 state.EXPECT().ClearRecoveryInhibition(mock.Anything, "blog", "web", "c-old").Return(nil).Once()
166
167 var probedURL string
168 traffic := &orderedTraffic{events: &order}
169 svc := deployment.NewService(deployment.Deps{
170 State: state, Runtime: runtime, Images: images, Secrets: secrets, Traffic: traffic,
171 }, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
172 func(_ context.Context, url, _ string) (int, error) {
173 probedURL = url
174 recordEvent(&order, "ready")
175 return 200, nil
176 },
177 func(context.Context, string) error { return nil },
178 ))
179
180 result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
181 require.NoError(t, err)
182 require.NotNil(t, result)
183 assert.Equal(t, "deployed", result.Services["web"].Result)
184 assert.Contains(t, probedURL, "127.0.0.1:18080", "readiness dials the replacement's fresh loopback bind")
185 assert.Equal(t,
186 []string{"withdraw", "stop-old", "remove-old", "create-new", "start-new", "ready", "publish-active", "traffic"},
187 order)
188}
189
190// TestDeploy_PreflightFailureLeavesTheWorkloadUntouched proves preflight runs
191// to completion before anything is disrupted: an unresolvable image fails the
192// deploy without withdrawing traffic or stopping, removing, creating, or
193// starting any container.
194func TestDeploy_PreflightFailureLeavesTheWorkloadUntouched(t *testing.T) {
195 ctx := context.Background()
196 state, runtime, images, secrets := mockDeps(t)
197 rev := mockRevision()
198
199 state.EXPECT().Recover(mock.Anything).Return(nil).Once()
200 state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil).Once()
201 state.EXPECT().LoadOwnership(mock.Anything, "blog").Return(domain.AppOwnership{App: "blog", ID: "app-blog"}, nil)
202 images.EXPECT().ResolveDigest(mock.Anything, mock.Anything).Return("", assert.AnError).Once()
203 state.EXPECT().SaveOperation(mock.Anything, mock.Anything).Return(nil)
204
205 var events []string
206 svc := deployment.NewService(deployment.Deps{
207 State: state, Runtime: runtime, Images: images, Secrets: secrets,
208 Traffic: &orderedTraffic{events: &events},
209 }, zerowrap.Default())
210
211 _, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
212 require.ErrorIs(t, err, domain.ErrAppImageUnresolvable)
213 assert.Empty(t, events, "preflight must not touch traffic before it succeeds")
214 runtime.AssertNotCalled(t, "StopContainer", mock.Anything, mock.Anything, mock.Anything)
215 runtime.AssertNotCalled(t, "RemoveContainer", mock.Anything, mock.Anything, mock.Anything)
216 runtime.AssertNotCalled(t, "CreateContainer", mock.Anything, mock.Anything)
217 runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
218 runtime.AssertNotCalled(t, "ListNetworks", mock.Anything)
219 state.AssertNotCalled(t, "SaveActive", mock.Anything, mock.Anything)
220}
221
222// TestDeploy_WithdrawalFailureBlocksMutation proves a traffic withdrawal that
223// cannot be applied leaves the running generation untouched: no container is
224// stopped, removed, created, or started, and no ACTIVE state is published.
225func TestDeploy_WithdrawalFailureBlocksMutation(t *testing.T) {
226 ctx := context.Background()
227 state, runtime, images, secrets := mockDeps(t)
228 rev := mockRevision()
229
230 expectDeployPreflight(state, runtime, images, secrets, rev)
231 state.EXPECT().LoadActive(mock.Anything, "blog").Return(replaceableActive(), true, nil)
232 state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
233
234 svc := deployment.NewService(deployment.Deps{
235 State: state, Runtime: runtime, Images: images, Secrets: secrets,
236 Traffic: &orderedTraffic{failWithdraw: true},
237 }, zerowrap.Default())
238
239 result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
240 require.Error(t, err)
241 require.NotNil(t, result)
242 assert.Equal(t, "failed", result.Services["web"].Result)
243 assert.Contains(t, result.Services["web"].Error, "withdraw")
244 runtime.AssertNotCalled(t, "StopContainer", mock.Anything, mock.Anything, mock.Anything)
245 runtime.AssertNotCalled(t, "RemoveContainer", mock.Anything, mock.Anything, mock.Anything)
246 runtime.AssertNotCalled(t, "CreateContainer", mock.Anything, mock.Anything)
247 runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
248 runtime.AssertNotCalled(t, "CreateVolume", mock.Anything, mock.Anything, mock.Anything)
249 runtime.AssertNotCalled(t, "ListNetworks", mock.Anything)
250 state.AssertNotCalled(t, "SaveActive", mock.Anything, mock.Anything)
251 state.AssertNotCalled(t, "ReleaseBackendBinds", mock.Anything, mock.Anything, mock.Anything)
252}
253
254// TestDeploy_FirstDeploymentRetiresNothing proves a service with no previous
255// ACTIVE generation is created, verified, and published without any
256// withdrawal or retirement step.
257func TestDeploy_FirstDeploymentRetiresNothing(t *testing.T) {
258 ctx := context.Background()
259 state, runtime, images, secrets := mockDeps(t)
260 rev := mockRevision()
261
262 expectDeployPreflight(state, runtime, images, secrets, rev)
263 expectNetworkProvision(runtime, "app-blog", 1)
264 state.EXPECT().LoadActive(mock.Anything, "blog").Return(domain.AppActive{
265 App: "blog", Services: map[string]domain.AppEffectiveService{},
266 }, true, nil)
267 state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
268
269 var order []string
270 expectSequentialCreate(state, runtime, &order, "c-new", 18080)
271 state.EXPECT().LoadActive(mock.Anything, "blog").Return(domain.AppActive{
272 App: "blog", Services: map[string]domain.AppEffectiveService{},
273 }, true, nil)
274 state.EXPECT().SaveActive(mock.Anything, mock.Anything).RunAndReturn(
275 func(context.Context, domain.AppActive) error {
276 recordEvent(&order, "publish-active")
277 return nil
278 }).Once()
279 state.EXPECT().SaveOwnership(mock.Anything, mock.Anything).Return(nil)
280
281 traffic := &orderedTraffic{events: &order}
282 svc := deployment.NewService(deployment.Deps{
283 State: state, Runtime: runtime, Images: images, Secrets: secrets, Traffic: traffic,
284 }, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
285 func(context.Context, string, string) (int, error) {
286 recordEvent(&order, "ready")
287 return 200, nil
288 },
289 func(context.Context, string) error { return nil },
290 ))
291
292 result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
293 require.NoError(t, err)
294 require.NotNil(t, result)
295 assert.Equal(t, "deployed", result.Services["web"].Result)
296 assert.Equal(t,
297 []string{"create-new", "start-new", "ready", "publish-active", "traffic"},
298 order,
299 "a first deployment withdraws nothing and retires nothing")
300 runtime.AssertNotCalled(t, "StopContainer", mock.Anything, mock.Anything, mock.Anything)
301 runtime.AssertNotCalled(t, "RemoveContainer", mock.Anything, mock.Anything, mock.Anything)
302}
303
304// TestDeploy_ReadinessFailureJournalsFailureAndRemovesCandidate proves a
305// replacement that is not ready is journaled as failed, that ACTIVE is never
306// published for it, and that the unready candidate is removed. The superseded
307// container is already gone and is never recreated.
308func TestDeploy_ReadinessFailureJournalsFailureAndRemovesCandidate(t *testing.T) {
309 ctx := context.Background()
310 state, runtime, images, secrets := mockDeps(t)
311 rev := mockRevision()
312 rev.Spec.Services[0].Readiness.Timeout = 50 * time.Millisecond
313
314 expectReplacementPreflight(state, runtime, images, secrets, rev)
315 expectNetworkProvision(runtime, "app-blog", 1)
316 state.EXPECT().LoadActive(mock.Anything, "blog").Return(replaceableActive(), true, nil)
317 state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
318
319 var order []string
320 expectRetireOld(state, runtime, &order, "c-old")
321 expectSequentialCreate(state, runtime, &order, "c-new", 18080)
322 runtime.EXPECT().GetContainerLogs(mock.Anything, "c-new", false).Return(nil, assert.AnError).Once()
323 runtime.EXPECT().RemoveContainer(mock.Anything, "c-new", true).RunAndReturn(
324 func(context.Context, string, bool) error {
325 recordEvent(&order, "remove-candidate")
326 return nil
327 }).Once()
328 state.EXPECT().ReleaseBackendBinds(mock.Anything, "blog", "c-new").Return(nil).Once()
329
330 var saved []domain.AppOperation
331 state.EXPECT().SaveOperation(mock.Anything, mock.Anything).RunAndReturn(
332 func(_ context.Context, op domain.AppOperation) error {
333 saved = append(saved, op)
334 return nil
335 })
336
337 svc := deployment.NewService(deployment.Deps{
338 State: state, Runtime: runtime, Images: images, Secrets: secrets,
339 Traffic: &orderedTraffic{events: &order},
340 }, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
341 func(context.Context, string, string) (int, error) { return 503, nil },
342 func(context.Context, string) error { return assert.AnError },
343 ))
344
345 result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
346 require.Error(t, err)
347 require.NotNil(t, result)
348 assert.Equal(t, "failed", result.Services["web"].Result)
349 assert.Contains(t, result.Services["web"].Error, "readiness", "the failure names the readiness probe")
350 assert.NotContains(t, result.Services["web"].Error, "logs:", "raw logs must never be embedded in the public error")
351 assert.Equal(t, "c-new", result.Services["web"].After)
352
353 // ACTIVE was never repointed and the replaced generation's inhibition was
354 // never cleared: nothing claims the old workload is available again.
355 state.AssertNotCalled(t, "SaveActive", mock.Anything, mock.Anything)
356 state.AssertNotCalled(t, "ClearRecoveryInhibition", mock.Anything, mock.Anything, mock.Anything, mock.Anything)
357
358 require.NotEmpty(t, saved)
359 last := saved[len(saved)-1]
360 assert.Equal(t, domain.AppOutcomeFailed, last.Outcome, "a failed replacement must not be journaled as success")
361 step, ok := opStep(last, "service.web.replace")
362 require.True(t, ok)
363 assert.Equal(t, domain.AppStepFailed, step.State)
364
365 assert.Equal(t, "remove-candidate", order[len(order)-1], "the unready candidate is the last thing removed")
366 runtime.AssertNotCalled(t, "StopContainer", mock.Anything, "c-new", mock.Anything)
367 runtime.AssertNotCalled(t, "RemoveContainer", mock.Anything, "c-old", true)
368}
369
370// TestDeploy_CandidateCleanupFailureStaysVisible proves a candidate that
371// cannot be removed does not turn a failed replacement into a success: the
372// leftover is reported as a cleanup warning on the failed service.
373func TestDeploy_CandidateCleanupFailureStaysVisible(t *testing.T) {
374 ctx := context.Background()
375 state, runtime, images, secrets := mockDeps(t)
376 rev := mockRevision()
377 rev.Spec.Services[0].Readiness.Timeout = 50 * time.Millisecond
378
379 expectDeployPreflight(state, runtime, images, secrets, rev)
380 expectNetworkProvision(runtime, "app-blog", 1)
381 state.EXPECT().LoadActive(mock.Anything, "blog").Return(replaceableActive(), true, nil)
382 state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
383
384 expectRetireOld(state, runtime, nil, "c-old")
385 expectSequentialCreate(state, runtime, nil, "c-new", 18080)
386 runtime.EXPECT().GetContainerLogs(mock.Anything, "c-new", false).Return(nil, assert.AnError).Once()
387 runtime.EXPECT().RemoveContainer(mock.Anything, "c-new", true).Return(assert.AnError).Once()
388
389 svc := deployment.NewService(deployment.Deps{
390 State: state, Runtime: runtime, Images: images, Secrets: secrets,
391 }, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
392 func(context.Context, string, string) (int, error) { return 503, nil },
393 func(context.Context, string) error { return assert.AnError },
394 ))
395
396 result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
397 require.Error(t, err)
398 require.NotNil(t, result)
399 svcResult := result.Services["web"]
400 assert.Equal(t, "failed", svcResult.Result)
401 require.Len(t, svcResult.CleanupWarnings, 1, "an unremovable candidate is reported, never hidden")
402 assert.Equal(t, "web", svcResult.CleanupWarnings[0].Service)
403 assert.Equal(t, "c-new", svcResult.CleanupWarnings[0].Leftover)
404 assert.Contains(t, svcResult.CleanupWarnings[0].Detail, "remove")
405 state.AssertNotCalled(t, "ReleaseBackendBinds", mock.Anything, "blog", "c-new")
406 state.AssertNotCalled(t, "SaveActive", mock.Anything, mock.Anything)
407}
408
409// TestDeploy_CancellationDuringReadinessIsNotReportedAsSuccess proves a
410// cancellation during the readiness wait never becomes a success: ACTIVE is
411// not published, no terminal outcome is written, and the candidate cleanup
412// that the canceled context could not complete is reported as a leftover.
413// The durable store and the runtime refuse work on a canceled context, so
414// this test models both.
415func TestDeploy_CancellationDuringReadinessIsNotReportedAsSuccess(t *testing.T) {
416 ctx, cancel := context.WithCancel(context.Background())
417 defer cancel()
418 state, runtime, images, secrets := mockDeps(t)
419 rev := mockRevision()
420 rev.Spec.Services[0].Readiness.Timeout = 30 * time.Second
421
422 expectReplacementPreflight(state, runtime, images, secrets, rev)
423 expectNetworkProvision(runtime, "app-blog", 1)
424 state.EXPECT().LoadActive(mock.Anything, "blog").Return(replaceableActive(), true, nil)
425 state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
426
427 expectRetireOld(state, runtime, nil, "c-old")
428 expectSequentialCreate(state, runtime, nil, "c-new", 18080)
429 // The runtime refuses to remove the candidate with a canceled context.
430 runtime.EXPECT().RemoveContainer(mock.Anything, "c-new", true).Return(context.Canceled).Once()
431
432 var saved []domain.AppOperation
433 state.EXPECT().SaveOperation(mock.Anything, mock.Anything).RunAndReturn(
434 func(opCtx context.Context, op domain.AppOperation) error {
435 if opCtx.Err() != nil {
436 // The durable store refuses writes on a canceled context, so
437 // the operation stays in flight instead of becoming terminal.
438 return opCtx.Err()
439 }
440 saved = append(saved, op)
441 return nil
442 })
443
444 probeStarted := make(chan struct{})
445 svc := deployment.NewService(deployment.Deps{
446 State: state, Runtime: runtime, Images: images, Secrets: secrets,
447 }, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
448 func(probeCtx context.Context, _, _ string) (int, error) {
449 select {
450 case <-probeStarted:
451 default:
452 close(probeStarted)
453 }
454 <-probeCtx.Done()
455 return 0, probeCtx.Err()
456 },
457 func(context.Context, string) error { return nil },
458 ))
459
460 done := make(chan error, 1)
461 var deployResult *deployment.DeployResult
462 go func() {
463 result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
464 deployResult = result
465 done <- err
466 }()
467 select {
468 case <-probeStarted:
469 case <-time.After(5 * time.Second):
470 t.Fatal("timed out waiting for the readiness probe")
471 }
472 cancel()
473 err := <-done
474 require.Error(t, err)
475 assert.Contains(t, err.Error(), context.Canceled.Error(), "the canceled probe is reported")
476
477 // No success is claimed anywhere: ACTIVE is untouched, and no journaled
478 // outcome records the replacement as succeeded. The durable store refuses
479 // the terminal write on a canceled context, so the operation stays in
480 // flight rather than becoming a false success.
481 state.AssertNotCalled(t, "SaveActive", mock.Anything, mock.Anything)
482 require.NotEmpty(t, saved)
483 for _, op := range saved {
484 assert.NotContains(t, []string{domain.AppOutcomeSuccess, domain.AppOutcomePartial}, op.Outcome,
485 "a canceled replacement is never journaled as a success")
486 if step, ok := opStep(op, "service.web.replace"); ok {
487 assert.NotEqual(t, domain.AppStepSucceeded, step.State,
488 "a canceled replacement step is never recorded as succeeded")
489 }
490 }
491
492 // The canceled cleanup is best effort, and what it could not finish is
493 // reported instead of being hidden.
494 require.NotNil(t, deployResult)
495 require.Len(t, deployResult.Services["web"].CleanupWarnings, 1,
496 "a candidate the canceled cleanup could not remove is reported")
497 assert.Equal(t, "c-new", deployResult.Services["web"].CleanupWarnings[0].Leftover)
498 runtime.AssertCalled(t, "RemoveContainer", mock.Anything, "c-new", true)
499 state.AssertNotCalled(t, "ReleaseBackendBinds", mock.Anything, "blog", "c-new")
500}
501
502// TestDeploy_MultiServiceStopsAtFirstFailure proves services are deployed in
503// sorted order and the first failure stops the run: the failing service is
504// cleaned up, later services are untouched, and the services already deployed
505// are kept without any rollback.
506func TestDeploy_MultiServiceStopsAtFirstFailure(t *testing.T) {
507 ctx := context.Background()
508 state, runtime, images, secrets := mockDeps(t)
509
510 httpService := func(name, host, image string, timeout time.Duration) domain.AppService {
511 return domain.AppService{
512 Name: name, Image: image,
513 Readiness: domain.AppReadiness{Type: domain.AppReadinessHTTP, Path: "/healthz", Timeout: timeout},
514 HTTP: []domain.AppHTTPInterface{{Host: host, Port: 8080, TLS: domain.AppTLSAuto}},
515 }
516 }
517 rev := testRevision("blog",
518 httpService("alpha", "alpha.example.com", "docker.io/example/alpha:1.4.2", time.Second),
519 httpService("beta", "beta.example.com", "docker.io/example/beta:1.4.2", 50*time.Millisecond),
520 httpService("gamma", "gamma.example.com", "docker.io/example/gamma:1.4.2", time.Second),
521 )
522
523 state.EXPECT().Recover(mock.Anything).Return(nil)
524 state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
525 images.EXPECT().ResolveDigest(mock.Anything, "docker.io/example/alpha:1.4.2").Return("sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", nil).Once()
526 images.EXPECT().ResolveDigest(mock.Anything, "docker.io/example/beta:1.4.2").Return("sha256:"+strings.Repeat("b", 64), nil).Once()
527 images.EXPECT().ResolveDigest(mock.Anything, "docker.io/example/gamma:1.4.2").Return("sha256:"+strings.Repeat("c", 64), nil).Once()
528 runtime.EXPECT().InspectImageVolumes(mock.Anything, "docker.io/example/alpha:1.4.2").Return(nil, nil).Once()
529 runtime.EXPECT().InspectImageVolumes(mock.Anything, "docker.io/example/beta:1.4.2").Return(nil, nil).Once()
530 runtime.EXPECT().InspectImageVolumes(mock.Anything, "docker.io/example/gamma:1.4.2").Return(nil, nil).Once()
531 state.EXPECT().LoadCheckpoint(mock.Anything).Return(domain.AppStoreCheckpoint{}, nil)
532 state.EXPECT().LoadOwnership(mock.Anything, "blog").Return(domain.AppOwnership{App: "blog", ID: "app-blog"}, nil)
533 state.EXPECT().LoadActive(mock.Anything, "blog").Return(domain.AppActive{
534 App: "blog", Services: map[string]domain.AppEffectiveService{},
535 }, true, nil)
536 state.EXPECT().LoadDesired(mock.Anything, "blog").Return(rev, true, nil)
537 expectNetworkProvision(runtime, "app-blog", 2)
538
539 expectSequentialCreate(state, runtime, nil, "c-alpha", 18080)
540 state.EXPECT().SaveActive(mock.Anything, mock.MatchedBy(func(active domain.AppActive) bool {
541 return active.Services["alpha"].Container == "c-alpha"
542 })).Return(nil).Once()
543 state.EXPECT().SaveOwnership(mock.Anything, mock.Anything).Return(nil)
544
545 runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-beta", mock.Anything).Return(
546 []domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18081, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
547 state.EXPECT().RegisterBackendBinds(mock.Anything, mock.Anything).Return(nil).Once()
548 runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).Return(&domain.Container{ID: "c-beta", Name: "beta"}, nil).Once()
549 runtime.EXPECT().StartContainer(mock.Anything, "c-beta").Return(nil).Once()
550 runtime.EXPECT().GetContainerLogs(mock.Anything, "c-beta", false).Return(nil, assert.AnError).Once()
551 runtime.EXPECT().RemoveContainer(mock.Anything, "c-beta", true).Return(nil).Once()
552 state.EXPECT().ReleaseBackendBinds(mock.Anything, "blog", "c-beta").Return(nil).Once()
553
554 var saved []domain.AppOperation
555 state.EXPECT().SaveOperation(mock.Anything, mock.Anything).RunAndReturn(
556 func(_ context.Context, op domain.AppOperation) error {
557 saved = append(saved, op)
558 return nil
559 })
560
561 traffic := &orderedTraffic{}
562 svc := deployment.NewService(deployment.Deps{
563 State: state, Runtime: runtime, Images: images, Secrets: secrets, Traffic: traffic,
564 }, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
565 func(_ context.Context, url, _ string) (int, error) {
566 if strings.Contains(url, "18081") {
567 return 503, nil
568 }
569 return 200, nil
570 },
571 func(context.Context, string) error { return nil },
572 ))
573
574 result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
575 require.Error(t, err)
576 require.NotNil(t, result)
577 assert.Equal(t, "deployed", result.Services["alpha"].Result, "an already deployed service is kept")
578 assert.Equal(t, "failed", result.Services["beta"].Result)
579 assert.NotContains(t, result.Services, "gamma", "a later service is never attempted after a failure")
580
581 runtime.AssertNumberOfCalls(t, "CreateContainer", 2)
582 runtime.AssertNotCalled(t, "CreateContainer", mock.Anything, mock.MatchedBy(func(cfg *domain.ContainerConfig) bool {
583 return cfg.Name == "gamma"
584 }))
585 // No rollback of the already deployed service.
586 runtime.AssertNotCalled(t, "StopContainer", mock.Anything, mock.Anything, mock.Anything)
587 runtime.AssertNotCalled(t, "RemoveContainer", mock.Anything, "c-alpha", mock.Anything)
588 state.AssertNumberOfCalls(t, "SaveActive", 1)
589
590 require.NotEmpty(t, saved)
591 last := saved[len(saved)-1]
592 assert.Equal(t, domain.AppOutcomePartial, last.Outcome,
593 "the run stops at the first failure; the earlier success is kept, not rolled back")
594}
595
596// TestDeploy_VolumeReplacementPreventsOverlappingWriters proves a stateful
597// replacement durably inhibits the superseded generation before it is
598// retired, stops and removes it before the candidate starts, and keeps the
599// inhibition while the candidate's removal cannot be confirmed. Exclusion
600// holds because the previous process is gone before the replacement can write.
601func TestDeploy_VolumeReplacementPreventsOverlappingWriters(t *testing.T) {
602 ctx := context.Background()
603 store := newTestStore(t)
604 runtime := outmocks.NewMockContainerRuntime(t)
605 images := outmocks.NewMockImageResolver(t)
606 secrets := outmocks.NewMockSecretProvider(t)
607
608 image := "registry.example.com/blog/web:1.4.2"
609 spec := domain.AppService{
610 Name: "web",
611 Image: image,
612 Secrets: map[string]string{},
613 Volumes: []domain.AppVolume{{Name: "data", Path: "/data"}},
614 HTTP: []domain.AppHTTPInterface{{Host: "blog.example.com", Port: 8080, TLS: domain.AppTLSAuto}},
615 Readiness: domain.AppReadiness{
616 Type: domain.AppReadinessHTTP, Path: "/healthz", Timeout: 50 * time.Millisecond,
617 },
618 }
619 rev := testRevision("blog", spec)
620
621 require.NoError(t, store.SaveIntent(ctx, domain.AppStopIntent{App: "blog"}))
622 require.NoError(t, store.SaveActive(ctx, domain.AppActive{
623 App: "blog",
624 Services: map[string]domain.AppEffectiveService{
625 "web": {
626 Container: "c-old", EffectiveRevision: "rev-0", Spec: spec,
627 BackendBinds: map[int]int{8080: 18080},
628 },
629 },
630 }))
631 seedDesired(t, ctx, store, rev)
632 // The app keeps one incarnation: the private network every service joins
633 // is derived from it.
634 require.NoError(t, store.SaveOwnership(ctx, domain.AppOwnership{App: "blog", ID: "app-blog"}))
635 expectNetworkProvision(runtime, "app-blog", 1)
636
637 images.EXPECT().ResolveDigest(mock.Anything, image).Return("sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", nil).Once()
638 runtime.EXPECT().InspectImageVolumes(mock.Anything, image).Return(nil, nil).Once()
639 runtime.EXPECT().VolumeExists(mock.Anything, mock.Anything).Return(false, nil).Once()
640
641 var order []string
642 inhibitedBeforeStop := false
643 runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).RunAndReturn(
644 func(context.Context, string, time.Duration) error {
645 recordEvent(&order, "stop-old")
646 // The durable marker must already exist before the superseded
647 // writer is stopped: recovery must never race the destructive step.
648 inhibitions, loadErr := store.LoadRecoveryInhibitions(context.Background(), "blog")
649 require.NoError(t, loadErr)
650 inhibitedBeforeStop = len(inhibitions) == 1 &&
651 inhibitions[0].ContainerID == "c-old" &&
652 inhibitions[0].Reason == domain.AppInhibitReplacementPending
653 return nil
654 }).Once()
655 runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).RunAndReturn(
656 func(context.Context, string, bool) error {
657 recordEvent(&order, "remove-old")
658 return nil
659 }).Once()
660 runtime.EXPECT().CreateVolume(mock.Anything, mock.Anything, mock.Anything).RunAndReturn(
661 func(context.Context, string, map[string]string) error {
662 recordEvent(&order, "create-volume")
663 return nil
664 }).Once()
665 runtime.EXPECT().CreateContainer(mock.Anything, mock.Anything).RunAndReturn(
666 func(context.Context, *domain.ContainerConfig) (*domain.Container, error) {
667 recordEvent(&order, "create-new")
668 return &domain.Container{ID: "c-new", Name: "web"}, nil
669 }).Once()
670 runtime.EXPECT().StartContainer(mock.Anything, "c-new").RunAndReturn(
671 func(context.Context, string) error {
672 recordEvent(&order, "start-new")
673 return nil
674 }).Once()
675 runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-new", mock.Anything).Return(
676 []domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 18081, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
677 runtime.EXPECT().GetContainerLogs(mock.Anything, "c-new", false).Return(nil, assert.AnError).Once()
678 // The failed replacement removes its candidate once. Boot reconciliation
679 // retries it because a terminal failed operation cannot know the earlier
680 // removal succeeded without re-checking the runtime; that retry fails, so
681 // recovery fails closed and never revives the superseded writer whose
682 // volume the orphan may still touch.
683 runtime.EXPECT().RemoveContainer(mock.Anything, "c-new", true).Return(nil).Once()
684 runtime.EXPECT().RemoveContainer(mock.Anything, "c-new", true).Return(assert.AnError).Once()
685
686 svc := deployment.NewService(deployment.Deps{
687 State: store, Runtime: runtime, Images: images, Secrets: secrets,
688 ImagePolicy: domain.ImageSourcePolicy{AllowedRegistries: []string{"registry.example.com"}},
689 }, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
690 func(context.Context, string, string) (int, error) { return 503, nil },
691 func(context.Context, string) error { return assert.AnError },
692 ))
693
694 _, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
695 require.Error(t, err)
696
697 assert.Equal(t, []string{"stop-old", "remove-old", "create-volume", "create-new", "start-new"}, order,
698 "the superseded writer is gone before the replacement can write")
699 assert.True(t, inhibitedBeforeStop,
700 "the durable recovery inhibition exists before the superseded writer is stopped")
701
702 // The failed replacement never cleared the durable inhibition: a later
703 // boot must not revive the old writer on top of data the candidate may
704 // already have touched.
705 inhibitions, loadErr := store.LoadRecoveryInhibitions(ctx, "blog")
706 require.NoError(t, loadErr)
707 require.Len(t, inhibitions, 1)
708 assert.Equal(t, "c-old", inhibitions[0].ContainerID)
709 assert.Equal(t, domain.AppInhibitReplacementPending, inhibitions[0].Reason)
710
711 // ACTIVE still names the superseded generation: the failure is explicit
712 // and no availability was restored on its behalf.
713 active, ok, loadErr := store.LoadActive(ctx, "blog")
714 require.NoError(t, loadErr)
715 require.True(t, ok)
716 assert.Equal(t, "c-old", active.Services["web"].Container)
717
718 // The failed replacement keeps its candidate in the journal: a leftover
719 // must stay traceable so reconciliation can converge it.
720 failedOp, hasOp, loadErr := store.LoadLatestOperation(ctx, "blog")
721 require.NoError(t, loadErr)
722 require.True(t, hasOp)
723 failedStep, found := opStep(failedOp, "service.web.replace")
724 require.True(t, found)
725 assert.Equal(t, domain.AppStepFailed, failedStep.State)
726 assert.Equal(t, "c-new", failedStep.After, "the created candidate stays recorded")
727
728 rebooted := deployment.NewService(deployment.Deps{
729 State: store, Runtime: runtime, Images: images, Secrets: secrets,
730 }, zerowrap.Default())
731 bootErr := rebooted.ReconcileBoot(ctx)
732 require.Error(t, bootErr)
733 assert.Contains(t, bootErr.Error(), "could not be removed")
734 runtime.AssertNotCalled(t, "StartContainer", mock.Anything, "c-old")
735 runtime.AssertNotCalled(t, "RestartContainer", mock.Anything, "c-old", mock.Anything)
736
737 // The orphan that could not be removed keeps the inhibition: the
738 // superseded writer is never revived on top of data the candidate may
739 // already have touched.
740 keptInhibitions, keptErr := store.LoadRecoveryInhibitions(ctx, "blog")
741 require.NoError(t, keptErr)
742 require.Len(t, keptInhibitions, 1)
743 assert.Equal(t, "c-old", keptInhibitions[0].ContainerID)
744 assert.Equal(t, domain.AppInhibitReplacementPending, keptInhibitions[0].Reason)
745}