1package deployment_test
2
3import (
4 "context"
5 "testing"
6 "time"
7
8 "github.com/bnema/zerowrap"
9 "github.com/stretchr/testify/assert"
10 "github.com/stretchr/testify/mock"
11 "github.com/stretchr/testify/require"
12
13 "github.com/bnema/gordon/internal/adapters/out/appstate"
14 outmocks "github.com/bnema/gordon/internal/boundaries/out/mocks"
15 "github.com/bnema/gordon/internal/domain"
16 "github.com/bnema/gordon/internal/usecase/deployment"
17)
18
19// newTestStore opens a real app-state store in a temporary directory.
20func newTestStore(t *testing.T) *appstate.Store {
21 t.Helper()
22 store, err := appstate.NewStore(t.TempDir(), zerowrap.Default())
23 require.NoError(t, err)
24 t.Cleanup(func() { require.NoError(t, store.Close()) })
25 return store
26}
27
28// TestRecoveryInhibition_RefusesInhibitedGenerationAfterReboot proves the
29// durable inhibition contract end to end: an old volume-owning container
30// that a replacement may already have superseded is never started again,
31// even by a completely reconstructed engine reading a reopened store.
32// Unrelated generations of the same app stay recoverable.
33func TestRecoveryInhibition_RefusesInhibitedGenerationAfterReboot(t *testing.T) {
34 ctx := context.Background()
35 dir := t.TempDir()
36
37 store, err := appstate.NewStore(dir, zerowrap.Default())
38 require.NoError(t, err)
39 require.NoError(t, store.SaveIntent(ctx, domain.AppStopIntent{App: "blog"}))
40 require.NoError(t, store.SaveActive(ctx, domain.AppActive{
41 App: "blog",
42 Services: map[string]domain.AppEffectiveService{
43 "web": {
44 Container: "c-old", EffectiveRevision: "rev-1", Spec: headlessSpec(),
45 BackendBinds: map[int]int{8080: 32770},
46 },
47 "worker": {
48 Container: "c-worker", EffectiveRevision: "rev-1", Spec: headlessSpec(),
49 },
50 },
51 }))
52 require.NoError(t, store.SaveRecoveryInhibition(ctx, domain.AppRecoveryInhibition{
53 App: "blog", Service: "web", ContainerID: "c-old",
54 Reason: domain.AppInhibitReplacementPending, Operation: "op-1",
55 }))
56 require.NoError(t, store.Close())
57
58 // Reboot: a fresh store handle and a fresh engine, no in-memory state.
59 reopened, err := appstate.NewStore(dir, zerowrap.Default())
60 require.NoError(t, err)
61 t.Cleanup(func() { require.NoError(t, reopened.Close()) })
62 inhibitions, err := reopened.LoadRecoveryInhibitions(ctx, "blog")
63 require.NoError(t, err)
64 require.Len(t, inhibitions, 1)
65 assert.Equal(t, "c-old", inhibitions[0].ContainerID)
66
67 runtime := outmocks.NewMockContainerRuntime(t)
68 runtime.EXPECT().InspectContainer(mock.Anything, "c-worker").
69 Return(&domain.Container{ID: "c-worker", Status: "running", StartedAt: time.Now().UTC()}, nil).Twice()
70 runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "c-worker").Return("", false, nil).Once()
71
72 svc := deployment.NewService(deployment.Deps{
73 State: reopened, Runtime: runtime, Traffic: &recordingTraffic{},
74 }, zerowrap.Default())
75
76 err = svc.ReconcileRunning(ctx)
77 require.Error(t, err)
78 assert.Contains(t, err.Error(), "recovery inhibited")
79 runtime.AssertNotCalled(t, "StartContainer", mock.Anything, "c-old")
80 runtime.AssertNotCalled(t, "RestartContainer", mock.Anything, "c-old", mock.Anything)
81 runtime.AssertCalled(t, "InspectContainer", mock.Anything, "c-worker")
82 runtime.AssertNotCalled(t, "RemoveContainer", mock.Anything, mock.Anything, mock.Anything)
83 runtime.AssertNotCalled(t, "RemoveVolume", mock.Anything, mock.Anything, mock.Anything)
84}
85
86// seedDesired writes one accepted desired revision through the real
87// staged/committed/materialized apply protocol.
88func seedDesired(t *testing.T, ctx context.Context, store *appstate.Store, rev domain.AppDesiredRevision) {
89 t.Helper()
90 intentID := "intent-1"
91 require.NoError(t, store.StageApply(ctx, domain.AppApplyIntent{
92 Intent: intentID, App: rev.App, Revision: rev.Revision,
93 SourceSHA256: "0f1e2d", Spec: rev.Spec, CreatedAt: time.Now().UTC(),
94 }))
95 require.NoError(t, store.CommitApply(ctx, rev.App, intentID))
96 require.NoError(t, store.MaterializeApply(ctx, rev.App, intentID))
97}
98
99// TestVolumeReplacementFailureLeavesOldGenerationInhibited proves the
100// marker is written BEFORE a volume-owning replacement can write and is
101// not cleared when that replacement fails: the old generation must never
102// be restarted on top of possibly-newer data.
103func TestVolumeReplacementFailureLeavesOldGenerationInhibited(t *testing.T) {
104 ctx := context.Background()
105 store := newTestStore(t)
106 runtime := outmocks.NewMockContainerRuntime(t)
107 images := outmocks.NewMockImageResolver(t)
108 secrets := outmocks.NewMockSecretProvider(t)
109
110 spec := domain.AppService{
111 Name: "web",
112 Image: "registry.example.com/blog/web:1.4.2",
113 Volumes: []domain.AppVolume{{Name: "data", Path: "/data"}},
114 Secrets: map[string]string{},
115 Readiness: domain.AppReadiness{
116 Type: domain.AppReadinessHTTP, Path: "/healthz", Timeout: 50 * time.Millisecond,
117 },
118 }
119 rev := testRevision("blog", spec)
120 rev.Spec.Env = map[string]string{}
121
122 require.NoError(t, store.SaveIntent(ctx, domain.AppStopIntent{App: "blog"}))
123 require.NoError(t, store.SaveActive(ctx, domain.AppActive{
124 App: "blog",
125 Services: map[string]domain.AppEffectiveService{
126 "web": {
127 Container: "c-old", EffectiveRevision: "rev-0", Spec: headlessSpec(),
128 BackendBinds: map[int]int{9000: 19000},
129 },
130 },
131 }))
132 seedDesired(t, ctx, store, rev)
133
134 images.EXPECT().ResolveDigest(mock.Anything, spec.Image).Return("sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", nil).Once()
135 runtime.EXPECT().InspectImageVolumes(mock.Anything, spec.Image).Return(nil, nil).Once()
136 runtime.EXPECT().VolumeExists(mock.Anything, mock.Anything).Return(false, nil).Once()
137 // The superseded writer cannot be removed: the replacement must abort
138 // rather than risk overlapping writers, and the inhibition written
139 // before the stop attempt stays in place.
140 runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(nil).Once()
141 runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(assert.AnError).Once()
142
143 svc := deployment.NewService(deployment.Deps{
144 State: store, Runtime: runtime, Images: images, Secrets: secrets,
145 ImagePolicy: domain.ImageSourcePolicy{AllowedRegistries: []string{"registry.example.com"}},
146 }, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
147 func(context.Context, string, string) (int, error) { return 500, nil },
148 func(context.Context, string) error { return assert.AnError },
149 ))
150
151 _, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
152 require.Error(t, err)
153
154 inhibitions, err := store.LoadRecoveryInhibitions(ctx, "blog")
155 require.NoError(t, err)
156 require.Len(t, inhibitions, 1)
157 assert.Equal(t, "c-old", inhibitions[0].ContainerID)
158 assert.Equal(t, domain.AppInhibitReplacementPending, inhibitions[0].Reason)
159
160 // A later boot must refuse to revive the inhibited generation, and a
161 // rebuilt engine must reach the same conclusion.
162 runtime2 := outmocks.NewMockContainerRuntime(t)
163 rebooted := deployment.NewService(deployment.Deps{
164 State: store, Runtime: runtime2, Images: images, Secrets: secrets,
165 }, zerowrap.Default())
166 bootErr := rebooted.ReconcileBoot(ctx)
167 require.Error(t, bootErr)
168 assert.Contains(t, bootErr.Error(), "recovery inhibited")
169 runtime2.AssertNotCalled(t, "StartContainer", mock.Anything, "c-old")
170 runtime2.AssertNotCalled(t, "RestartContainer", mock.Anything, "c-old", mock.Anything)
171 runtime2.AssertNotCalled(t, "RemoveVolume", mock.Anything, mock.Anything, mock.Anything)
172}
173
174// TestRemove_InhibitsBeforeRuntimeWithdrawal proves remove is durably
175// fail-closed: stopped intent and an inhibition per removed container are
176// persisted before any runtime effect, and volumes are never deleted.
177func TestRemove_InhibitsBeforeRuntimeWithdrawal(t *testing.T) {
178 ctx := context.Background()
179 state, runtime, images, secrets := mockDeps(t)
180
181 active := domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
182 "web": {Container: "c-1", EffectiveRevision: "rev-1", Spec: headlessSpec()},
183 }}
184 state.EXPECT().Recover(mock.Anything).Return(nil).Once()
185 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil).Once()
186
187 var order []string
188 state.EXPECT().SaveIntent(mock.Anything, mock.MatchedBy(func(intent domain.AppStopIntent) bool {
189 return intent.Stopped
190 })).RunAndReturn(func(context.Context, domain.AppStopIntent) error {
191 order = append(order, "intent-stopped")
192 return nil
193 }).Once()
194 state.EXPECT().SaveRecoveryInhibition(mock.Anything, mock.MatchedBy(func(inhibition domain.AppRecoveryInhibition) bool {
195 return inhibition.App == "blog" && inhibition.Service == "web" && inhibition.ContainerID == "c-1"
196 })).RunAndReturn(func(context.Context, domain.AppRecoveryInhibition) error {
197 order = append(order, "inhibited")
198 return nil
199 }).Once()
200 runtime.EXPECT().StopContainer(mock.Anything, "c-1", mock.Anything).RunAndReturn(func(context.Context, string, time.Duration) error {
201 order = append(order, "stopped")
202 return nil
203 }).Once()
204 runtime.EXPECT().RemoveContainer(mock.Anything, "c-1", false).RunAndReturn(func(context.Context, string, bool) error {
205 order = append(order, "removed")
206 return nil
207 }).Once()
208 // Confirmed disappearance releases the container's backend claims and
209 // clears its recovery inhibition before the incarnation is retired.
210 state.EXPECT().ReleaseBackendBinds(mock.Anything, "blog", "c-1").Return(nil).Once()
211 state.EXPECT().ClearRecoveryInhibition(mock.Anything, "blog", "web", "c-1").Return(nil).Once()
212 // Ownership is read while it is still live; the name-only record owns
213 // no network, so nothing is reclaimed.
214 state.EXPECT().LoadOwnership(mock.Anything, "blog").Return(domain.AppOwnership{App: "blog"}, nil).Once()
215 // The incarnation is retired atomically after the workload is gone.
216 state.EXPECT().RetireApp(mock.Anything, "blog").RunAndReturn(func(context.Context, string) error {
217 order = append(order, "retired")
218 return nil
219 }).Once()
220 state.EXPECT().SaveOperation(mock.Anything, mock.Anything).Return(nil)
221
222 svc := deployment.NewService(deployment.Deps{
223 State: state, Runtime: runtime, Images: images, Secrets: secrets,
224 }, zerowrap.Default())
225 result, err := svc.Remove(ctx, "blog", "")
226 require.NoError(t, err)
227 require.NotNil(t, result)
228 assert.Equal(t, domain.AppServiceRemoved, result.Services["web"].Result)
229 assert.Equal(t, []string{"intent-stopped", "inhibited", "stopped", "removed", "retired"}, order)
230 runtime.AssertNotCalled(t, "RemoveVolume", mock.Anything, mock.Anything, mock.Anything)
231}
232
233// TestBootStoppedConvergenceStopsRevivedContainer proves durable stopped
234// intent wins over native restart policy: a container revived while the
235// daemon was away is stopped again, without removal, and is never started.
236func TestBootStoppedConvergenceStopsRevivedContainer(t *testing.T) {
237 ctx := context.Background()
238 state, runtime, images, secrets := mockDeps(t)
239
240 active := domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
241 "web": {Container: "c-1", EffectiveRevision: "rev-1", Spec: headlessSpec()},
242 }}
243 state.EXPECT().Recover(mock.Anything).Return(nil).Once()
244 state.EXPECT().ListApps(mock.Anything).Return([]string{"blog"}, nil).Once()
245 state.EXPECT().LoadIntent(mock.Anything, "blog").
246 Return(domain.AppStopIntent{App: "blog", Stopped: true}, nil).Once()
247 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil).Once()
248 runtime.EXPECT().InspectContainer(mock.Anything, "c-1").
249 Return(&domain.Container{ID: "c-1", Status: "running"}, nil).Once()
250 runtime.EXPECT().StopContainer(mock.Anything, "c-1", mock.Anything).Return(nil).Once()
251
252 svc := deployment.NewService(deployment.Deps{
253 State: state, Runtime: runtime, Images: images, Secrets: secrets,
254 }, zerowrap.Default())
255 require.NoError(t, svc.ReconcileBoot(ctx))
256 runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
257 runtime.AssertNotCalled(t, "RestartContainer", mock.Anything, mock.Anything, mock.Anything)
258 runtime.AssertNotCalled(t, "RemoveContainer", mock.Anything, mock.Anything, mock.Anything)
259}
260
261// TestBootRefusesInhibitedGenerationBeforeAnyRuntimeEffect proves boot
262// recovery checks the durable marker before touching the workload.
263func TestBootRefusesInhibitedGenerationBeforeAnyRuntimeEffect(t *testing.T) {
264 ctx := context.Background()
265 state, runtime, images, secrets := mockDeps(t)
266
267 active := domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
268 "web": {Container: "c-old", EffectiveRevision: "rev-1", Spec: headlessSpec()},
269 }}
270 state.EXPECT().Recover(mock.Anything).Return(nil)
271 state.EXPECT().ListApps(mock.Anything).Return([]string{"blog"}, nil).Once()
272 state.EXPECT().LoadIntent(mock.Anything, "blog").Return(domain.AppStopIntent{App: "blog"}, nil).Once()
273 state.EXPECT().SaveOperation(mock.Anything, mock.Anything).Return(nil)
274 state.EXPECT().SaveIntent(mock.Anything, mock.Anything).Return(nil)
275 state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").
276 Return([]domain.AppRecoveryInhibition{{App: "blog", Service: "web", ContainerID: "c-old"}}, nil).Once()
277 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil)
278
279 svc := deployment.NewService(deployment.Deps{
280 State: state, Runtime: runtime, Images: images, Secrets: secrets,
281 }, zerowrap.Default())
282 err := svc.ReconcileBoot(ctx)
283 require.Error(t, err)
284 assert.Contains(t, err.Error(), "recovery inhibited")
285 runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
286 runtime.AssertNotCalled(t, "RestartContainer", mock.Anything, mock.Anything, mock.Anything)
287 runtime.AssertNotCalled(t, "CreateContainer", mock.Anything, mock.Anything)
288}
289
290// TestWorkloadMutationsSerializePerApp proves a periodic pass cannot
291// interleave with an in-flight user mutation of the same app: the pass
292// skips the busy app instead of waiting for it.
293func TestWorkloadMutationsSerializePerApp(t *testing.T) {
294 ctx := context.Background()
295 state, runtime, images, secrets := mockDeps(t)
296
297 entered := make(chan struct{})
298 release := make(chan struct{})
299 state.EXPECT().Recover(mock.Anything).RunAndReturn(func(context.Context) error {
300 close(entered)
301 <-release
302 return assert.AnError
303 }).Once()
304 // Only the app list may be read: the pass must not touch intent or
305 // ACTIVE while the mutation holds the app lock.
306 state.EXPECT().ListApps(mock.Anything).Return([]string{"blog"}, nil).Once()
307
308 svc := deployment.NewService(deployment.Deps{
309 State: state, Runtime: runtime, Images: images, Secrets: secrets,
310 }, zerowrap.Default())
311
312 done := make(chan error, 1)
313 go func() {
314 _, err := svc.Stop(ctx, "blog", "")
315 done <- err
316 }()
317 <-entered
318
319 // The mutation holds the app lock. The periodic pass must skip the
320 // app instead of waiting: LoadIntent has no expectation, so reaching
321 // it would fail the test.
322 require.NoError(t, svc.ReconcileRunning(ctx))
323 state.AssertNotCalled(t, "LoadIntent", mock.Anything, mock.Anything)
324
325 close(release)
326 require.Error(t, <-done)
327}
328
329// TestRestart_RefusesInhibitedGeneration proves an explicit operator
330// restart cannot revive a generation whose recovery is durably inhibited:
331// a replacement may already have written to the volume it still owns.
332func TestRestart_RefusesInhibitedGeneration(t *testing.T) {
333 ctx := context.Background()
334 state, runtime, images, secrets := mockDeps(t)
335
336 active := domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
337 "web": {Container: "c-old", EffectiveRevision: "rev-1", Spec: headlessSpec()},
338 }}
339 state.EXPECT().Recover(mock.Anything).Return(nil).Once()
340 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil).Once()
341 state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return([]domain.AppRecoveryInhibition{{
342 App: "blog", Service: "web", ContainerID: "c-old", Reason: domain.AppInhibitReplacementPending,
343 }}, nil).Once()
344 state.EXPECT().SaveOperation(mock.Anything, mock.Anything).Return(nil)
345
346 svc := deployment.NewService(deployment.Deps{
347 State: state, Runtime: runtime, Images: images, Secrets: secrets,
348 }, zerowrap.Default())
349 result, err := svc.Restart(ctx, "blog", "", "")
350 require.ErrorIs(t, err, domain.ErrAppStateConflict)
351 require.NotNil(t, result)
352 assert.Equal(t, "failed", result.Services["web"].Result)
353 assert.Contains(t, result.Services["web"].Error, "recovery inhibited")
354 runtime.AssertNotCalled(t, "RestartContainer", mock.Anything, mock.Anything, mock.Anything)
355}
356
357// TestRemove_KeepsStateWhenAContainerCannotBeRemoved proves remove never
358// loses track of a survivor: ACTIVE, stopped intent, and the inhibition
359// stay in place so the periodic pass keeps converging it.
360func TestRemove_KeepsStateWhenAContainerCannotBeRemoved(t *testing.T) {
361 ctx := context.Background()
362 state, runtime, images, secrets := mockDeps(t)
363
364 active := domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
365 "web": {Container: "c-1", EffectiveRevision: "rev-1", Spec: headlessSpec()},
366 }}
367 state.EXPECT().Recover(mock.Anything).Return(nil).Once()
368 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil).Once()
369 state.EXPECT().SaveIntent(mock.Anything, mock.MatchedBy(func(intent domain.AppStopIntent) bool {
370 return intent.Stopped
371 })).Return(nil).Once()
372 state.EXPECT().SaveRecoveryInhibition(mock.Anything, mock.Anything).Return(nil).Once()
373 // The claim journal is written before the runtime effects and is
374 // rewritten terminally when the removal cannot complete: a repeat of
375 // the same key replays the failure instead of reading a stuck claim.
376 state.EXPECT().SaveOperation(mock.Anything, mock.MatchedBy(func(op domain.AppOperation) bool {
377 return op.Op != "" && op.Kind == "remove" && !op.Terminal()
378 })).Return(nil).Once()
379 state.EXPECT().SaveOperation(mock.Anything, mock.MatchedBy(func(op domain.AppOperation) bool {
380 return op.Op != "" && op.Outcome == domain.AppOutcomeFailed && op.Terminal()
381 })).Return(nil).Once()
382 runtime.EXPECT().StopContainer(mock.Anything, "c-1", mock.Anything).Return(assert.AnError).Once()
383
384 svc := deployment.NewService(deployment.Deps{
385 State: state, Runtime: runtime, Images: images, Secrets: secrets,
386 }, zerowrap.Default())
387 _, err := svc.Remove(ctx, "blog", "")
388 require.Error(t, err)
389 assert.Contains(t, err.Error(), "not confirmed gone")
390 // ACTIVE and the stopped intent were never discarded: the monitor can
391 // still converge the survivor. The survivor's claims and inhibition
392 // stay in place too: only a confirmed disappearance releases them.
393 state.AssertNotCalled(t, "SaveActive", mock.Anything, mock.Anything)
394 state.AssertNotCalled(t, "RetireApp", mock.Anything, mock.Anything)
395 state.AssertNotCalled(t, "ReleaseBackendBinds", mock.Anything, mock.Anything, mock.Anything)
396 state.AssertNotCalled(t, "ClearRecoveryInhibition", mock.Anything, mock.Anything, mock.Anything, mock.Anything)
397 runtime.AssertNotCalled(t, "RemoveVolume", mock.Anything, mock.Anything, mock.Anything)
398}
399
400// TestRemovalOfMissingContainerIsIdempotent proves a container the
401// runtime already dropped is a successful removal, not a failure.
402func TestRemovalOfMissingContainerIsIdempotent(t *testing.T) {
403 ctx := context.Background()
404 state, runtime, images, secrets := mockDeps(t)
405
406 active := domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
407 "web": {Container: "c-1", EffectiveRevision: "rev-1", Spec: headlessSpec()},
408 }}
409 state.EXPECT().Recover(mock.Anything).Return(nil).Once()
410 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil).Once()
411 state.EXPECT().SaveOperation(mock.Anything, mock.Anything).Return(nil)
412 state.EXPECT().SaveIntent(mock.Anything, mock.Anything).Return(nil)
413 state.EXPECT().SaveRecoveryInhibition(mock.Anything, mock.Anything).Return(nil).Once()
414 runtime.EXPECT().StopContainer(mock.Anything, "c-1", mock.Anything).Return(domain.ErrContainerNotFound).Once()
415 runtime.EXPECT().RemoveContainer(mock.Anything, "c-1", false).Return(domain.ErrContainerNotFound).Once()
416 // Not-found is confirmation: the stale claims and the inhibition of
417 // the disappeared container are released.
418 state.EXPECT().ReleaseBackendBinds(mock.Anything, "blog", "c-1").Return(nil).Once()
419 state.EXPECT().ClearRecoveryInhibition(mock.Anything, "blog", "web", "c-1").Return(nil).Once()
420 state.EXPECT().LoadOwnership(mock.Anything, "blog").Return(domain.AppOwnership{App: "blog"}, nil).Once()
421 state.EXPECT().RetireApp(mock.Anything, "blog").Return(nil).Once()
422
423 svc := deployment.NewService(deployment.Deps{
424 State: state, Runtime: runtime, Images: images, Secrets: secrets,
425 }, zerowrap.Default())
426 result, err := svc.Remove(ctx, "blog", "")
427 require.NoError(t, err)
428 require.NotNil(t, result)
429}