1package deployment_test
2
3import (
4 "context"
5 "testing"
6 "time"
7
8 "github.com/bnema/zerowrap"
9 "github.com/stretchr/testify/assert"
10 "github.com/stretchr/testify/mock"
11 "github.com/stretchr/testify/require"
12
13 "github.com/bnema/gordon/internal/adapters/out/appstate"
14 outmocks "github.com/bnema/gordon/internal/boundaries/out/mocks"
15 "github.com/bnema/gordon/internal/domain"
16 "github.com/bnema/gordon/internal/usecase/deployment"
17)
18
19// newTestStore opens a real app-state store in a temporary directory.
20func newTestStore(t *testing.T) *appstate.Store {
21 t.Helper()
22 store, err := appstate.NewStore(t.TempDir(), zerowrap.Default())
23 require.NoError(t, err)
24 t.Cleanup(func() { require.NoError(t, store.Close()) })
25 return store
26}
27
28// TestRecoveryInhibition_RefusesInhibitedGenerationAfterReboot proves the
29// durable inhibition contract end to end: an old volume-owning container
30// that a replacement may already have superseded is never started again,
31// even by a completely reconstructed engine reading a reopened store.
32// Unrelated generations of the same app stay recoverable.
33func TestRecoveryInhibition_RefusesInhibitedGenerationAfterReboot(t *testing.T) {
34 ctx := context.Background()
35 dir := t.TempDir()
36
37 store, err := appstate.NewStore(dir, zerowrap.Default())
38 require.NoError(t, err)
39 require.NoError(t, store.SaveIntent(ctx, domain.AppStopIntent{App: "blog"}))
40 require.NoError(t, store.SaveActive(ctx, domain.AppActive{
41 App: "blog",
42 Services: map[string]domain.AppEffectiveService{
43 "web": {
44 Container: "c-old", EffectiveRevision: "rev-1", Spec: headlessSpec(),
45 BackendBinds: map[int]int{8080: 32770},
46 },
47 "worker": {
48 Container: "c-worker", EffectiveRevision: "rev-1", Spec: headlessSpec(),
49 },
50 },
51 }))
52 require.NoError(t, store.SaveRecoveryInhibition(ctx, domain.AppRecoveryInhibition{
53 App: "blog", Service: "web", ContainerID: "c-old",
54 Reason: domain.AppInhibitReplacementPending, Operation: "op-1",
55 }))
56 require.NoError(t, store.Close())
57
58 // Reboot: a fresh store handle and a fresh engine, no in-memory state.
59 reopened, err := appstate.NewStore(dir, zerowrap.Default())
60 require.NoError(t, err)
61 t.Cleanup(func() { require.NoError(t, reopened.Close()) })
62 inhibitions, err := reopened.LoadRecoveryInhibitions(ctx, "blog")
63 require.NoError(t, err)
64 require.Len(t, inhibitions, 1)
65 assert.Equal(t, "c-old", inhibitions[0].ContainerID)
66
67 runtime := outmocks.NewMockContainerRuntime(t)
68 runtime.EXPECT().InspectContainer(mock.Anything, "c-worker").
69 Return(&domain.Container{ID: "c-worker", Status: "running", StartedAt: time.Now().UTC()}, nil).Twice()
70 runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "c-worker").Return("", false, nil).Once()
71
72 svc := deployment.NewService(deployment.Deps{
73 State: reopened, Runtime: runtime, Traffic: &recordingTraffic{},
74 }, zerowrap.Default())
75
76 err = svc.ReconcileRunning(ctx)
77 require.Error(t, err)
78 assert.Contains(t, err.Error(), "recovery inhibited")
79 runtime.AssertNotCalled(t, "StartContainer", mock.Anything, "c-old")
80 runtime.AssertNotCalled(t, "RestartContainer", mock.Anything, "c-old", mock.Anything)
81 runtime.AssertCalled(t, "InspectContainer", mock.Anything, "c-worker")
82 runtime.AssertNotCalled(t, "RemoveContainer", mock.Anything, mock.Anything, mock.Anything)
83 runtime.AssertNotCalled(t, "RemoveVolume", mock.Anything, mock.Anything, mock.Anything)
84}
85
86// seedDesired writes one accepted desired revision through the real
87// staged/committed/materialized apply protocol.
88func seedDesired(t *testing.T, ctx context.Context, store *appstate.Store, rev domain.AppDesiredRevision) {
89 t.Helper()
90 intentID := "intent-1"
91 require.NoError(t, store.StageApply(ctx, domain.AppApplyIntent{
92 Intent: intentID, App: rev.App, Revision: rev.Revision,
93 SourceSHA256: "0f1e2d", Spec: rev.Spec, CreatedAt: time.Now().UTC(),
94 }))
95 require.NoError(t, store.CommitApply(ctx, rev.App, intentID))
96 require.NoError(t, store.MaterializeApply(ctx, rev.App, intentID))
97}
98
99// TestVolumeReplacementFailureLeavesOldGenerationInhibited proves the
100// marker is written BEFORE a volume-owning replacement can write and is
101// not cleared when that replacement fails: the old generation must never
102// be restarted on top of possibly-newer data.
103func TestVolumeReplacementFailureLeavesOldGenerationInhibited(t *testing.T) {
104 ctx := context.Background()
105 store := newTestStore(t)
106 runtime := outmocks.NewMockContainerRuntime(t)
107 images := outmocks.NewMockImageResolver(t)
108 secrets := outmocks.NewMockSecretProvider(t)
109
110 spec := domain.AppService{
111 Name: "web",
112 Image: "registry.example.com/blog/web:1.4.2",
113 Volumes: []domain.AppVolume{{Name: "data", Path: "/data"}},
114 Secrets: map[string]string{},
115 Readiness: domain.AppReadiness{
116 Type: domain.AppReadinessHTTP, Path: "/healthz", Timeout: 50 * time.Millisecond,
117 },
118 }
119 rev := testRevision("blog", spec)
120 rev.Spec.Env = map[string]string{}
121
122 require.NoError(t, store.SaveIntent(ctx, domain.AppStopIntent{App: "blog"}))
123 require.NoError(t, store.SaveActive(ctx, domain.AppActive{
124 App: "blog",
125 Services: map[string]domain.AppEffectiveService{
126 "web": {
127 Container: "c-old", EffectiveRevision: "rev-0", Spec: headlessSpec(),
128 BackendBinds: map[int]int{9000: 19000},
129 },
130 },
131 }))
132 seedDesired(t, ctx, store, rev)
133
134 images.EXPECT().ResolveDigest(mock.Anything, spec.Image).Return("sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", nil).Once()
135 runtime.EXPECT().InspectImageVolumes(mock.Anything, spec.Image).Return(nil, nil).Once()
136 runtime.EXPECT().VolumeExists(mock.Anything, mock.Anything).Return(false, nil).Once()
137 // The superseded writer cannot be removed: the replacement must abort
138 // rather than risk overlapping writers, and the inhibition written
139 // before the stop attempt stays in place.
140 runtime.EXPECT().StopContainer(mock.Anything, "c-old", mock.Anything).Return(nil).Once()
141 runtime.EXPECT().RemoveContainer(mock.Anything, "c-old", false).Return(assert.AnError).Once()
142
143 svc := deployment.NewService(deployment.Deps{
144 State: store, Runtime: runtime, Images: images, Secrets: secrets,
145 ImagePolicy: domain.ImageSourcePolicy{AllowedRegistries: []string{"registry.example.com"}},
146 }, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
147 func(context.Context, string, string) (int, error) { return 500, nil },
148 func(context.Context, string) error { return assert.AnError },
149 ))
150
151 _, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog"})
152 require.Error(t, err)
153
154 inhibitions, err := store.LoadRecoveryInhibitions(ctx, "blog")
155 require.NoError(t, err)
156 require.Len(t, inhibitions, 1)
157 assert.Equal(t, "c-old", inhibitions[0].ContainerID)
158 assert.Equal(t, domain.AppInhibitReplacementPending, inhibitions[0].Reason)
159
160 // A later boot must refuse to revive the inhibited generation, and a
161 // rebuilt engine must reach the same conclusion.
162 runtime2 := outmocks.NewMockContainerRuntime(t)
163 rebooted := deployment.NewService(deployment.Deps{
164 State: store, Runtime: runtime2, Images: images, Secrets: secrets,
165 }, zerowrap.Default())
166 bootErr := rebooted.ReconcileBoot(ctx)
167 require.Error(t, bootErr)
168 assert.Contains(t, bootErr.Error(), "recovery inhibited")
169 runtime2.AssertNotCalled(t, "StartContainer", mock.Anything, "c-old")
170 runtime2.AssertNotCalled(t, "RestartContainer", mock.Anything, "c-old", mock.Anything)
171 runtime2.AssertNotCalled(t, "RemoveVolume", mock.Anything, mock.Anything, mock.Anything)
172}
173
174// TestRemove_InhibitsBeforeRuntimeWithdrawal proves remove is durably
175// fail-closed: stopped intent and an inhibition per removed container are
176// persisted before any runtime effect, and volumes are never deleted.
177func TestRemove_InhibitsBeforeRuntimeWithdrawal(t *testing.T) {
178 ctx := context.Background()
179 state, runtime, images, secrets := mockDeps(t)
180
181 active := domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
182 "web": {Container: "c-1", EffectiveRevision: "rev-1", Spec: headlessSpec()},
183 }}
184 state.EXPECT().Recover(mock.Anything).Return(nil).Once()
185 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil).Once()
186
187 var order []string
188 state.EXPECT().SaveIntent(mock.Anything, mock.MatchedBy(func(intent domain.AppStopIntent) bool {
189 return intent.Stopped
190 })).RunAndReturn(func(context.Context, domain.AppStopIntent) error {
191 order = append(order, "intent-stopped")
192 return nil
193 }).Once()
194 state.EXPECT().SaveRecoveryInhibition(mock.Anything, mock.MatchedBy(func(inhibition domain.AppRecoveryInhibition) bool {
195 return inhibition.App == "blog" && inhibition.Service == "web" && inhibition.ContainerID == "c-1"
196 })).RunAndReturn(func(context.Context, domain.AppRecoveryInhibition) error {
197 order = append(order, "inhibited")
198 return nil
199 }).Once()
200 runtime.EXPECT().StopContainer(mock.Anything, "c-1", mock.Anything).RunAndReturn(func(context.Context, string, time.Duration) error {
201 order = append(order, "stopped")
202 return nil
203 }).Once()
204 runtime.EXPECT().RemoveContainer(mock.Anything, "c-1", false).RunAndReturn(func(context.Context, string, bool) error {
205 order = append(order, "removed")
206 return nil
207 }).Once()
208 // Confirmed disappearance releases the container's backend claims and
209 // clears its recovery inhibition before the incarnation is retired.
210 state.EXPECT().ReleaseBackendBinds(mock.Anything, "blog", "c-1").Return(nil).Once()
211 state.EXPECT().ClearRecoveryInhibition(mock.Anything, "blog", "web", "c-1").Return(nil).Once()
212 // Ownership is read while it is still live; the name-only record owns
213 // no network, so nothing is reclaimed.
214 state.EXPECT().LoadOwnership(mock.Anything, "blog").Return(domain.AppOwnership{App: "blog"}, nil).Once()
215 // The incarnation is retired atomically after the workload is gone.
216 state.EXPECT().RetireApp(mock.Anything, "blog").RunAndReturn(func(context.Context, string) error {
217 order = append(order, "retired")
218 return nil
219 }).Once()
220 state.EXPECT().SaveOperation(mock.Anything, mock.Anything).Return(nil)
221
222 svc := deployment.NewService(deployment.Deps{
223 State: state, Runtime: runtime, Images: images, Secrets: secrets,
224 }, zerowrap.Default())
225 result, err := svc.Remove(ctx, "blog", "")
226 require.NoError(t, err)
227 require.NotNil(t, result)
228 assert.Equal(t, []string{"intent-stopped", "inhibited", "stopped", "removed", "retired"}, order)
229 runtime.AssertNotCalled(t, "RemoveVolume", mock.Anything, mock.Anything, mock.Anything)
230}
231
232// TestBootStoppedConvergenceStopsRevivedContainer proves durable stopped
233// intent wins over native restart policy: a container revived while the
234// daemon was away is stopped again, without removal, and is never started.
235func TestBootStoppedConvergenceStopsRevivedContainer(t *testing.T) {
236 ctx := context.Background()
237 state, runtime, images, secrets := mockDeps(t)
238
239 active := domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
240 "web": {Container: "c-1", EffectiveRevision: "rev-1", Spec: headlessSpec()},
241 }}
242 state.EXPECT().Recover(mock.Anything).Return(nil).Once()
243 state.EXPECT().ListApps(mock.Anything).Return([]string{"blog"}, nil).Once()
244 state.EXPECT().LoadIntent(mock.Anything, "blog").
245 Return(domain.AppStopIntent{App: "blog", Stopped: true}, nil).Once()
246 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil).Once()
247 runtime.EXPECT().InspectContainer(mock.Anything, "c-1").
248 Return(&domain.Container{ID: "c-1", Status: "running"}, nil).Once()
249 runtime.EXPECT().StopContainer(mock.Anything, "c-1", mock.Anything).Return(nil).Once()
250
251 svc := deployment.NewService(deployment.Deps{
252 State: state, Runtime: runtime, Images: images, Secrets: secrets,
253 }, zerowrap.Default())
254 require.NoError(t, svc.ReconcileBoot(ctx))
255 runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
256 runtime.AssertNotCalled(t, "RestartContainer", mock.Anything, mock.Anything, mock.Anything)
257 runtime.AssertNotCalled(t, "RemoveContainer", mock.Anything, mock.Anything, mock.Anything)
258}
259
260// TestBootRefusesInhibitedGenerationBeforeAnyRuntimeEffect proves boot
261// recovery checks the durable marker before touching the workload.
262func TestBootRefusesInhibitedGenerationBeforeAnyRuntimeEffect(t *testing.T) {
263 ctx := context.Background()
264 state, runtime, images, secrets := mockDeps(t)
265
266 active := domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
267 "web": {Container: "c-old", EffectiveRevision: "rev-1", Spec: headlessSpec()},
268 }}
269 state.EXPECT().Recover(mock.Anything).Return(nil)
270 state.EXPECT().ListApps(mock.Anything).Return([]string{"blog"}, nil).Once()
271 state.EXPECT().LoadIntent(mock.Anything, "blog").Return(domain.AppStopIntent{App: "blog"}, nil).Once()
272 state.EXPECT().SaveOperation(mock.Anything, mock.Anything).Return(nil)
273 state.EXPECT().SaveIntent(mock.Anything, mock.Anything).Return(nil)
274 state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").
275 Return([]domain.AppRecoveryInhibition{{App: "blog", Service: "web", ContainerID: "c-old"}}, nil).Once()
276 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil)
277
278 svc := deployment.NewService(deployment.Deps{
279 State: state, Runtime: runtime, Images: images, Secrets: secrets,
280 }, zerowrap.Default())
281 err := svc.ReconcileBoot(ctx)
282 require.Error(t, err)
283 assert.Contains(t, err.Error(), "recovery inhibited")
284 runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
285 runtime.AssertNotCalled(t, "RestartContainer", mock.Anything, mock.Anything, mock.Anything)
286 runtime.AssertNotCalled(t, "CreateContainer", mock.Anything, mock.Anything)
287}
288
289// TestWorkloadMutationsSerializePerApp proves a periodic pass cannot
290// interleave with an in-flight user mutation of the same app: the pass
291// skips the busy app instead of waiting for it.
292func TestWorkloadMutationsSerializePerApp(t *testing.T) {
293 ctx := context.Background()
294 state, runtime, images, secrets := mockDeps(t)
295
296 entered := make(chan struct{})
297 release := make(chan struct{})
298 state.EXPECT().Recover(mock.Anything).RunAndReturn(func(context.Context) error {
299 close(entered)
300 <-release
301 return assert.AnError
302 }).Once()
303 // Only the app list may be read: the pass must not touch intent or
304 // ACTIVE while the mutation holds the app lock.
305 state.EXPECT().ListApps(mock.Anything).Return([]string{"blog"}, nil).Once()
306
307 svc := deployment.NewService(deployment.Deps{
308 State: state, Runtime: runtime, Images: images, Secrets: secrets,
309 }, zerowrap.Default())
310
311 done := make(chan error, 1)
312 go func() {
313 _, err := svc.Stop(ctx, "blog", "")
314 done <- err
315 }()
316 <-entered
317
318 // The mutation holds the app lock. The periodic pass must skip the
319 // app instead of waiting: LoadIntent has no expectation, so reaching
320 // it would fail the test.
321 require.NoError(t, svc.ReconcileRunning(ctx))
322 state.AssertNotCalled(t, "LoadIntent", mock.Anything, mock.Anything)
323
324 close(release)
325 require.Error(t, <-done)
326}
327
328// TestRestart_RefusesInhibitedGeneration proves an explicit operator
329// restart cannot revive a generation whose recovery is durably inhibited:
330// a replacement may already have written to the volume it still owns.
331func TestRestart_RefusesInhibitedGeneration(t *testing.T) {
332 ctx := context.Background()
333 state, runtime, images, secrets := mockDeps(t)
334
335 active := domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
336 "web": {Container: "c-old", EffectiveRevision: "rev-1", Spec: headlessSpec()},
337 }}
338 state.EXPECT().Recover(mock.Anything).Return(nil).Once()
339 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil).Once()
340 state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return([]domain.AppRecoveryInhibition{{
341 App: "blog", Service: "web", ContainerID: "c-old", Reason: domain.AppInhibitReplacementPending,
342 }}, nil).Once()
343 state.EXPECT().SaveOperation(mock.Anything, mock.Anything).Return(nil)
344
345 svc := deployment.NewService(deployment.Deps{
346 State: state, Runtime: runtime, Images: images, Secrets: secrets,
347 }, zerowrap.Default())
348 result, err := svc.Restart(ctx, "blog", "", "")
349 require.ErrorIs(t, err, domain.ErrAppStateConflict)
350 require.NotNil(t, result)
351 assert.Equal(t, "failed", result.Services["web"].Result)
352 assert.Contains(t, result.Services["web"].Error, "recovery inhibited")
353 runtime.AssertNotCalled(t, "RestartContainer", mock.Anything, mock.Anything, mock.Anything)
354}
355
356// TestRemove_KeepsStateWhenAContainerCannotBeRemoved proves remove never
357// loses track of a survivor: ACTIVE, stopped intent, and the inhibition
358// stay in place so the periodic pass keeps converging it.
359func TestRemove_KeepsStateWhenAContainerCannotBeRemoved(t *testing.T) {
360 ctx := context.Background()
361 state, runtime, images, secrets := mockDeps(t)
362
363 active := domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
364 "web": {Container: "c-1", EffectiveRevision: "rev-1", Spec: headlessSpec()},
365 }}
366 state.EXPECT().Recover(mock.Anything).Return(nil).Once()
367 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil).Once()
368 state.EXPECT().SaveIntent(mock.Anything, mock.MatchedBy(func(intent domain.AppStopIntent) bool {
369 return intent.Stopped
370 })).Return(nil).Once()
371 state.EXPECT().SaveRecoveryInhibition(mock.Anything, mock.Anything).Return(nil).Once()
372 // The claim journal is written before the runtime effects and is
373 // rewritten terminally when the removal cannot complete: a repeat of
374 // the same key replays the failure instead of reading a stuck claim.
375 state.EXPECT().SaveOperation(mock.Anything, mock.MatchedBy(func(op domain.AppOperation) bool {
376 return op.Op != "" && op.Kind == "remove" && !op.Terminal()
377 })).Return(nil).Once()
378 state.EXPECT().SaveOperation(mock.Anything, mock.MatchedBy(func(op domain.AppOperation) bool {
379 return op.Op != "" && op.Outcome == domain.AppOutcomeFailed && op.Terminal()
380 })).Return(nil).Once()
381 runtime.EXPECT().StopContainer(mock.Anything, "c-1", mock.Anything).Return(assert.AnError).Once()
382
383 svc := deployment.NewService(deployment.Deps{
384 State: state, Runtime: runtime, Images: images, Secrets: secrets,
385 }, zerowrap.Default())
386 _, err := svc.Remove(ctx, "blog", "")
387 require.Error(t, err)
388 assert.Contains(t, err.Error(), "not confirmed gone")
389 // ACTIVE and the stopped intent were never discarded: the monitor can
390 // still converge the survivor. The survivor's claims and inhibition
391 // stay in place too: only a confirmed disappearance releases them.
392 state.AssertNotCalled(t, "SaveActive", mock.Anything, mock.Anything)
393 state.AssertNotCalled(t, "RetireApp", mock.Anything, mock.Anything)
394 state.AssertNotCalled(t, "ReleaseBackendBinds", mock.Anything, mock.Anything, mock.Anything)
395 state.AssertNotCalled(t, "ClearRecoveryInhibition", mock.Anything, mock.Anything, mock.Anything, mock.Anything)
396 runtime.AssertNotCalled(t, "RemoveVolume", mock.Anything, mock.Anything, mock.Anything)
397}
398
399// TestRemovalOfMissingContainerIsIdempotent proves a container the
400// runtime already dropped is a successful removal, not a failure.
401func TestRemovalOfMissingContainerIsIdempotent(t *testing.T) {
402 ctx := context.Background()
403 state, runtime, images, secrets := mockDeps(t)
404
405 active := domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
406 "web": {Container: "c-1", EffectiveRevision: "rev-1", Spec: headlessSpec()},
407 }}
408 state.EXPECT().Recover(mock.Anything).Return(nil).Once()
409 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil).Once()
410 state.EXPECT().SaveOperation(mock.Anything, mock.Anything).Return(nil)
411 state.EXPECT().SaveIntent(mock.Anything, mock.Anything).Return(nil)
412 state.EXPECT().SaveRecoveryInhibition(mock.Anything, mock.Anything).Return(nil).Once()
413 runtime.EXPECT().StopContainer(mock.Anything, "c-1", mock.Anything).Return(domain.ErrContainerNotFound).Once()
414 runtime.EXPECT().RemoveContainer(mock.Anything, "c-1", false).Return(domain.ErrContainerNotFound).Once()
415 // Not-found is confirmation: the stale claims and the inhibition of
416 // the disappeared container are released.
417 state.EXPECT().ReleaseBackendBinds(mock.Anything, "blog", "c-1").Return(nil).Once()
418 state.EXPECT().ClearRecoveryInhibition(mock.Anything, "blog", "web", "c-1").Return(nil).Once()
419 state.EXPECT().LoadOwnership(mock.Anything, "blog").Return(domain.AppOwnership{App: "blog"}, nil).Once()
420 state.EXPECT().RetireApp(mock.Anything, "blog").Return(nil).Once()
421
422 svc := deployment.NewService(deployment.Deps{
423 State: state, Runtime: runtime, Images: images, Secrets: secrets,
424 }, zerowrap.Default())
425 result, err := svc.Remove(ctx, "blog", "")
426 require.NoError(t, err)
427 require.NotNil(t, result)
428}