1package deployment_test
2
3import (
4 "context"
5 "fmt"
6 "io"
7 "strings"
8 "sync"
9 "testing"
10 "time"
11
12 "github.com/bnema/zerowrap"
13 "github.com/stretchr/testify/assert"
14 "github.com/stretchr/testify/mock"
15 "github.com/stretchr/testify/require"
16
17 outmocks "github.com/bnema/gordon/internal/boundaries/out/mocks"
18 "github.com/bnema/gordon/internal/domain"
19 "github.com/bnema/gordon/internal/usecase/deployment"
20)
21
22// recordingTraffic is a fail-closed traffic boundary double: it records
23// withdrawals and publications and can fail either operation.
24type recordingTraffic struct {
25 mu sync.Mutex
26 attempts int
27 withdrawals []string
28 rebuilds int
29 failWithdraw bool
30 failRebuild bool
31}
32
33func (t *recordingTraffic) WithdrawService(_ context.Context, app, service string) error {
34 t.mu.Lock()
35 defer t.mu.Unlock()
36 t.attempts++
37 if t.failWithdraw {
38 return assert.AnError
39 }
40 t.withdrawals = append(t.withdrawals, app+"/"+service)
41 return nil
42}
43
44func (t *recordingTraffic) WithdrawServiceState(_ context.Context, app, service string) error {
45 t.mu.Lock()
46 defer t.mu.Unlock()
47 if t.failWithdraw {
48 return assert.AnError
49 }
50 t.withdrawals = append(t.withdrawals, app+"/"+service)
51 return nil
52}
53
54func (t *recordingTraffic) withdrawAttempts() int {
55 t.mu.Lock()
56 defer t.mu.Unlock()
57 return t.attempts
58}
59
60func (t *recordingTraffic) RebuildTraffic(context.Context) error {
61 t.mu.Lock()
62 defer t.mu.Unlock()
63 if t.failRebuild {
64 return assert.AnError
65 }
66 t.rebuilds++
67 return nil
68}
69
70func (t *recordingTraffic) withdrawn() []string {
71 t.mu.Lock()
72 defer t.mu.Unlock()
73 return append([]string(nil), t.withdrawals...)
74}
75
76func (t *recordingTraffic) rebuildCount() int {
77 t.mu.Lock()
78 defer t.mu.Unlock()
79 return t.rebuilds
80}
81
82// recoveringService builds the engine with a recording traffic boundary
83// and injectable L4 probes so recovery passes stay deterministic.
84func recoveringService(
85 t *testing.T,
86 state *outmocks.MockAppState,
87 runtime *outmocks.MockContainerRuntime,
88 traffic *recordingTraffic,
89 httpGet func(context.Context, string, string) (int, error),
90) *deployment.Service {
91 t.Helper()
92 return deployment.NewService(deployment.Deps{
93 State: state, Runtime: runtime, Traffic: traffic,
94 }, zerowrap.Default()).WithProbeDeps(deployment.NewTestProbeDeps(runtime,
95 httpGet,
96 func(context.Context, string) error { return nil },
97 ))
98}
99
100// headlessSpec is a service with no interfaces and no readiness, so a
101// recovery pass performs no bind inspection and no probe.
102func headlessSpec() domain.AppService {
103 return domain.AppService{Name: "web", Image: "registry.example.com/blog/web:1.4.2"}
104}
105
106func TestReconcileRunning_EmptyActiveTakesNoRuntimeAction(t *testing.T) {
107 ctx := context.Background()
108 state, runtime, _, _ := mockDeps(t)
109 traffic := &recordingTraffic{}
110
111 state.EXPECT().ListApps(mock.Anything).Return([]string{"blog"}, nil).Once()
112 state.EXPECT().LoadIntent(mock.Anything, "blog").Return(domain.AppStopIntent{App: "blog"}, nil).Once()
113 state.EXPECT().LoadActive(mock.Anything, "blog").Return(domain.AppActive{}, false, nil).Once()
114
115 svc := recoveringService(t, state, runtime, traffic, nil)
116 require.NoError(t, svc.ReconcileRunning(ctx))
117 runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
118 runtime.AssertNotCalled(t, "InspectContainer", mock.Anything, mock.Anything)
119 assert.Zero(t, traffic.rebuildCount())
120}
121
122func TestReconcileRunning_StoppedIntentStopsRevivedExactIDWithoutRemoval(t *testing.T) {
123 ctx := context.Background()
124 state, runtime, _, _ := mockDeps(t)
125 traffic := &recordingTraffic{}
126
127 active := domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
128 "web": {Container: "c-1", EffectiveRevision: "rev-1", Spec: headlessSpec()},
129 }}
130 state.EXPECT().ListApps(mock.Anything).Return([]string{"blog"}, nil).Once()
131 state.EXPECT().LoadIntent(mock.Anything, "blog").Return(domain.AppStopIntent{App: "blog", Stopped: true}, nil).Once()
132 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil)
133 state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil).Once()
134 runtime.EXPECT().InspectContainer(mock.Anything, "c-1").
135 Return(&domain.Container{ID: "c-1", Status: "running"}, nil).Once()
136 runtime.EXPECT().StopContainer(mock.Anything, "c-1", mock.Anything).Return(nil).Once()
137
138 svc := recoveringService(t, state, runtime, traffic, nil)
139 require.NoError(t, svc.ReconcileRunning(ctx))
140 runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
141 runtime.AssertNotCalled(t, "RestartContainer", mock.Anything, mock.Anything, mock.Anything)
142 runtime.AssertNotCalled(t, "RemoveContainer", mock.Anything, mock.Anything, mock.Anything)
143 assert.GreaterOrEqual(t, traffic.rebuildCount(), 1, "a stopped app is republished without its backend")
144}
145
146func TestReconcileRunning_AbstainsWhenStoppedContainerAlreadyDown(t *testing.T) {
147 ctx := context.Background()
148 state, runtime, _, _ := mockDeps(t)
149 traffic := &recordingTraffic{}
150
151 active := domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
152 "web": {Container: "c-1", Spec: headlessSpec()},
153 }}
154 state.EXPECT().ListApps(mock.Anything).Return([]string{"blog"}, nil).Once()
155 state.EXPECT().LoadIntent(mock.Anything, "blog").Return(domain.AppStopIntent{App: "blog", Stopped: true}, nil).Once()
156 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil)
157 state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil).Once()
158 runtime.EXPECT().InspectContainer(mock.Anything, "c-1").
159 Return(&domain.Container{ID: "c-1", Status: "exited"}, nil).Once()
160
161 svc := recoveringService(t, state, runtime, traffic, nil)
162 require.NoError(t, svc.ReconcileRunning(ctx))
163 runtime.AssertNotCalled(t, "StopContainer", mock.Anything, mock.Anything, mock.Anything)
164 runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
165}
166
167func TestReconcileRunning_StartsExitedExactIDThenVerifiesAndPublishes(t *testing.T) {
168 ctx := context.Background()
169 state, runtime, _, _ := mockDeps(t)
170 traffic := &recordingTraffic{}
171 spec := webService()
172 spec.Readiness.Timeout = time.Second
173
174 active := domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
175 "web": {Container: "c-1", EffectiveRevision: "rev-1", Spec: spec, BackendBinds: map[int]int{8080: 32770}},
176 }}
177 state.EXPECT().ListApps(mock.Anything).Return([]string{"blog"}, nil).Once()
178 state.EXPECT().LoadIntent(mock.Anything, "blog").Return(domain.AppStopIntent{App: "blog"}, nil).Once()
179 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil).Twice()
180 state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil).Once()
181 startedAt := time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC)
182 runtime.EXPECT().InspectContainer(mock.Anything, "c-1").
183 Return(&domain.Container{ID: "c-1", Status: "exited", StartedAt: startedAt}, nil).Once()
184 runtime.EXPECT().StartContainer(mock.Anything, "c-1").Return(nil).Once()
185 runtime.EXPECT().InspectContainer(mock.Anything, "c-1").
186 Return(&domain.Container{ID: "c-1", Status: "running", StartedAt: startedAt}, nil).Once()
187 runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "c-1").Return("", false, nil).Once()
188 runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-1", mock.Anything).
189 Return([]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 32771, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
190 state.EXPECT().RegisterBackendBinds(mock.Anything, mock.MatchedBy(func(claims []domain.AppListenerReservation) bool {
191 return len(claims) == 1 && claims[0].Port == 32771 && claims[0].ContainerID == "c-1"
192 })).Return(nil).Once()
193 state.EXPECT().SaveActive(mock.Anything, mock.MatchedBy(func(a domain.AppActive) bool {
194 return a.Services["web"].BackendBinds[8080] == 32771
195 })).Return(nil).Once()
196
197 var probedURL string
198 svc := recoveringService(t, state, runtime, traffic, func(_ context.Context, url, _ string) (int, error) {
199 probedURL = url
200 return 200, nil
201 })
202 require.NoError(t, svc.ReconcileRunning(ctx))
203
204 assert.Contains(t, probedURL, "127.0.0.1:32771", "readiness follows the refreshed bind")
205 assert.Equal(t, []string{"blog/web"}, traffic.withdrawn(), "the stale backend is withdrawn before recovery")
206 assert.Equal(t, 1, traffic.rebuildCount())
207 runtime.AssertNotCalled(t, "CreateContainer", mock.Anything, mock.Anything)
208 runtime.AssertNotCalled(t, "PullImage", mock.Anything, mock.Anything)
209 runtime.AssertNotCalled(t, "RemoveContainer", mock.Anything, mock.Anything, mock.Anything)
210}
211
212func TestReconcileRunning_ConfirmedMissingWithdrawsWithoutReconstruction(t *testing.T) {
213 ctx := context.Background()
214 state, runtime, _, _ := mockDeps(t)
215 traffic := &recordingTraffic{}
216
217 active := domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
218 "web": {Container: "c-1", EffectiveRevision: "rev-1", Spec: headlessSpec()},
219 }}
220 state.EXPECT().ListApps(mock.Anything).Return([]string{"blog"}, nil).Once()
221 state.EXPECT().LoadIntent(mock.Anything, "blog").Return(domain.AppStopIntent{App: "blog"}, nil).Once()
222 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil)
223 state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil).Once()
224 runtime.EXPECT().InspectContainer(mock.Anything, "c-1").
225 Return(nil, fmt.Errorf("inspect: %w", domain.ErrContainerNotFound)).Once()
226 // A confirmed-missing container must not keep its loopback claims.
227 state.EXPECT().ReleaseBackendBinds(mock.Anything, "blog", "c-1").Return(nil).Once()
228
229 svc := recoveringService(t, state, runtime, traffic, nil)
230 err := svc.ReconcileRunning(ctx)
231 require.Error(t, err)
232 assert.ErrorIs(t, err, domain.ErrContainerNotFound)
233 assert.Contains(t, err.Error(), "no reconstruction")
234 assert.Equal(t, []string{"blog/web"}, traffic.withdrawn())
235 runtime.AssertNotCalled(t, "CreateContainer", mock.Anything, mock.Anything)
236 runtime.AssertNotCalled(t, "PullImage", mock.Anything, mock.Anything)
237 runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
238 runtime.AssertNotCalled(t, "RemoveContainer", mock.Anything, mock.Anything, mock.Anything)
239}
240
241func TestReconcileRunning_TransientInspectErrorTakesNoAction(t *testing.T) {
242 ctx := context.Background()
243 state, runtime, _, _ := mockDeps(t)
244 traffic := &recordingTraffic{}
245
246 active := domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
247 "web": {Container: "c-1", EffectiveRevision: "rev-1", Spec: headlessSpec()},
248 }}
249 state.EXPECT().ListApps(mock.Anything).Return([]string{"blog"}, nil).Once()
250 state.EXPECT().LoadIntent(mock.Anything, "blog").Return(domain.AppStopIntent{App: "blog"}, nil).Once()
251 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil)
252 state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil).Once()
253 runtime.EXPECT().InspectContainer(mock.Anything, "c-1").Return(nil, assert.AnError).Once()
254
255 svc := recoveringService(t, state, runtime, traffic, nil)
256 err := svc.ReconcileRunning(ctx)
257 require.Error(t, err)
258 assert.NotErrorIs(t, err, domain.ErrContainerNotFound)
259 assert.Empty(t, traffic.withdrawn(), "a transient error is not a dead backend")
260 assert.Zero(t, traffic.rebuildCount())
261 runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
262 runtime.AssertNotCalled(t, "CreateContainer", mock.Anything, mock.Anything)
263 runtime.AssertNotCalled(t, "RemoveContainer", mock.Anything, mock.Anything, mock.Anything)
264}
265
266func TestReconcileRunning_InhibitedGenerationIsNeverStarted(t *testing.T) {
267 ctx := context.Background()
268 state, runtime, _, _ := mockDeps(t)
269 traffic := &recordingTraffic{}
270
271 active := domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
272 "web": {Container: "c-old", EffectiveRevision: "rev-1", Spec: headlessSpec(), BackendBinds: map[int]int{8080: 32770}},
273 }}
274 state.EXPECT().ListApps(mock.Anything).Return([]string{"blog"}, nil).Once()
275 state.EXPECT().LoadIntent(mock.Anything, "blog").Return(domain.AppStopIntent{App: "blog"}, nil).Once()
276 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil).Once()
277 state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return([]domain.AppRecoveryInhibition{{
278 App: "blog", Service: "web", ContainerID: "c-old", Reason: domain.AppInhibitReplacementPending,
279 }}, nil).Once()
280
281 svc := recoveringService(t, state, runtime, traffic, nil)
282 err := svc.ReconcileRunning(ctx)
283 require.Error(t, err)
284 assert.Contains(t, err.Error(), "recovery inhibited")
285 runtime.AssertNotCalled(t, "InspectContainer", mock.Anything, mock.Anything)
286 runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
287 runtime.AssertNotCalled(t, "RestartContainer", mock.Anything, mock.Anything, mock.Anything)
288 assert.Equal(t, []string{"blog/web"}, traffic.withdrawn(),
289 "an inhibited generation must not stay published")
290}
291
292func TestReconcileRunning_NativeRestartRaceIsNotCharged(t *testing.T) {
293 ctx := context.Background()
294 state, runtime, _, _ := mockDeps(t)
295 traffic := &recordingTraffic{}
296
297 active := domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
298 "web": {Container: "c-1", EffectiveRevision: "rev-1", Spec: headlessSpec()},
299 }}
300 state.EXPECT().ListApps(mock.Anything).Return([]string{"blog"}, nil).Once()
301 state.EXPECT().LoadIntent(mock.Anything, "blog").Return(domain.AppStopIntent{App: "blog"}, nil).Once()
302 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil).Twice()
303 state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil).Once()
304 startedAt := time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC)
305 runtime.EXPECT().InspectContainer(mock.Anything, "c-1").
306 Return(&domain.Container{ID: "c-1", Status: "exited", StartedAt: startedAt}, nil).Once()
307 runtime.EXPECT().StartContainer(mock.Anything, "c-1").Return(assert.AnError).Once()
308 // Native restart policy already revived it: the failed start is not
309 // charged and no second restart is issued.
310 runtime.EXPECT().InspectContainer(mock.Anything, "c-1").
311 Return(&domain.Container{ID: "c-1", Status: "running", StartedAt: startedAt}, nil).Twice()
312 runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "c-1").Return("", false, nil).Once()
313
314 svc := recoveringService(t, state, runtime, traffic, nil)
315 require.NoError(t, svc.ReconcileRunning(ctx))
316 runtime.AssertNotCalled(t, "RestartContainer", mock.Anything, mock.Anything, mock.Anything)
317 assert.Equal(t, 1, traffic.rebuildCount())
318}
319
320func TestReconcileRunning_NativeRestartRefreshesChangedBinds(t *testing.T) {
321 ctx := context.Background()
322 state, runtime, _, _ := mockDeps(t)
323 traffic := &recordingTraffic{}
324 spec := webService()
325 spec.Readiness = domain.AppReadiness{Type: "none"}
326
327 active := domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
328 "web": {Container: "c-1", EffectiveRevision: "rev-1", Spec: spec, BackendBinds: map[int]int{8080: 32770}},
329 }}
330 state.EXPECT().ListApps(mock.Anything).Return([]string{"blog"}, nil).Once()
331 state.EXPECT().LoadIntent(mock.Anything, "blog").Return(domain.AppStopIntent{App: "blog"}, nil).Once()
332 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil).Twice()
333 state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil).Once()
334 runtime.EXPECT().InspectContainer(mock.Anything, "c-1").Return(&domain.Container{
335 ID: "c-1", Status: "running",
336 StartedAt: time.Date(2026, 9, 10, 12, 5, 0, 0, time.UTC),
337 }, nil).Twice()
338 runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "c-1").Return("", false, nil).Once()
339 runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-1", mock.Anything).
340 Return([]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 32799, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
341 state.EXPECT().RegisterBackendBinds(mock.Anything, mock.Anything).Return(nil).Once()
342 state.EXPECT().SaveActive(mock.Anything, mock.MatchedBy(func(a domain.AppActive) bool {
343 return a.Services["web"].BackendBinds[8080] == 32799
344 })).Return(nil).Once()
345
346 svc := recoveringService(t, state, runtime, traffic, nil)
347 require.NoError(t, svc.ReconcileRunning(ctx))
348 runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
349 runtime.AssertNotCalled(t, "RestartContainer", mock.Anything, mock.Anything, mock.Anything)
350 assert.Equal(t, 1, traffic.rebuildCount(), "a changed bind is republished once")
351}
352
353func TestReconcileRunning_RestartingObservesAndRetries(t *testing.T) {
354 ctx := context.Background()
355 state, runtime, _, _ := mockDeps(t)
356 traffic := &recordingTraffic{}
357
358 active := domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
359 "web": {Container: "c-1", EffectiveRevision: "rev-1", Spec: headlessSpec()},
360 }}
361 state.EXPECT().ListApps(mock.Anything).Return([]string{"blog"}, nil).Once()
362 state.EXPECT().LoadIntent(mock.Anything, "blog").Return(domain.AppStopIntent{App: "blog"}, nil).Once()
363 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil).Once()
364 state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil).Once()
365 runtime.EXPECT().InspectContainer(mock.Anything, "c-1").
366 Return(&domain.Container{ID: "c-1", Status: "restarting"}, nil).Once()
367
368 svc := recoveringService(t, state, runtime, traffic, nil)
369 require.NoError(t, svc.ReconcileRunning(ctx))
370 runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
371 runtime.AssertNotCalled(t, "RestartContainer", mock.Anything, mock.Anything, mock.Anything)
372}
373
374func TestReconcileRunning_PausedRefusesNonDestructiveRecovery(t *testing.T) {
375 ctx := context.Background()
376 state, runtime, _, _ := mockDeps(t)
377 traffic := &recordingTraffic{}
378
379 active := domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
380 "web": {Container: "c-1", EffectiveRevision: "rev-1", Spec: headlessSpec()},
381 }}
382 state.EXPECT().ListApps(mock.Anything).Return([]string{"blog"}, nil).Once()
383 state.EXPECT().LoadIntent(mock.Anything, "blog").Return(domain.AppStopIntent{App: "blog"}, nil).Once()
384 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil).Once()
385 state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil).Once()
386 runtime.EXPECT().InspectContainer(mock.Anything, "c-1").
387 Return(&domain.Container{ID: "c-1", Status: "paused"}, nil).Once()
388
389 svc := recoveringService(t, state, runtime, traffic, nil)
390 err := svc.ReconcileRunning(ctx)
391 require.Error(t, err)
392 assert.Contains(t, err.Error(), "paused")
393 runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
394 runtime.AssertNotCalled(t, "RestartContainer", mock.Anything, mock.Anything, mock.Anything)
395 runtime.AssertNotCalled(t, "RemoveContainer", mock.Anything, mock.Anything, mock.Anything)
396}
397
398func TestReconcileRunning_OneFailureDoesNotBlockLaterApps(t *testing.T) {
399 ctx := context.Background()
400 state, runtime, _, _ := mockDeps(t)
401 traffic := &recordingTraffic{}
402
403 badActive := domain.AppActive{App: "bad", Services: map[string]domain.AppEffectiveService{
404 "web": {Container: "c-bad", EffectiveRevision: "rev-1", Spec: headlessSpec()},
405 }}
406 bindSpec := webService()
407 bindSpec.Readiness = domain.AppReadiness{Type: "none"}
408 goodActive := domain.AppActive{App: "good", Services: map[string]domain.AppEffectiveService{
409 "web": {Container: "c-good", EffectiveRevision: "rev-1", Spec: bindSpec, BackendBinds: map[int]int{8080: 32770}},
410 }}
411 state.EXPECT().ListApps(mock.Anything).Return([]string{"bad", "good"}, nil).Once()
412 state.EXPECT().LoadIntent(mock.Anything, "bad").Return(domain.AppStopIntent{App: "bad"}, nil).Once()
413 state.EXPECT().LoadIntent(mock.Anything, "good").Return(domain.AppStopIntent{App: "good"}, nil).Once()
414 state.EXPECT().LoadActive(mock.Anything, "bad").Return(badActive, true, nil)
415 state.EXPECT().LoadActive(mock.Anything, "good").Return(goodActive, true, nil).Twice()
416 state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "bad").Return(nil, nil).Once()
417 state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "good").Return(nil, nil).Once()
418 runtime.EXPECT().InspectContainer(mock.Anything, "c-bad").Return(nil, assert.AnError).Once()
419 runtime.EXPECT().InspectContainer(mock.Anything, "c-good").Return(&domain.Container{
420 ID: "c-good", Status: "running",
421 StartedAt: time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC),
422 }, nil)
423 runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "c-good").Return("", false, nil).Once()
424 runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-good", mock.Anything).
425 Return([]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 32771, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
426 state.EXPECT().RegisterBackendBinds(mock.Anything, mock.Anything).Return(nil).Once()
427 state.EXPECT().SaveActive(mock.Anything, mock.Anything).Return(nil).Once()
428
429 svc := recoveringService(t, state, runtime, traffic, nil)
430 err := svc.ReconcileRunning(ctx)
431 require.Error(t, err)
432 assert.Contains(t, err.Error(), `"bad"`)
433 runtime.AssertCalled(t, "InspectContainer", mock.Anything, "c-good")
434 assert.Equal(t, 1, traffic.rebuildCount(), "the healthy app is still published")
435}
436
437func TestReconcileRunning_WithdrawalFailureBlocksRuntimeRecovery(t *testing.T) {
438 ctx := context.Background()
439 state, runtime, _, _ := mockDeps(t)
440 traffic := &recordingTraffic{failWithdraw: true}
441
442 active := domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
443 "web": {Container: "c-1", EffectiveRevision: "rev-1", Spec: headlessSpec(), BackendBinds: map[int]int{8080: 32770}},
444 }}
445 state.EXPECT().ListApps(mock.Anything).Return([]string{"blog"}, nil).Once()
446 state.EXPECT().LoadIntent(mock.Anything, "blog").Return(domain.AppStopIntent{App: "blog"}, nil).Once()
447 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil).Once()
448 state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil).Once()
449 runtime.EXPECT().InspectContainer(mock.Anything, "c-1").
450 Return(&domain.Container{ID: "c-1", Status: "exited"}, nil).Once()
451
452 svc := recoveringService(t, state, runtime, traffic, nil)
453 err := svc.ReconcileRunning(ctx)
454 require.Error(t, err)
455 assert.Contains(t, err.Error(), "withdraw")
456 runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything,
457 "runtime recovery requires a proven fail-closed withdrawal")
458 assert.Zero(t, traffic.rebuildCount())
459}
460
461func TestReconcileRunning_RetriesWithdrawalBeforeAnyLaterPublication(t *testing.T) {
462 ctx := context.Background()
463 state, runtime, _, _ := mockDeps(t)
464 traffic := &recordingTraffic{failWithdraw: true}
465
466 active := domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
467 "web": {Container: "c-1", EffectiveRevision: "rev-1", Spec: headlessSpec(), BackendBinds: map[int]int{8080: 32770}},
468 }}
469 state.EXPECT().ListApps(mock.Anything).Return([]string{"blog"}, nil)
470 state.EXPECT().LoadIntent(mock.Anything, "blog").Return(domain.AppStopIntent{App: "blog"}, nil)
471 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil)
472 state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil)
473 runtime.EXPECT().InspectContainer(mock.Anything, "c-1").
474 Return(&domain.Container{ID: "c-1", Status: "exited"}, nil).Times(3)
475 runtime.EXPECT().StartContainer(mock.Anything, "c-1").Return(nil).Once()
476 runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "c-1").Return("", false, nil).Once()
477 state.EXPECT().SaveActive(mock.Anything, mock.Anything).Return(nil).Once()
478
479 svc := recoveringService(t, state, runtime, traffic, nil)
480 require.Error(t, svc.ReconcileRunning(ctx))
481 runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
482
483 // Recovery is retried on a later pass and must withdraw again before
484 // it may start or publish anything.
485 traffic.mu.Lock()
486 traffic.failWithdraw = false
487 traffic.mu.Unlock()
488 require.NoError(t, svc.ReconcileRunning(ctx))
489 runtime.AssertCalled(t, "StartContainer", mock.Anything, "c-1")
490 assert.Equal(t, 2, traffic.withdrawAttempts(), "withdrawal is retried, never skipped")
491}
492
493func TestReconcileRunning_RebuildFailureDoesNotRestartHealthyContainer(t *testing.T) {
494 ctx := context.Background()
495 state, runtime, _, _ := mockDeps(t)
496 traffic := &recordingTraffic{failRebuild: true}
497
498 bindSpec := webService()
499 bindSpec.Readiness = domain.AppReadiness{Type: "none"}
500 active := domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
501 "web": {Container: "c-1", EffectiveRevision: "rev-1", Spec: bindSpec, BackendBinds: map[int]int{8080: 32770}},
502 }}
503 state.EXPECT().ListApps(mock.Anything).Return([]string{"blog"}, nil).Once()
504 state.EXPECT().LoadIntent(mock.Anything, "blog").Return(domain.AppStopIntent{App: "blog"}, nil).Once()
505 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil).Twice()
506 state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil).Once()
507 runtime.EXPECT().InspectContainer(mock.Anything, "c-1").Return(&domain.Container{
508 ID: "c-1", Status: "running", StartedAt: time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC),
509 }, nil).Twice()
510 runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "c-1").Return("", false, nil).Once()
511 runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-1", mock.Anything).
512 Return([]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 32771, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
513 state.EXPECT().RegisterBackendBinds(mock.Anything, mock.Anything).Return(nil).Once()
514 state.EXPECT().SaveActive(mock.Anything, mock.Anything).Return(nil).Once()
515
516 svc := recoveringService(t, state, runtime, traffic, nil)
517 err := svc.ReconcileRunning(ctx)
518 require.Error(t, err)
519 runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
520 runtime.AssertNotCalled(t, "RestartContainer", mock.Anything, mock.Anything, mock.Anything)
521 runtime.AssertNotCalled(t, "RemoveContainer", mock.Anything, mock.Anything, mock.Anything)
522}
523
524func TestReconcileRunning_UnhealthyRestartsOnlyAfterConsecutiveObservations(t *testing.T) {
525 ctx := context.Background()
526 state, runtime, _, _ := mockDeps(t)
527 traffic := &recordingTraffic{}
528
529 active := domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
530 "web": {Container: "c-1", EffectiveRevision: "rev-1", Spec: headlessSpec()},
531 }}
532 startedAt := time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC)
533 state.EXPECT().ListApps(mock.Anything).Return([]string{"blog"}, nil)
534 state.EXPECT().LoadIntent(mock.Anything, "blog").Return(domain.AppStopIntent{App: "blog"}, nil)
535 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil)
536 state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil)
537 runtime.EXPECT().InspectContainer(mock.Anything, "c-1").
538 Return(&domain.Container{ID: "c-1", Status: "running", StartedAt: startedAt}, nil)
539 runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "c-1").Return("unhealthy", true, nil)
540 runtime.EXPECT().RestartContainer(mock.Anything, "c-1", mock.Anything).Return(nil).Once()
541
542 svc := recoveringService(t, state, runtime, traffic, nil)
543 // First observation only arms the streak.
544 require.NoError(t, svc.ReconcileRunning(ctx))
545 runtime.AssertNotCalled(t, "RestartContainer", mock.Anything, mock.Anything, mock.Anything)
546
547 // The second consecutive unhealthy observation restarts the exact ID
548 // with the effective stop grace.
549 require.NoError(t, svc.ReconcileRunning(ctx))
550 runtime.AssertCalled(t, "RestartContainer", mock.Anything, "c-1", domain.AppDefaultStopGrace)
551}
552
553func TestReconcileRunning_HealthStartingWaits(t *testing.T) {
554 ctx := context.Background()
555 state, runtime, _, _ := mockDeps(t)
556 traffic := &recordingTraffic{}
557
558 active := domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
559 "web": {Container: "c-1", EffectiveRevision: "rev-1", Spec: headlessSpec()},
560 }}
561 state.EXPECT().ListApps(mock.Anything).Return([]string{"blog"}, nil)
562 state.EXPECT().LoadIntent(mock.Anything, "blog").Return(domain.AppStopIntent{App: "blog"}, nil)
563 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil)
564 state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil)
565 runtime.EXPECT().InspectContainer(mock.Anything, "c-1").Return(&domain.Container{
566 ID: "c-1", Status: "running", StartedAt: time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC),
567 }, nil)
568 runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "c-1").Return("starting", true, nil)
569
570 svc := recoveringService(t, state, runtime, traffic, nil)
571 for range 3 {
572 require.NoError(t, svc.ReconcileRunning(ctx))
573 }
574 runtime.AssertNotCalled(t, "RestartContainer", mock.Anything, mock.Anything, mock.Anything)
575}
576
577func TestReconcileRunning_NoHealthcheckNeverRestartsOnReadiness(t *testing.T) {
578 ctx := context.Background()
579 state, runtime, _, _ := mockDeps(t)
580 traffic := &recordingTraffic{}
581 spec := webService()
582 spec.Readiness.Timeout = 20 * time.Millisecond
583
584 active := domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
585 "web": {Container: "c-1", EffectiveRevision: "rev-1", Spec: spec, BackendBinds: map[int]int{8080: 32770}},
586 }}
587 firstExecution := time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC)
588 state.EXPECT().ListApps(mock.Anything).Return([]string{"blog"}, nil)
589 state.EXPECT().LoadIntent(mock.Anything, "blog").Return(domain.AppStopIntent{App: "blog"}, nil)
590 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil)
591 state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil)
592 runtime.EXPECT().InspectContainer(mock.Anything, "c-1").
593 Return(&domain.Container{ID: "c-1", Status: "running", StartedAt: firstExecution}, nil).Once()
594 runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "c-1").Return("", false, nil)
595 runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-1", mock.Anything).
596 Return([]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 32770, Protocol: domain.NetworkProtocolTCP}}, nil)
597 state.EXPECT().RegisterBackendBinds(mock.Anything, mock.Anything).Return(nil)
598
599 // An execution unseen by this daemon is verified immediately because it
600 // may have restarted while Gordon was absent. Readiness failure keeps it
601 // withdrawn, but never restarts an otherwise running workload.
602 svc := recoveringService(t, state, runtime, traffic, func(context.Context, string, string) (int, error) {
603 return 500, nil
604 })
605 require.Error(t, svc.ReconcileRunning(ctx), "the unseen execution fails readiness and stays withdrawn")
606 runtime.AssertNotCalled(t, "RestartContainer", mock.Anything, mock.Anything, mock.Anything)
607 runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
608}
609
610func TestReconcileRunning_StaleGenerationIsNeverPublished(t *testing.T) {
611 ctx := context.Background()
612 state, runtime, _, _ := mockDeps(t)
613 traffic := &recordingTraffic{}
614
615 active := domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
616 "web": {Container: "c-1", EffectiveRevision: "rev-1", Spec: headlessSpec()},
617 }}
618 // A concurrent mutation replaced the generation while recovery held
619 // the app lock: the reloaded ACTIVE names a different container.
620 replaced := domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
621 "web": {Container: "c-2", EffectiveRevision: "rev-2", Spec: headlessSpec()},
622 }}
623 state.EXPECT().ListApps(mock.Anything).Return([]string{"blog"}, nil).Once()
624 state.EXPECT().LoadIntent(mock.Anything, "blog").Return(domain.AppStopIntent{App: "blog"}, nil).Once()
625 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil).Once()
626 state.EXPECT().LoadActive(mock.Anything, "blog").Return(replaced, true, nil).Once()
627 state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil).Once()
628 runtime.EXPECT().InspectContainer(mock.Anything, "c-1").Return(&domain.Container{
629 ID: "c-1", Status: "running", StartedAt: time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC),
630 }, nil).Twice()
631 runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "c-1").Return("", false, nil).Once()
632
633 svc := recoveringService(t, state, runtime, traffic, nil)
634 err := svc.ReconcileRunning(ctx)
635 require.Error(t, err)
636 assert.Contains(t, err.Error(), "generation changed")
637 assert.Zero(t, traffic.rebuildCount(), "stale recovery evidence must not publish")
638}
639
640func TestReconcileRunning_LogReadinessUsesTheExactExecutionScope(t *testing.T) {
641 ctx := context.Background()
642 state, runtime, _, _ := mockDeps(t)
643 traffic := &recordingTraffic{}
644 spec := headlessSpec()
645 spec.Readiness = domain.AppReadiness{Type: domain.AppReadinessLog, Contains: "ready", Timeout: time.Second}
646
647 active := domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
648 "web": {Container: "c-1", EffectiveRevision: "rev-1", Spec: spec},
649 }}
650 startedAt := time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC)
651 state.EXPECT().ListApps(mock.Anything).Return([]string{"blog"}, nil).Once()
652 state.EXPECT().LoadIntent(mock.Anything, "blog").Return(domain.AppStopIntent{App: "blog"}, nil).Once()
653 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil)
654 state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil).Once()
655 runtime.EXPECT().InspectContainer(mock.Anything, "c-1").
656 Return(&domain.Container{ID: "c-1", Status: "exited", StartedAt: startedAt}, nil).Once()
657 runtime.EXPECT().StartContainer(mock.Anything, "c-1").Return(nil).Once()
658 runtime.EXPECT().InspectContainer(mock.Anything, "c-1").
659 Return(&domain.Container{ID: "c-1", Status: "running", StartedAt: startedAt}, nil).Twice()
660 runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "c-1").Return("", false, nil).Once()
661 var observedID string
662 var observedSince time.Time
663 runtime.EXPECT().GetContainerLogsSince(mock.Anything, "c-1", mock.Anything, false).
664 RunAndReturn(func(_ context.Context, containerID string, since time.Time, _ bool) (io.ReadCloser, error) {
665 observedID = containerID
666 observedSince = since
667 return io.NopCloser(strings.NewReader("boot\nready\n")), nil
668 }).Once()
669
670 svc := recoveringService(t, state, runtime, traffic, nil)
671 require.NoError(t, svc.ReconcileRunning(ctx))
672 assert.Equal(t, "c-1", observedID, "readiness reads the exact ACTIVE container")
673 assert.Equal(t, startedAt, observedSince, "readiness is scoped to the current execution start")
674 assert.Equal(t, 1, traffic.rebuildCount())
675}
676
677func TestReconcileRunning_OldExecutionMarkerCannotSatisfyReadiness(t *testing.T) {
678 ctx := context.Background()
679 state, runtime, _, _ := mockDeps(t)
680 traffic := &recordingTraffic{}
681 spec := headlessSpec()
682 spec.Readiness = domain.AppReadiness{Type: domain.AppReadinessLog, Contains: "ready", Timeout: 20 * time.Millisecond}
683
684 active := domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
685 "web": {Container: "c-1", EffectiveRevision: "rev-1", Spec: spec},
686 }}
687 startedAt := time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC)
688 state.EXPECT().ListApps(mock.Anything).Return([]string{"blog"}, nil).Once()
689 state.EXPECT().LoadIntent(mock.Anything, "blog").Return(domain.AppStopIntent{App: "blog"}, nil).Once()
690 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil)
691 state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil).Once()
692 runtime.EXPECT().InspectContainer(mock.Anything, "c-1").
693 Return(&domain.Container{ID: "c-1", Status: "exited", StartedAt: startedAt}, nil).Once()
694 runtime.EXPECT().StartContainer(mock.Anything, "c-1").Return(nil).Once()
695 runtime.EXPECT().InspectContainer(mock.Anything, "c-1").
696 Return(&domain.Container{ID: "c-1", Status: "running", StartedAt: startedAt}, nil).Once()
697 runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "c-1").Return("", false, nil).Once()
698 // The runtime only returns the current execution's logs; the old
699 // marker is simply not part of the stream.
700 runtime.EXPECT().GetContainerLogsSince(mock.Anything, "c-1", startedAt, false).
701 Return(io.NopCloser(strings.NewReader("booting\n")), nil)
702
703 svc := recoveringService(t, state, runtime, traffic, nil)
704 err := svc.ReconcileRunning(ctx)
705 require.Error(t, err)
706 assert.Contains(t, err.Error(), "log readiness timeout")
707 runtime.AssertNotCalled(t, "RestartContainer", mock.Anything, mock.Anything, mock.Anything)
708}
709
710// TestReconcileRunning_CrashLoopIsChargedEvenWhenStartsSucceed proves the
711// crash-loop budget cannot be escaped by starts that succeed and then die
712// before the next pass: after three unconfirmed interventions the
713// generation is delayed instead of restarted every 15 seconds.
714func TestReconcileRunning_CrashLoopIsChargedEvenWhenStartsSucceed(t *testing.T) {
715 ctx := context.Background()
716 state, runtime, _, _ := mockDeps(t)
717 traffic := &recordingTraffic{}
718
719 active := domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
720 "web": {Container: "c-1", EffectiveRevision: "rev-1", Spec: headlessSpec()},
721 }}
722 state.EXPECT().ListApps(mock.Anything).Return([]string{"blog"}, nil)
723 state.EXPECT().LoadIntent(mock.Anything, "blog").Return(domain.AppStopIntent{App: "blog"}, nil)
724 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil)
725 state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil)
726 // Every pass finds the exact ACTIVE ID exited again.
727 started := time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC)
728 runtime.EXPECT().InspectContainer(mock.Anything, "c-1").Return(&domain.Container{
729 ID: "c-1", Status: "exited", StartedAt: started,
730 }, nil)
731 runtime.EXPECT().StartContainer(mock.Anything, "c-1").Return(nil).Times(3)
732 runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "c-1").Return("", false, nil)
733
734 svc := recoveringService(t, state, runtime, traffic, nil)
735 for pass := 1; pass <= 4; pass++ {
736 require.NoError(t, svc.ReconcileRunning(ctx), "pass %d", pass)
737 }
738 runtime.AssertNumberOfCalls(t, "StartContainer", 3)
739}
740
741// TestReconcileRunning_StaleExecutionAfterReadinessNeverPublishes proves
742// the last revalidation: a container that restarted while this pass
743// waited for readiness must not republish the previous execution's binds.
744func TestReconcileRunning_StaleExecutionAfterReadinessNeverPublishes(t *testing.T) {
745 ctx := context.Background()
746 state, runtime, _, _ := mockDeps(t)
747 traffic := &recordingTraffic{}
748
749 active := domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
750 "web": {Container: "c-1", EffectiveRevision: "rev-1", Spec: headlessSpec()},
751 }}
752 state.EXPECT().ListApps(mock.Anything).Return([]string{"blog"}, nil).Once()
753 state.EXPECT().LoadIntent(mock.Anything, "blog").Return(domain.AppStopIntent{App: "blog"}, nil).Once()
754 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil).Once()
755 state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil).Once()
756 firstExecution := time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC)
757 runtime.EXPECT().InspectContainer(mock.Anything, "c-1").
758 Return(&domain.Container{ID: "c-1", Status: "exited", StartedAt: firstExecution}, nil).Once()
759 runtime.EXPECT().StartContainer(mock.Anything, "c-1").Return(nil).Once()
760 runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "c-1").Return("", false, nil).Once()
761 // The workload restarted while the pass was verifying it.
762 runtime.EXPECT().InspectContainer(mock.Anything, "c-1").
763 Return(&domain.Container{ID: "c-1", Status: "running", StartedAt: firstExecution.Add(time.Minute)}, nil).Once()
764
765 svc := recoveringService(t, state, runtime, traffic, nil)
766 err := svc.ReconcileRunning(ctx)
767 require.Error(t, err)
768 assert.Contains(t, err.Error(), "restarted during recovery")
769 assert.Zero(t, traffic.rebuildCount(), "stale evidence must not publish")
770 state.AssertNotCalled(t, "SaveActive", mock.Anything, mock.Anything)
771}
772
773// TestReconcileRunning_FailedReadinessPersistsNoBinds proves the ordering
774// the plan requires: binds reach ACTIVE only after readiness passed, so a
775// failed verification cannot be projected by any other publication.
776func TestReconcileRunning_FailedReadinessPersistsNoBinds(t *testing.T) {
777 ctx := context.Background()
778 state, runtime, _, _ := mockDeps(t)
779 traffic := &recordingTraffic{}
780 spec := webService()
781 spec.Readiness.Timeout = 20 * time.Millisecond
782
783 active := domain.AppActive{App: "blog", Services: map[string]domain.AppEffectiveService{
784 "web": {Container: "c-1", EffectiveRevision: "rev-1", Spec: spec},
785 }}
786 state.EXPECT().ListApps(mock.Anything).Return([]string{"blog"}, nil).Once()
787 state.EXPECT().LoadIntent(mock.Anything, "blog").Return(domain.AppStopIntent{App: "blog"}, nil).Once()
788 state.EXPECT().LoadActive(mock.Anything, "blog").Return(active, true, nil).Once()
789 state.EXPECT().LoadRecoveryInhibitions(mock.Anything, "blog").Return(nil, nil).Once()
790 startedAt := time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC)
791 runtime.EXPECT().InspectContainer(mock.Anything, "c-1").
792 Return(&domain.Container{ID: "c-1", Status: "exited", StartedAt: startedAt}, nil).Once()
793 runtime.EXPECT().StartContainer(mock.Anything, "c-1").Return(nil).Once()
794 runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "c-1").Return("", false, nil).Once()
795 runtime.EXPECT().GetContainerBackendBinds(mock.Anything, "c-1", mock.Anything).
796 Return([]domain.ContainerBackendBind{{ContainerPort: 8080, HostPort: 32771, Protocol: domain.NetworkProtocolTCP}}, nil).Once()
797 state.EXPECT().RegisterBackendBinds(mock.Anything, mock.Anything).Return(nil).Once()
798
799 svc := recoveringService(t, state, runtime, traffic, func(context.Context, string, string) (int, error) {
800 return 500, nil
801 })
802 require.Error(t, svc.ReconcileRunning(ctx))
803 state.AssertNotCalled(t, "SaveActive", mock.Anything, mock.Anything)
804 assert.Zero(t, traffic.rebuildCount())
805 runtime.AssertNotCalled(t, "RestartContainer", mock.Anything, mock.Anything, mock.Anything)
806}