1package container
2
3import (
4 "context"
5 "testing"
6 "time"
7
8 "github.com/bnema/zerowrap"
9 "github.com/stretchr/testify/assert"
10 "github.com/stretchr/testify/mock"
11
12 "github.com/bnema/gordon/internal/boundaries/out/mocks"
13 "github.com/bnema/gordon/internal/domain"
14)
15
16func newTestService(runtime *mocks.MockContainerRuntime) *Service {
17 return &Service{
18 runtime: runtime,
19 config: Config{},
20 containers: make(map[string]*domain.Container),
21 attachments: make(map[string][]string),
22 }
23}
24
25func monitorTestContext() context.Context {
26 return zerowrap.WithCtx(context.Background(), zerowrap.Default())
27}
28
29func TestMonitor_RestartsCrashedContainer(t *testing.T) {
30 runtime := mocks.NewMockContainerRuntime(t)
31 svc := newTestService(runtime)
32
33 // Track a container.
34 svc.containers["app.example.com"] = &domain.Container{
35 ID: "ctr-1",
36 Image: "myapp:latest",
37 }
38
39 // Inspect returns exited with non-zero exit code.
40 runtime.EXPECT().InspectContainer(mock.Anything, "ctr-1").Return(&domain.Container{
41 ID: "ctr-1",
42 Status: string(domain.ContainerStatusExited),
43 ExitCode: 1,
44 }, nil)
45
46 // Monitor should restart it.
47 runtime.EXPECT().StartContainer(mock.Anything, "ctr-1").Return(nil)
48
49 m := newMonitor(svc)
50 m.check(monitorTestContext())
51
52 runtime.AssertExpectations(t)
53}
54
55func TestMonitor_SkipsGracefulExit(t *testing.T) {
56 runtime := mocks.NewMockContainerRuntime(t)
57 svc := newTestService(runtime)
58
59 svc.containers["app.example.com"] = &domain.Container{
60 ID: "ctr-1",
61 Image: "myapp:latest",
62 }
63
64 // Exited with code 0 — graceful stop.
65 runtime.EXPECT().InspectContainer(mock.Anything, "ctr-1").Return(&domain.Container{
66 ID: "ctr-1",
67 Status: string(domain.ContainerStatusExited),
68 ExitCode: 0,
69 }, nil)
70
71 // Should NOT call StartContainer.
72
73 m := newMonitor(svc)
74 m.check(monitorTestContext())
75
76 runtime.AssertExpectations(t)
77 runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
78}
79
80func TestMonitor_SkipsRunningContainer(t *testing.T) {
81 runtime := mocks.NewMockContainerRuntime(t)
82 svc := newTestService(runtime)
83
84 svc.containers["app.example.com"] = &domain.Container{
85 ID: "ctr-1",
86 Image: "myapp:latest",
87 }
88
89 runtime.EXPECT().InspectContainer(mock.Anything, "ctr-1").Return(&domain.Container{
90 ID: "ctr-1",
91 Status: string(domain.ContainerStatusRunning),
92 }, nil)
93
94 // Running container — check health status.
95 runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "ctr-1").Return("healthy", true, nil)
96
97 m := newMonitor(svc)
98 m.check(monitorTestContext())
99
100 runtime.AssertExpectations(t)
101 runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
102}
103
104func TestMonitor_RestartsUnhealthyContainer(t *testing.T) {
105 runtime := mocks.NewMockContainerRuntime(t)
106 svc := newTestService(runtime)
107
108 svc.containers["app.example.com"] = &domain.Container{
109 ID: "ctr-1",
110 Image: "myapp:latest",
111 }
112
113 runtime.EXPECT().InspectContainer(mock.Anything, "ctr-1").Return(&domain.Container{
114 ID: "ctr-1",
115 Status: string(domain.ContainerStatusRunning),
116 }, nil)
117
118 // Running but Docker health says unhealthy.
119 runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "ctr-1").Return("unhealthy", true, nil)
120
121 // Should restart the unhealthy container.
122 runtime.EXPECT().RestartContainer(mock.Anything, "ctr-1").Return(nil)
123
124 m := newMonitor(svc)
125 m.check(monitorTestContext())
126
127 runtime.AssertExpectations(t)
128}
129
130func TestMonitor_SkipsUnhealthyRestartWhenContainerWasUntracked(t *testing.T) {
131 runtime := mocks.NewMockContainerRuntime(t)
132 svc := newTestService(runtime)
133
134 tracked := &domain.Container{ID: "ctr-1", Image: "myapp:latest"}
135 svc.containers["app.example.com"] = tracked
136
137 runtime.EXPECT().InspectContainer(mock.Anything, "ctr-1").Run(func(context.Context, string) {
138 svc.mu.Lock()
139 delete(svc.containers, "app.example.com")
140 svc.mu.Unlock()
141 }).Return(&domain.Container{
142 ID: "ctr-1",
143 Status: string(domain.ContainerStatusRunning),
144 }, nil)
145 runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "ctr-1").Return("unhealthy", true, nil)
146
147 m := newMonitor(svc)
148 m.check(monitorTestContext())
149
150 runtime.AssertNotCalled(t, "RestartContainer", mock.Anything, "ctr-1")
151}
152
153func TestMonitor_CrashLoopBackoff(t *testing.T) {
154 runtime := mocks.NewMockContainerRuntime(t)
155 svc := newTestService(runtime)
156
157 svc.containers["app.example.com"] = &domain.Container{
158 ID: "ctr-1",
159 Image: "myapp:latest",
160 }
161
162 m := newMonitor(svc)
163 ctx := monitorTestContext()
164 now := time.Now()
165
166 // Simulate 3 crashes within the crash loop window.
167 m.mu.Lock()
168 m.history["app.example.com"] = &restartRecord{
169 attempts: []time.Time{now.Add(-2 * time.Minute), now.Add(-1 * time.Minute)},
170 consecutive: 2,
171 }
172 m.mu.Unlock()
173
174 // Third crash — this should trigger backoff.
175 runtime.EXPECT().InspectContainer(mock.Anything, "ctr-1").Return(&domain.Container{
176 ID: "ctr-1",
177 Status: string(domain.ContainerStatusExited),
178 ExitCode: 137,
179 }, nil)
180
181 // Should NOT call StartContainer because crash loop is detected.
182 m.check(ctx)
183
184 runtime.AssertExpectations(t)
185 runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
186
187 // Verify backoff was set.
188 m.mu.Lock()
189 rec := m.history["app.example.com"]
190 assert.False(t, rec.backoffEnd.IsZero(), "backoff should be set after crash loop detection")
191 assert.True(t, rec.backoffEnd.After(time.Now()), "backoff end should be in the future")
192 m.mu.Unlock()
193}
194
195func TestMonitor_BackoffPreventsRestart(t *testing.T) {
196 runtime := mocks.NewMockContainerRuntime(t)
197 svc := newTestService(runtime)
198
199 svc.containers["app.example.com"] = &domain.Container{
200 ID: "ctr-1",
201 Image: "myapp:latest",
202 }
203
204 m := newMonitor(svc)
205
206 // Set backoff into the future.
207 m.mu.Lock()
208 m.history["app.example.com"] = &restartRecord{
209 backoffEnd: time.Now().Add(5 * time.Minute),
210 consecutive: 3,
211 }
212 m.mu.Unlock()
213
214 // Container is crashed.
215 runtime.EXPECT().InspectContainer(mock.Anything, "ctr-1").Return(&domain.Container{
216 ID: "ctr-1",
217 Status: string(domain.ContainerStatusExited),
218 ExitCode: 1,
219 }, nil)
220
221 // Should NOT restart — in backoff.
222 m.check(monitorTestContext())
223
224 runtime.AssertExpectations(t)
225 runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
226}
227
228func TestMonitor_ClearsCrashHistoryAfterStableRunning(t *testing.T) {
229 runtime := mocks.NewMockContainerRuntime(t)
230 svc := newTestService(runtime)
231
232 svc.containers["app.example.com"] = &domain.Container{
233 ID: "ctr-1",
234 Image: "myapp:latest",
235 }
236
237 m := newMonitor(svc)
238
239 // Container has been running for longer than stableRunningDuration.
240 m.mu.Lock()
241 m.history["app.example.com"] = &restartRecord{
242 attempts: []time.Time{time.Now().Add(-10 * time.Minute)},
243 consecutive: 2,
244 lastSeen: time.Now().Add(-6 * time.Minute), // 6 min ago > stableRunningDuration (5 min)
245 }
246 m.mu.Unlock()
247
248 runtime.EXPECT().InspectContainer(mock.Anything, "ctr-1").Return(&domain.Container{
249 ID: "ctr-1",
250 Status: string(domain.ContainerStatusRunning),
251 }, nil)
252
253 runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "ctr-1").Return("healthy", true, nil)
254
255 m.check(monitorTestContext())
256
257 // History should be cleared.
258 m.mu.Lock()
259 _, exists := m.history["app.example.com"]
260 m.mu.Unlock()
261 assert.False(t, exists, "crash history should be cleared after stable running")
262
263 runtime.AssertExpectations(t)
264}
265
266func TestMonitor_SkipsReplacedContainer(t *testing.T) {
267 runtime := mocks.NewMockContainerRuntime(t)
268 svc := newTestService(runtime)
269
270 // Track the OLD container ID.
271 svc.containers["app.example.com"] = &domain.Container{
272 ID: "ctr-NEW",
273 Image: "myapp:latest",
274 }
275
276 m := newMonitor(svc)
277
278 // Monitor snapshot captured "ctr-OLD" but a deploy replaced it with "ctr-NEW".
279 // We simulate this by directly calling checkContainer with the old ID.
280 ctx := monitorTestContext()
281 log := zerowrap.FromCtx(ctx)
282
283 // Inspect the old container — it's exited.
284 runtime.EXPECT().InspectContainer(mock.Anything, "ctr-OLD").Return(&domain.Container{
285 ID: "ctr-OLD",
286 Status: string(domain.ContainerStatusExited),
287 ExitCode: 1,
288 }, nil)
289
290 // Should NOT restart — tracked container has been replaced.
291 oldTracked := &domain.Container{ID: "ctr-OLD", Image: "myapp:latest"}
292 m.checkContainer(ctx, log, "app.example.com", oldTracked, time.Now())
293
294 runtime.AssertExpectations(t)
295 runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
296}
297
298func TestMonitor_ComputeBackoff(t *testing.T) {
299 m := &Monitor{}
300
301 tests := []struct {
302 consecutive int
303 expected time.Duration
304 }{
305 {3, 1 * time.Minute}, // first backoff
306 {4, 2 * time.Minute}, // second
307 {5, 4 * time.Minute}, // third
308 {6, 8 * time.Minute}, // fourth
309 {7, 15 * time.Minute}, // capped
310 {10, 15 * time.Minute}, // still capped
311 }
312
313 for _, tt := range tests {
314 got := m.computeBackoff(tt.consecutive)
315 assert.Equal(t, tt.expected, got, "consecutive=%d", tt.consecutive)
316 }
317}
318
319func TestMonitor_StartStop(t *testing.T) {
320 runtime := mocks.NewMockContainerRuntime(t)
321 svc := newTestService(runtime)
322
323 m := newMonitor(svc)
324 m.interval = 50 * time.Millisecond // fast ticking for test
325
326 ctx, cancel := context.WithCancel(monitorTestContext())
327 defer cancel()
328
329 m.Start(ctx)
330
331 // Let it tick a couple times.
332 time.Sleep(150 * time.Millisecond)
333
334 // Stop should return promptly.
335 done := make(chan struct{})
336 go func() {
337 m.Stop()
338 close(done)
339 }()
340
341 select {
342 case <-done:
343 // OK
344 case <-time.After(2 * time.Second):
345 t.Fatal("monitor.Stop() did not return within 2 seconds")
346 }
347}