1package container
2
3import (
4 "context"
5 "testing"
6 "time"
7
8 "github.com/bnema/zerowrap"
9 "github.com/stretchr/testify/assert"
10 "github.com/stretchr/testify/mock"
11
12 "github.com/bnema/gordon/internal/boundaries/out/mocks"
13 "github.com/bnema/gordon/internal/domain"
14)
15
16func newTestService(runtime *mocks.MockContainerRuntime) *Service {
17 return &Service{
18 runtime: runtime,
19 config: Config{},
20 containers: make(map[string]*domain.Container),
21 attachments: make(map[string][]string),
22 }
23}
24
25func monitorTestContext() context.Context {
26 return zerowrap.WithCtx(context.Background(), zerowrap.Default())
27}
28
29func TestMonitor_RestartsCrashedContainer(t *testing.T) {
30 runtime := mocks.NewMockContainerRuntime(t)
31 svc := newTestService(runtime)
32
33 // Track a container.
34 svc.containers["app.example.com"] = &domain.Container{
35 ID: "ctr-1",
36 Image: "myapp:latest",
37 }
38
39 // Inspect returns exited with non-zero exit code.
40 runtime.EXPECT().InspectContainer(mock.Anything, "ctr-1").Return(&domain.Container{
41 ID: "ctr-1",
42 Status: string(domain.ContainerStatusExited),
43 ExitCode: 1,
44 }, nil)
45
46 // Monitor should restart it.
47 runtime.EXPECT().StartContainer(mock.Anything, "ctr-1").Return(nil)
48
49 m := newMonitor(svc)
50 m.check(monitorTestContext())
51
52 runtime.AssertExpectations(t)
53}
54
55func TestMonitor_SkipsGracefulExit(t *testing.T) {
56 runtime := mocks.NewMockContainerRuntime(t)
57 svc := newTestService(runtime)
58
59 svc.containers["app.example.com"] = &domain.Container{
60 ID: "ctr-1",
61 Image: "myapp:latest",
62 }
63
64 // Exited with code 0 — graceful stop.
65 runtime.EXPECT().InspectContainer(mock.Anything, "ctr-1").Return(&domain.Container{
66 ID: "ctr-1",
67 Status: string(domain.ContainerStatusExited),
68 ExitCode: 0,
69 }, nil)
70
71 // Should NOT call StartContainer.
72
73 m := newMonitor(svc)
74 m.check(monitorTestContext())
75
76 runtime.AssertExpectations(t)
77 runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
78}
79
80func TestMonitor_SkipsRunningContainer(t *testing.T) {
81 runtime := mocks.NewMockContainerRuntime(t)
82 svc := newTestService(runtime)
83
84 svc.containers["app.example.com"] = &domain.Container{
85 ID: "ctr-1",
86 Image: "myapp:latest",
87 }
88
89 runtime.EXPECT().InspectContainer(mock.Anything, "ctr-1").Return(&domain.Container{
90 ID: "ctr-1",
91 Status: string(domain.ContainerStatusRunning),
92 }, nil)
93
94 // Running container — check health status.
95 runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "ctr-1").Return("healthy", true, nil)
96
97 m := newMonitor(svc)
98 m.check(monitorTestContext())
99
100 runtime.AssertExpectations(t)
101 runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
102}
103
104func TestMonitor_RestartsUnhealthyContainer(t *testing.T) {
105 runtime := mocks.NewMockContainerRuntime(t)
106 svc := newTestService(runtime)
107
108 svc.containers["app.example.com"] = &domain.Container{
109 ID: "ctr-1",
110 Image: "myapp:latest",
111 }
112
113 runtime.EXPECT().InspectContainer(mock.Anything, "ctr-1").Return(&domain.Container{
114 ID: "ctr-1",
115 Status: string(domain.ContainerStatusRunning),
116 }, nil)
117
118 // Running but Docker health says unhealthy.
119 runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "ctr-1").Return("unhealthy", true, nil)
120
121 // Should restart the unhealthy container.
122 runtime.EXPECT().RestartContainer(mock.Anything, "ctr-1").Return(nil)
123
124 m := newMonitor(svc)
125 m.check(monitorTestContext())
126
127 runtime.AssertExpectations(t)
128}
129
130func TestMonitor_CrashLoopBackoff(t *testing.T) {
131 runtime := mocks.NewMockContainerRuntime(t)
132 svc := newTestService(runtime)
133
134 svc.containers["app.example.com"] = &domain.Container{
135 ID: "ctr-1",
136 Image: "myapp:latest",
137 }
138
139 m := newMonitor(svc)
140 ctx := monitorTestContext()
141 now := time.Now()
142
143 // Simulate 3 crashes within the crash loop window.
144 m.mu.Lock()
145 m.history["app.example.com"] = &restartRecord{
146 attempts: []time.Time{now.Add(-2 * time.Minute), now.Add(-1 * time.Minute)},
147 consecutive: 2,
148 }
149 m.mu.Unlock()
150
151 // Third crash — this should trigger backoff.
152 runtime.EXPECT().InspectContainer(mock.Anything, "ctr-1").Return(&domain.Container{
153 ID: "ctr-1",
154 Status: string(domain.ContainerStatusExited),
155 ExitCode: 137,
156 }, nil)
157
158 // Should NOT call StartContainer because crash loop is detected.
159 m.check(ctx)
160
161 runtime.AssertExpectations(t)
162 runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
163
164 // Verify backoff was set.
165 m.mu.Lock()
166 rec := m.history["app.example.com"]
167 assert.False(t, rec.backoffEnd.IsZero(), "backoff should be set after crash loop detection")
168 assert.True(t, rec.backoffEnd.After(time.Now()), "backoff end should be in the future")
169 m.mu.Unlock()
170}
171
172func TestMonitor_BackoffPreventsRestart(t *testing.T) {
173 runtime := mocks.NewMockContainerRuntime(t)
174 svc := newTestService(runtime)
175
176 svc.containers["app.example.com"] = &domain.Container{
177 ID: "ctr-1",
178 Image: "myapp:latest",
179 }
180
181 m := newMonitor(svc)
182
183 // Set backoff into the future.
184 m.mu.Lock()
185 m.history["app.example.com"] = &restartRecord{
186 backoffEnd: time.Now().Add(5 * time.Minute),
187 consecutive: 3,
188 }
189 m.mu.Unlock()
190
191 // Container is crashed.
192 runtime.EXPECT().InspectContainer(mock.Anything, "ctr-1").Return(&domain.Container{
193 ID: "ctr-1",
194 Status: string(domain.ContainerStatusExited),
195 ExitCode: 1,
196 }, nil)
197
198 // Should NOT restart — in backoff.
199 m.check(monitorTestContext())
200
201 runtime.AssertExpectations(t)
202 runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
203}
204
205func TestMonitor_ClearsCrashHistoryAfterStableRunning(t *testing.T) {
206 runtime := mocks.NewMockContainerRuntime(t)
207 svc := newTestService(runtime)
208
209 svc.containers["app.example.com"] = &domain.Container{
210 ID: "ctr-1",
211 Image: "myapp:latest",
212 }
213
214 m := newMonitor(svc)
215
216 // Container has been running for longer than stableRunningDuration.
217 m.mu.Lock()
218 m.history["app.example.com"] = &restartRecord{
219 attempts: []time.Time{time.Now().Add(-10 * time.Minute)},
220 consecutive: 2,
221 lastSeen: time.Now().Add(-6 * time.Minute), // 6 min ago > stableRunningDuration (5 min)
222 }
223 m.mu.Unlock()
224
225 runtime.EXPECT().InspectContainer(mock.Anything, "ctr-1").Return(&domain.Container{
226 ID: "ctr-1",
227 Status: string(domain.ContainerStatusRunning),
228 }, nil)
229
230 runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "ctr-1").Return("healthy", true, nil)
231
232 m.check(monitorTestContext())
233
234 // History should be cleared.
235 m.mu.Lock()
236 _, exists := m.history["app.example.com"]
237 m.mu.Unlock()
238 assert.False(t, exists, "crash history should be cleared after stable running")
239
240 runtime.AssertExpectations(t)
241}
242
243func TestMonitor_SkipsReplacedContainer(t *testing.T) {
244 runtime := mocks.NewMockContainerRuntime(t)
245 svc := newTestService(runtime)
246
247 // Track the OLD container ID.
248 svc.containers["app.example.com"] = &domain.Container{
249 ID: "ctr-NEW",
250 Image: "myapp:latest",
251 }
252
253 m := newMonitor(svc)
254
255 // Monitor snapshot captured "ctr-OLD" but a deploy replaced it with "ctr-NEW".
256 // We simulate this by directly calling checkContainer with the old ID.
257 ctx := monitorTestContext()
258 log := zerowrap.FromCtx(ctx)
259
260 // Inspect the old container — it's exited.
261 runtime.EXPECT().InspectContainer(mock.Anything, "ctr-OLD").Return(&domain.Container{
262 ID: "ctr-OLD",
263 Status: string(domain.ContainerStatusExited),
264 ExitCode: 1,
265 }, nil)
266
267 // Should NOT restart — tracked container has been replaced.
268 oldTracked := &domain.Container{ID: "ctr-OLD", Image: "myapp:latest"}
269 m.checkContainer(ctx, log, "app.example.com", oldTracked, time.Now())
270
271 runtime.AssertExpectations(t)
272 runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
273}
274
275func TestMonitor_ComputeBackoff(t *testing.T) {
276 m := &Monitor{}
277
278 tests := []struct {
279 consecutive int
280 expected time.Duration
281 }{
282 {3, 1 * time.Minute}, // first backoff
283 {4, 2 * time.Minute}, // second
284 {5, 4 * time.Minute}, // third
285 {6, 8 * time.Minute}, // fourth
286 {7, 15 * time.Minute}, // capped
287 {10, 15 * time.Minute}, // still capped
288 }
289
290 for _, tt := range tests {
291 got := m.computeBackoff(tt.consecutive)
292 assert.Equal(t, tt.expected, got, "consecutive=%d", tt.consecutive)
293 }
294}
295
296func TestMonitor_StartStop(t *testing.T) {
297 runtime := mocks.NewMockContainerRuntime(t)
298 svc := newTestService(runtime)
299
300 m := newMonitor(svc)
301 m.interval = 50 * time.Millisecond // fast ticking for test
302
303 ctx, cancel := context.WithCancel(monitorTestContext())
304 defer cancel()
305
306 m.Start(ctx)
307
308 // Let it tick a couple times.
309 time.Sleep(150 * time.Millisecond)
310
311 // Stop should return promptly.
312 done := make(chan struct{})
313 go func() {
314 m.Stop()
315 close(done)
316 }()
317
318 select {
319 case <-done:
320 // OK
321 case <-time.After(2 * time.Second):
322 t.Fatal("monitor.Stop() did not return within 2 seconds")
323 }
324}