1package deployment
2
3import (
4 "context"
5 "io"
6 "strings"
7 "sync"
8 "testing"
9 "time"
10
11 "github.com/stretchr/testify/assert"
12 "github.com/stretchr/testify/require"
13
14 "github.com/bnema/gordon/internal/domain"
15)
16
17func TestAppCoordinator_SerializesOneApp(t *testing.T) {
18 coord := newAppCoordinator()
19 release, err := coord.acquire(context.Background(), "blog")
20 require.NoError(t, err)
21
22 acquired := make(chan struct{})
23 go func() {
24 release2, err := coord.acquire(context.Background(), "blog")
25 if err == nil {
26 close(acquired)
27 release2()
28 }
29 }()
30
31 select {
32 case <-acquired:
33 t.Fatal("second acquisition ran while the app lock was held")
34 case <-time.After(50 * time.Millisecond):
35 }
36 release()
37 select {
38 case <-acquired:
39 case <-time.After(time.Second):
40 t.Fatal("second acquisition never ran after release")
41 }
42}
43
44func TestAppCoordinator_AllowsParallelApps(t *testing.T) {
45 coord := newAppCoordinator()
46 releaseBlog, err := coord.acquire(context.Background(), "blog")
47 require.NoError(t, err)
48 defer releaseBlog()
49
50 releaseWiki, err := coord.acquire(context.Background(), "wiki")
51 require.NoError(t, err)
52 defer releaseWiki()
53
54 // Distinct apps must not block each other.
55 _, ok := coord.tryAcquire("other")
56 assert.True(t, ok)
57}
58
59func TestAppCoordinator_TryAcquireSkipsBusyApp(t *testing.T) {
60 coord := newAppCoordinator()
61 release, ok := coord.tryAcquire("blog")
62 require.True(t, ok)
63
64 _, ok = coord.tryAcquire("blog")
65 assert.False(t, ok, "a periodic pass must skip a busy app instead of waiting")
66
67 release()
68 _, ok = coord.tryAcquire("blog")
69 assert.True(t, ok, "the lock is reusable after release")
70}
71
72func TestAppCoordinator_HonorsCancellation(t *testing.T) {
73 coord := newAppCoordinator()
74 release, ok := coord.tryAcquire("blog")
75 require.True(t, ok)
76
77 ctx, cancel := context.WithCancel(context.Background())
78 cancel()
79 _, err := coord.acquire(ctx, "blog")
80 require.ErrorIs(t, err, context.Canceled)
81 release()
82}
83
84func TestAppCoordinator_ConcurrentUseIsRaceFree(t *testing.T) {
85 coord := newAppCoordinator()
86 var wg sync.WaitGroup
87 var counter int64
88 var mu sync.Mutex
89 for range 8 {
90 wg.Add(1)
91 go func() {
92 defer wg.Done()
93 for range 50 {
94 release, err := coord.acquire(context.Background(), "blog")
95 if err != nil {
96 return
97 }
98 mu.Lock()
99 counter++
100 mu.Unlock()
101 release()
102 }
103 }()
104 }
105 wg.Wait()
106 assert.Equal(t, int64(400), counter)
107}
108
109func TestRecoveryBackoff_ThreeAttemptsThenEscalatingDelays(t *testing.T) {
110 now := time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC)
111 backoff := newRecoveryBackoff(func() time.Time { return now })
112 key := recoveryKey{app: "blog", service: "web", container: "c-1"}
113
114 for attempt := 1; attempt <= 3; attempt++ {
115 assert.True(t, backoff.allow(key), "attempt %d must be permitted", attempt)
116 backoff.recordFailure(key)
117 }
118 assert.False(t, backoff.allow(key), "the fourth attempt is delayed by one minute")
119
120 expected := []time.Duration{time.Minute, 2 * time.Minute, 4 * time.Minute, 8 * time.Minute, 15 * time.Minute}
121 for i, delay := range expected {
122 now = now.Add(delay)
123 assert.True(t, backoff.allow(key), "delay %d must expire", i)
124 backoff.recordFailure(key)
125 assert.False(t, backoff.allow(key))
126 }
127 // Further failures stay capped at fifteen minutes.
128 now = now.Add(15 * time.Minute)
129 assert.True(t, backoff.allow(key))
130 backoff.recordFailure(key)
131 now = now.Add(14 * time.Minute)
132 assert.False(t, backoff.allow(key), "the cap holds at fifteen minutes")
133 now = now.Add(time.Minute)
134 assert.True(t, backoff.allow(key))
135}
136
137func TestRecoveryBackoff_EscalationSurvivesWithoutStableObservation(t *testing.T) {
138 now := time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC)
139 backoff := newRecoveryBackoff(func() time.Time { return now })
140 key := recoveryKey{app: "blog", service: "web", container: "c-1"}
141
142 for range 3 {
143 backoff.recordFailure(key)
144 }
145 require.False(t, backoff.allow(key))
146
147 // A long idle period permits the delayed attempt, but the escalation
148 // itself only resets after five continuously stable minutes.
149 now = now.Add(30 * time.Minute)
150 require.True(t, backoff.allow(key))
151 backoff.recordFailure(key)
152 assert.False(t, backoff.allow(key), "the failure budget is not escaped by waiting")
153}
154
155func TestRecoveryBackoff_StableObservationResetsHistory(t *testing.T) {
156 now := time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC)
157 backoff := newRecoveryBackoff(func() time.Time { return now })
158 key := recoveryKey{app: "blog", service: "web", container: "c-1"}
159
160 for range 3 {
161 backoff.recordFailure(key)
162 }
163 require.False(t, backoff.allow(key))
164
165 now = now.Add(30 * time.Second)
166 backoff.observeStable(key)
167 assert.Equal(t, 3, recoveredFailures(backoff, key), "one healthy observation is not enough")
168
169 now = now.Add(stableReadyWindow - time.Second)
170 backoff.observeStable(key)
171 assert.Equal(t, 3, recoveredFailures(backoff, key), "history resets only after five stable minutes")
172
173 now = now.Add(2 * time.Second)
174 backoff.observeStable(key)
175 assert.Equal(t, 0, recoveredFailures(backoff, key), "five stable minutes reset the failure history")
176 assert.True(t, backoff.allow(key))
177}
178
179// recoveredFailures reports the escalation counter of one generation; 0
180// means the history was dropped.
181func recoveredFailures(backoff *recoveryBackoff, key recoveryKey) int {
182 backoff.mu.Lock()
183 defer backoff.mu.Unlock()
184 entry, ok := backoff.entries[key]
185 if !ok {
186 return 0
187 }
188 return entry.failures
189}
190
191func TestRecoveryBackoff_FailureClearsStability(t *testing.T) {
192 now := time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC)
193 backoff := newRecoveryBackoff(func() time.Time { return now })
194 key := recoveryKey{app: "blog", service: "web", container: "c-1"}
195
196 backoff.observeStable(key)
197 now = now.Add(4 * time.Minute)
198 backoff.recordFailure(key)
199 now = now.Add(2 * time.Minute)
200 backoff.observeStable(key)
201 assert.True(t, backoff.allow(key), "the streak restarted at the failure")
202}
203
204func TestRecoveryBackoff_IsGenerationScoped(t *testing.T) {
205 now := time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC)
206 backoff := newRecoveryBackoff(func() time.Time { return now })
207 failing := recoveryKey{app: "blog", service: "web", container: "c-1"}
208 for range 4 {
209 backoff.recordFailure(failing)
210 }
211 assert.False(t, backoff.allow(failing))
212 assert.True(t, backoff.allow(recoveryKey{app: "blog", service: "web", container: "c-2"}),
213 "a replacement generation starts with a fresh budget")
214 assert.True(t, backoff.allow(recoveryKey{app: "blog", service: "worker", container: "c-1"}))
215 assert.True(t, backoff.allow(recoveryKey{app: "wiki", service: "web", container: "c-1"}))
216}
217
218func TestRecoveryBackoff_ForgetDropsHistory(t *testing.T) {
219 now := time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC)
220 backoff := newRecoveryBackoff(func() time.Time { return now })
221 key := recoveryKey{app: "blog", service: "web", container: "c-1"}
222 for range 4 {
223 backoff.recordFailure(key)
224 }
225 require.False(t, backoff.allow(key))
226 backoff.forget(key)
227 assert.True(t, backoff.allow(key))
228}
229
230func TestRecoveryBackoff_UnhealthyStreak(t *testing.T) {
231 backoff := newRecoveryBackoff(time.Now)
232 key := recoveryKey{app: "blog", service: "web", container: "c-1"}
233
234 assert.Equal(t, 1, backoff.observeUnhealthy(key))
235 assert.Equal(t, 2, backoff.observeUnhealthy(key))
236 backoff.clearUnhealthy(key)
237 assert.Equal(t, 1, backoff.observeUnhealthy(key), "a non-unhealthy observation resets the streak")
238}
239
240func TestPublicationInhibition_MarkAndClear(t *testing.T) {
241 var inhibition publicationInhibition
242 assert.False(t, inhibition.inhibited("blog", "web"))
243 inhibition.mark("blog", "web")
244 assert.True(t, inhibition.inhibited("blog", "web"))
245 assert.False(t, inhibition.inhibited("blog", "worker"))
246 assert.False(t, inhibition.inhibited("wiki", "web"))
247 inhibition.clear("blog", "web")
248 assert.False(t, inhibition.inhibited("blog", "web"))
249 inhibition.clear("blog", "web") // clearing twice is harmless
250}
251
252func TestExecutionTracker_DetectsNewExecution(t *testing.T) {
253 var tracker executionTracker
254 key := recoveryKey{app: "blog", service: "web", container: "c-1"}
255 first := time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC)
256
257 assert.True(t, tracker.isNewExecution(key, first),
258 "an unseen execution must be verified because the runtime may have restarted it while Gordon was absent")
259 tracker.record(key, first)
260 assert.False(t, tracker.isNewExecution(key, first), "a stable execution is not new")
261 assert.True(t, tracker.isNewExecution(key, first.Add(time.Minute)), "a native restart is a new execution")
262 tracker.forget(key)
263 assert.True(t, tracker.isNewExecution(key, first))
264}
265
266func TestRecoveryBackoff_PendingAttemptIsCharged(t *testing.T) {
267 now := time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC)
268 backoff := newRecoveryBackoff(func() time.Time { return now })
269 key := recoveryKey{app: "blog", service: "web", container: "c-1"}
270
271 assert.False(t, backoff.attemptPending(key))
272 backoff.markAttempt(key)
273 assert.True(t, backoff.attemptPending(key))
274 backoff.observeStable(key)
275 assert.True(t, backoff.attemptPending(key),
276 "one stable observation does not confirm an intervention: only five stable minutes do")
277 backoff.observeStable(key)
278 assert.True(t, backoff.attemptPending(key))
279 now = now.Add(stableReadyWindow + time.Second)
280 backoff.now = func() time.Time { return now }
281 backoff.observeStable(key)
282 assert.False(t, backoff.attemptPending(key), "the stability window confirms and resets the generation")
283}
284
285func TestRecoveryBackoff_UnstableObservationInvalidatesTheWindow(t *testing.T) {
286 now := time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC)
287 backoff := newRecoveryBackoff(func() time.Time { return now })
288 key := recoveryKey{app: "blog", service: "web", container: "c-1"}
289
290 for range 3 {
291 backoff.recordFailure(key)
292 }
293 now = now.Add(time.Minute)
294 backoff.observeStable(key)
295 now = now.Add(4 * time.Minute)
296 backoff.invalidateStable(key)
297 now = now.Add(time.Minute)
298 backoff.observeStable(key)
299 assert.Equal(t, 3, recoveredFailures(backoff, key),
300 "an unhealthy or unverifiable observation restarts the five-minute window")
301}
302
303func TestRecoveryBackoff_StableObservationNeverCreatesHistory(t *testing.T) {
304 backoff := newRecoveryBackoff(time.Now)
305 key := recoveryKey{app: "blog", service: "web", container: "c-1"}
306 backoff.observeStable(key)
307 assert.Equal(t, 0, recoveredFailures(backoff, key))
308 backoff.mu.Lock()
309 _, tracked := backoff.entries[key]
310 backoff.mu.Unlock()
311 assert.False(t, tracked, "healthy services must not accumulate monitor state")
312}
313
314func TestPublicationInhibition_RetainsActiveServices(t *testing.T) {
315 var inhibition publicationInhibition
316 inhibition.mark("blog", "web")
317 inhibition.mark("blog", "worker")
318 inhibition.mark("wiki", "web")
319 assert.Equal(t, []string{"web", "worker"}, inhibition.pending("blog"))
320 assert.Equal(t, []string{"web"}, inhibition.pending("wiki"))
321
322 inhibition.retainAppServices("blog", map[string]struct{}{"web": {}})
323
324 assert.Equal(t, []string{"web"}, inhibition.pending("blog"),
325 "an active service remains inhibited until withdrawal succeeds")
326 assert.Equal(t, []string{"web"}, inhibition.pending("wiki"))
327}
328
329// TestWaitLogReady_RejectsAnUnknownExecutionBoundary proves the probe
330// fails closed when the runtime reports no execution start: without a
331// boundary it would scan the container's whole history and could match a
332// marker from a previous execution.
333func TestWaitLogReady_RejectsAnUnknownExecutionBoundary(t *testing.T) {
334 deps := ProbeDeps{
335 containerStart: func(context.Context, string) (time.Time, error) { return time.Time{}, nil },
336 logStream: func(context.Context, string, time.Time) (io.ReadCloser, error) {
337 return io.NopCloser(strings.NewReader("ready\n")), nil
338 },
339 }
340 spec := domain.AppService{Readiness: domain.AppReadiness{Type: domain.AppReadinessLog, Contains: "ready"}}
341 err := waitLogReadyWithDeps(context.Background(), deps, "c-1", spec)
342 require.Error(t, err)
343 assert.Contains(t, err.Error(), "runtime reported none")
344}
345
346// TestWaitLogReady_UsesTheExactContainerAndExecution proves the probe
347// reads the passed container ID and the current execution boundary.
348func TestWaitLogReady_UsesTheExactContainerAndExecution(t *testing.T) {
349 startedAt := time.Date(2026, 9, 10, 12, 0, 0, 123456789, time.UTC)
350 var gotID string
351 var gotSince time.Time
352 deps := ProbeDeps{
353 containerStart: func(context.Context, string) (time.Time, error) { return startedAt, nil },
354 logStream: func(_ context.Context, containerID string, since time.Time) (io.ReadCloser, error) {
355 gotID, gotSince = containerID, since
356 return io.NopCloser(strings.NewReader("2026-09-10T12:00:01Z ready\n")), nil
357 },
358 }
359 spec := domain.AppService{Readiness: domain.AppReadiness{Type: domain.AppReadinessLog, Contains: "ready"}}
360 require.NoError(t, waitLogReadyWithDeps(context.Background(), deps, "c-exact", spec))
361 assert.Equal(t, "c-exact", gotID)
362 assert.Equal(t, startedAt, gotSince)
363}
364
365// TestWaitLogReady_StreamFailureIsActionable proves a transport failure
366// is wrapped with the container it belongs to.
367func TestWaitLogReady_StreamFailureIsActionable(t *testing.T) {
368 deps := ProbeDeps{
369 containerStart: func(context.Context, string) (time.Time, error) { return time.Now(), nil },
370 logStream: func(context.Context, string, time.Time) (io.ReadCloser, error) {
371 return nil, assert.AnError
372 },
373 }
374 spec := domain.AppService{Readiness: domain.AppReadiness{Type: domain.AppReadinessLog, Contains: "ready"}}
375 err := waitLogReadyWithDeps(context.Background(), deps, "c-1", spec)
376 require.Error(t, err)
377 assert.Contains(t, err.Error(), "log readiness stream for c-1")
378 assert.ErrorIs(t, err, assert.AnError)
379}