1package container
2
3import (
4 "context"
5 "sync"
6 "time"
7
8 "github.com/bnema/zerowrap"
9
10 "github.com/bnema/gordon/internal/domain"
11)
12
13const (
14 monitorDefaultInterval = 15 * time.Second
15 crashLoopThreshold = 3
16 crashLoopWindow = 5 * time.Minute
17 backoffCap = 15 * time.Minute
18 stableRunningDuration = 5 * time.Minute
19)
20
21// restartRecord tracks restart attempts for crash loop detection.
22type restartRecord struct {
23 attempts []time.Time
24 backoffEnd time.Time
25 consecutive int
26 lastSeen time.Time // last time the container was seen running
27}
28
29// Monitor watches tracked containers and restarts crashed ones.
30type Monitor struct {
31 service *Service
32 stopCh chan struct{}
33 stopped chan struct{}
34 interval time.Duration
35 mu sync.Mutex
36 history map[string]*restartRecord // keyed by domain
37}
38
39// newMonitor creates a new container monitor.
40func newMonitor(service *Service) *Monitor {
41 return &Monitor{
42 service: service,
43 stopCh: make(chan struct{}),
44 stopped: make(chan struct{}),
45 interval: monitorDefaultInterval,
46 history: make(map[string]*restartRecord),
47 }
48}
49
50// Start begins the background monitoring loop.
51func (m *Monitor) Start(ctx context.Context) {
52 log := zerowrap.FromCtx(ctx)
53 log.Info().Dur("interval", m.interval).Msg("container monitor started")
54
55 go m.run(ctx)
56}
57
58// Stop signals the monitor to stop and waits for it to finish.
59func (m *Monitor) Stop() {
60 close(m.stopCh)
61 <-m.stopped
62}
63
64func (m *Monitor) run(ctx context.Context) {
65 defer close(m.stopped)
66
67 ticker := time.NewTicker(m.interval)
68 defer ticker.Stop()
69
70 for {
71 select {
72 case <-m.stopCh:
73 return
74 case <-ctx.Done():
75 return
76 case <-ticker.C:
77 m.check(ctx)
78 }
79 }
80}
81
82func (m *Monitor) check(ctx context.Context) {
83 log := zerowrap.FromCtx(ctx)
84
85 // Snapshot tracked containers under lock.
86 m.service.mu.RLock()
87 snapshot := make(map[string]*domain.Container, len(m.service.containers))
88 for d, c := range m.service.containers {
89 snapshot[d] = c
90 }
91 m.service.mu.RUnlock()
92
93 now := time.Now()
94
95 for domainName, tracked := range snapshot {
96 m.checkContainer(ctx, log, domainName, tracked, now)
97 }
98}
99
100func (m *Monitor) checkContainer(ctx context.Context, log zerowrap.Logger, domainName string, tracked *domain.Container, now time.Time) {
101 if tracked == nil {
102 return
103 }
104
105 inspected, err := m.service.runtime.InspectContainer(ctx, tracked.ID)
106 if err != nil {
107 log.Debug().Err(err).Str("domain", domainName).Str("container_id", tracked.ID).
108 Msg("monitor: failed to inspect container, may have been replaced by deploy")
109 return
110 }
111
112 switch {
113 case inspected.Status == string(domain.ContainerStatusRunning):
114 m.handleRunning(ctx, log, domainName, tracked.ID, now)
115
116 case inspected.Status == string(domain.ContainerStatusExited):
117 if inspected.ExitCode == 0 {
118 // Graceful stop (exit 0): respect user intent, don't restart.
119 log.Debug().Str("domain", domainName).Msg("monitor: container exited gracefully (code 0), skipping")
120 return
121 }
122 // Crashed (non-zero exit): restart if not in backoff.
123 m.handleCrash(ctx, log, domainName, tracked.ID, inspected.ExitCode, now)
124 }
125}
126
127func (m *Monitor) handleRunning(ctx context.Context, log zerowrap.Logger, domainName, containerID string, now time.Time) {
128 // Check Docker health status for running containers.
129 healthStatus, hasHealthcheck, err := m.service.runtime.GetContainerHealthStatus(ctx, containerID)
130 if err != nil {
131 return
132 }
133
134 if hasHealthcheck && healthStatus == "unhealthy" {
135 log.Warn().Str("domain", domainName).Msg("monitor: container is unhealthy, restarting")
136 if err := m.service.runtime.RestartContainer(ctx, containerID); err != nil {
137 log.Warn().Err(err).Str("domain", domainName).Msg("monitor: failed to restart unhealthy container")
138 }
139 return
140 }
141
142 // Container is running and healthy — clear backoff if stable long enough.
143 m.mu.Lock()
144 rec, exists := m.history[domainName]
145 if exists {
146 if rec.lastSeen.IsZero() {
147 rec.lastSeen = now
148 }
149 if now.Sub(rec.lastSeen) >= stableRunningDuration {
150 // Container has been running for 5+ min — clear crash history.
151 delete(m.history, domainName)
152 m.mu.Unlock()
153 log.Info().Str("domain", domainName).Msg("monitor: container stable, cleared crash history")
154 return
155 }
156 }
157 m.mu.Unlock()
158}
159
160func (m *Monitor) handleCrash(ctx context.Context, log zerowrap.Logger, domainName, containerID string, exitCode int, now time.Time) {
161 m.mu.Lock()
162 rec := m.history[domainName]
163 if rec == nil {
164 rec = &restartRecord{}
165 m.history[domainName] = rec
166 }
167
168 // Check backoff.
169 if now.Before(rec.backoffEnd) {
170 m.mu.Unlock()
171 log.Debug().Str("domain", domainName).
172 Time("backoff_until", rec.backoffEnd).
173 Msg("monitor: in crash loop backoff, skipping restart")
174 return
175 }
176
177 // Record this crash attempt.
178 rec.attempts = append(rec.attempts, now)
179 rec.consecutive++
180 rec.lastSeen = time.Time{} // reset stable timer
181
182 // Prune old attempts outside the crash loop window.
183 cutoff := now.Add(-crashLoopWindow)
184 pruned := rec.attempts[:0]
185 for _, t := range rec.attempts {
186 if t.After(cutoff) {
187 pruned = append(pruned, t)
188 }
189 }
190 rec.attempts = pruned
191
192 // Check for crash loop.
193 if len(rec.attempts) >= crashLoopThreshold {
194 backoff := m.computeBackoff(rec.consecutive)
195 rec.backoffEnd = now.Add(backoff)
196 m.mu.Unlock()
197
198 log.Error().Str("domain", domainName).
199 Int("consecutive", rec.consecutive).
200 Dur("backoff", backoff).
201 Msg("monitor: crash loop detected, backing off")
202 return
203 }
204 m.mu.Unlock()
205
206 // Verify container ID still matches tracked state (deploy may have replaced it).
207 m.service.mu.RLock()
208 current := m.service.containers[domainName]
209 m.service.mu.RUnlock()
210 if current == nil || current.ID != containerID {
211 log.Debug().Str("domain", domainName).Msg("monitor: container replaced by deploy, skipping restart")
212 return
213 }
214
215 log.Warn().Str("domain", domainName).Str("container_id", containerID).
216 Int("exit_code", exitCode).
217 Msg("monitor: restarting crashed container")
218
219 if err := m.service.runtime.StartContainer(ctx, containerID); err != nil {
220 log.Warn().Err(err).Str("domain", domainName).Msg("monitor: failed to restart container")
221 }
222}
223
224func (m *Monitor) computeBackoff(consecutive int) time.Duration {
225 // Exponential backoff: 1min, 2min, 4min, 8min, cap 15min.
226 base := time.Minute
227 shift := consecutive - crashLoopThreshold
228 if shift < 0 {
229 shift = 0
230 }
231 if shift > 4 {
232 shift = 4 // cap shift to avoid overflow (max 16min, then capped to backoffCap)
233 }
234 backoff := base << uint(shift) // #nosec G115 -- shift is bounded [0,4]
235 if backoff > backoffCap {
236 backoff = backoffCap
237 }
238 return backoff
239}