1package container
2
3import (
4 "context"
5 "sync"
6 "time"
7
8 "github.com/bnema/zerowrap"
9 "go.opentelemetry.io/otel/attribute"
10 "go.opentelemetry.io/otel/metric"
11
12 "github.com/bnema/gordon/internal/domain"
13)
14
15const (
16 monitorDefaultInterval = 15 * time.Second
17 crashLoopThreshold = 3
18 crashLoopWindow = 5 * time.Minute
19 backoffCap = 15 * time.Minute
20 stableRunningDuration = 5 * time.Minute
21)
22
23// restartRecord tracks restart attempts for crash loop detection.
24type restartRecord struct {
25 attempts []time.Time
26 backoffEnd time.Time
27 consecutive int
28 lastSeen time.Time // last time the container was seen running
29}
30
31// Monitor watches tracked containers and restarts crashed ones.
32type Monitor struct {
33 service *Service
34 stopCh chan struct{}
35 stopped chan struct{}
36 interval time.Duration
37 mu sync.Mutex
38 history map[string]*restartRecord // keyed by domain
39}
40
41// newMonitor creates a new container monitor.
42func newMonitor(service *Service) *Monitor {
43 return &Monitor{
44 service: service,
45 stopCh: make(chan struct{}),
46 stopped: make(chan struct{}),
47 interval: monitorDefaultInterval,
48 history: make(map[string]*restartRecord),
49 }
50}
51
52// Start begins the background monitoring loop.
53func (m *Monitor) Start(ctx context.Context) {
54 log := zerowrap.FromCtx(ctx)
55 log.Info().Dur("interval", m.interval).Msg("container monitor started")
56
57 go m.run(ctx)
58}
59
60// Stop signals the monitor to stop and waits for it to finish.
61func (m *Monitor) Stop() {
62 close(m.stopCh)
63 <-m.stopped
64}
65
66func (m *Monitor) run(ctx context.Context) {
67 defer close(m.stopped)
68
69 ticker := time.NewTicker(m.interval)
70 defer ticker.Stop()
71
72 for {
73 select {
74 case <-m.stopCh:
75 return
76 case <-ctx.Done():
77 return
78 case <-ticker.C:
79 m.check(ctx)
80 }
81 }
82}
83
84func (m *Monitor) check(ctx context.Context) {
85 log := zerowrap.FromCtx(ctx)
86
87 // Snapshot tracked containers under lock.
88 m.service.mu.RLock()
89 snapshot := make(map[string]*domain.Container, len(m.service.containers))
90 for d, c := range m.service.containers {
91 snapshot[d] = c
92 }
93 m.service.mu.RUnlock()
94
95 now := time.Now()
96
97 for domainName, tracked := range snapshot {
98 m.checkContainer(ctx, log, domainName, tracked, now)
99 }
100}
101
102func (m *Monitor) checkContainer(ctx context.Context, log zerowrap.Logger, domainName string, tracked *domain.Container, now time.Time) {
103 if tracked == nil {
104 return
105 }
106
107 inspected, err := m.service.runtime.InspectContainer(ctx, tracked.ID)
108 if err != nil {
109 log.Debug().Err(err).Str("domain", domainName).Str("container_id", tracked.ID).
110 Msg("monitor: failed to inspect container, may have been replaced by deploy")
111 return
112 }
113
114 switch {
115 case inspected.Status == string(domain.ContainerStatusRunning):
116 m.handleRunning(ctx, log, domainName, tracked.ID, now)
117
118 case inspected.Status == string(domain.ContainerStatusExited):
119 if inspected.ExitCode == 0 {
120 // Graceful stop (exit 0): respect user intent, don't restart.
121 log.Debug().Str("domain", domainName).Msg("monitor: container exited gracefully (code 0), skipping")
122 return
123 }
124 // Crashed (non-zero exit): restart if not in backoff.
125 m.handleCrash(ctx, log, domainName, tracked.ID, inspected.ExitCode, now)
126 }
127}
128
129func (m *Monitor) handleRunning(ctx context.Context, log zerowrap.Logger, domainName, containerID string, now time.Time) {
130 // Check Docker health status for running containers.
131 healthStatus, hasHealthcheck, err := m.service.runtime.GetContainerHealthStatus(ctx, containerID)
132 if err != nil {
133 return
134 }
135
136 if hasHealthcheck && healthStatus == "unhealthy" {
137 if !m.isContainerStillTracked(domainName, containerID) {
138 log.Debug().Str("domain", domainName).Msg("monitor: container no longer tracked, skipping unhealthy restart")
139 return
140 }
141 log.Warn().Str("domain", domainName).Msg("monitor: container is unhealthy, restarting")
142 if err := m.service.runtime.RestartContainer(ctx, containerID); err != nil {
143 log.Warn().Err(err).Str("domain", domainName).Msg("monitor: failed to restart unhealthy container")
144 }
145 return
146 }
147
148 // Container is running and healthy — clear backoff if stable long enough.
149 m.mu.Lock()
150 rec, exists := m.history[domainName]
151 if exists {
152 if rec.lastSeen.IsZero() {
153 rec.lastSeen = now
154 }
155 if now.Sub(rec.lastSeen) >= stableRunningDuration {
156 // Container has been running for 5+ min — clear crash history.
157 delete(m.history, domainName)
158 m.mu.Unlock()
159 log.Info().Str("domain", domainName).Msg("monitor: container stable, cleared crash history")
160 return
161 }
162 }
163 m.mu.Unlock()
164}
165
166func (m *Monitor) isContainerStillTracked(domainName, containerID string) bool {
167 m.service.mu.RLock()
168 current := m.service.containers[domainName]
169 m.service.mu.RUnlock()
170 return current != nil && current.ID == containerID
171}
172
173func (m *Monitor) handleCrash(ctx context.Context, log zerowrap.Logger, domainName, containerID string, exitCode int, now time.Time) {
174 m.mu.Lock()
175 rec := m.history[domainName]
176 if rec == nil {
177 rec = &restartRecord{}
178 m.history[domainName] = rec
179 }
180
181 // Check backoff.
182 if now.Before(rec.backoffEnd) {
183 m.mu.Unlock()
184 log.Debug().Str("domain", domainName).
185 Time("backoff_until", rec.backoffEnd).
186 Msg("monitor: in crash loop backoff, skipping restart")
187 return
188 }
189
190 // Record this crash attempt.
191 rec.attempts = append(rec.attempts, now)
192 rec.consecutive++
193 rec.lastSeen = time.Time{} // reset stable timer
194
195 // Prune old attempts outside the crash loop window.
196 cutoff := now.Add(-crashLoopWindow)
197 pruned := rec.attempts[:0]
198 for _, t := range rec.attempts {
199 if t.After(cutoff) {
200 pruned = append(pruned, t)
201 }
202 }
203 rec.attempts = pruned
204
205 // Check for crash loop.
206 if len(rec.attempts) >= crashLoopThreshold {
207 backoff := m.computeBackoff(rec.consecutive)
208 rec.backoffEnd = now.Add(backoff)
209 m.mu.Unlock()
210
211 log.Error().Str("domain", domainName).
212 Int("consecutive", rec.consecutive).
213 Dur("backoff", backoff).
214 Msg("monitor: crash loop detected, backing off")
215
216 // Record crash loop metric
217 if m.service.metrics != nil {
218 attrs := metric.WithAttributes(attribute.String("domain", domainName))
219 m.service.metrics.ContainerCrashLoops.Add(ctx, 1, attrs)
220 }
221 return
222 }
223 m.mu.Unlock()
224
225 // Verify container ID still matches tracked state (deploy may have replaced it).
226 if !m.isContainerStillTracked(domainName, containerID) {
227 log.Debug().Str("domain", domainName).Msg("monitor: container replaced by deploy, skipping restart")
228 return
229 }
230
231 log.Warn().Str("domain", domainName).Str("container_id", containerID).
232 Int("exit_code", exitCode).
233 Msg("monitor: restarting crashed container")
234
235 if err := m.service.runtime.StartContainer(ctx, containerID); err != nil {
236 log.Warn().Err(err).Str("domain", domainName).Msg("monitor: failed to restart container")
237 return
238 }
239
240 // Record restart metric only on successful restart
241 if m.service.metrics != nil {
242 attrs := metric.WithAttributes(
243 attribute.String("domain", domainName),
244 attribute.String("source", "monitor"),
245 )
246 m.service.metrics.ContainerRestarts.Add(ctx, 1, attrs)
247 }
248}
249
250func (m *Monitor) computeBackoff(consecutive int) time.Duration {
251 // Exponential backoff: 1min, 2min, 4min, 8min, cap 15min.
252 base := time.Minute
253 shift := consecutive - crashLoopThreshold
254 if shift < 0 {
255 shift = 0
256 }
257 if shift > 4 {
258 shift = 4 // cap shift to avoid overflow (max 16min, then capped to backoffCap)
259 }
260 backoff := base << uint(shift) // #nosec G115 -- shift is bounded [0,4]
261 if backoff > backoffCap {
262 backoff = backoffCap
263 }
264 return backoff
265}