1package container
2
3import (
4 "context"
5 "sync"
6 "time"
7
8 "github.com/bnema/zerowrap"
9 "go.opentelemetry.io/otel/attribute"
10 "go.opentelemetry.io/otel/metric"
11
12 "github.com/bnema/gordon/internal/domain"
13)
14
15const (
16 monitorDefaultInterval = 15 * time.Second
17 crashLoopThreshold = 3
18 crashLoopWindow = 5 * time.Minute
19 backoffCap = 15 * time.Minute
20 stableRunningDuration = 5 * time.Minute
21)
22
23// restartRecord tracks restart attempts for crash loop detection.
24type restartRecord struct {
25 attempts []time.Time
26 backoffEnd time.Time
27 consecutive int
28 lastSeen time.Time // last time the container was seen running
29}
30
31// Monitor watches tracked containers and restarts crashed ones.
32type Monitor struct {
33 service *Service
34 stopCh chan struct{}
35 stopped chan struct{}
36 interval time.Duration
37 mu sync.Mutex
38 history map[string]*restartRecord // keyed by domain
39}
40
41// newMonitor creates a new container monitor.
42func newMonitor(service *Service) *Monitor {
43 return &Monitor{
44 service: service,
45 stopCh: make(chan struct{}),
46 stopped: make(chan struct{}),
47 interval: monitorDefaultInterval,
48 history: make(map[string]*restartRecord),
49 }
50}
51
52// Start begins the background monitoring loop.
53func (m *Monitor) Start(ctx context.Context) {
54 log := zerowrap.FromCtx(ctx)
55 log.Info().Dur("interval", m.interval).Msg("container monitor started")
56
57 go m.run(ctx)
58}
59
60// Stop signals the monitor to stop and waits for it to finish.
61func (m *Monitor) Stop() {
62 close(m.stopCh)
63 <-m.stopped
64}
65
66func (m *Monitor) run(ctx context.Context) {
67 defer close(m.stopped)
68
69 ticker := time.NewTicker(m.interval)
70 defer ticker.Stop()
71
72 for {
73 select {
74 case <-m.stopCh:
75 return
76 case <-ctx.Done():
77 return
78 case <-ticker.C:
79 m.check(ctx)
80 }
81 }
82}
83
84func (m *Monitor) check(ctx context.Context) {
85 log := zerowrap.FromCtx(ctx)
86
87 // Snapshot tracked containers under lock.
88 m.service.mu.RLock()
89 snapshot := make(map[string]*domain.Container, len(m.service.containers))
90 for d, c := range m.service.containers {
91 snapshot[d] = c
92 }
93 m.service.mu.RUnlock()
94
95 now := time.Now()
96
97 for domainName, tracked := range snapshot {
98 m.checkContainer(ctx, log, domainName, tracked, now)
99 }
100}
101
102func (m *Monitor) checkContainer(ctx context.Context, log zerowrap.Logger, domainName string, tracked *domain.Container, now time.Time) {
103 if tracked == nil {
104 return
105 }
106
107 inspected, err := m.service.runtime.InspectContainer(ctx, tracked.ID)
108 if err != nil {
109 log.Debug().Err(err).Str("domain", domainName).Str("container_id", tracked.ID).
110 Msg("monitor: failed to inspect container, may have been replaced by deploy")
111 return
112 }
113
114 switch {
115 case inspected.Status == string(domain.ContainerStatusRunning):
116 m.handleRunning(ctx, log, domainName, tracked.ID, now)
117
118 case inspected.Status == string(domain.ContainerStatusExited):
119 if inspected.ExitCode == 0 {
120 // Graceful stop (exit 0): respect user intent, don't restart.
121 log.Debug().Str("domain", domainName).Msg("monitor: container exited gracefully (code 0), skipping")
122 return
123 }
124 // Crashed (non-zero exit): restart if not in backoff.
125 m.handleCrash(ctx, log, domainName, tracked.ID, inspected.ExitCode, now)
126 }
127}
128
129func (m *Monitor) handleRunning(ctx context.Context, log zerowrap.Logger, domainName, containerID string, now time.Time) {
130 // Check Docker health status for running containers.
131 healthStatus, hasHealthcheck, err := m.service.runtime.GetContainerHealthStatus(ctx, containerID)
132 if err != nil {
133 return
134 }
135
136 if hasHealthcheck && healthStatus == "unhealthy" {
137 log.Warn().Str("domain", domainName).Msg("monitor: container is unhealthy, restarting")
138 if err := m.service.runtime.RestartContainer(ctx, containerID); err != nil {
139 log.Warn().Err(err).Str("domain", domainName).Msg("monitor: failed to restart unhealthy container")
140 }
141 return
142 }
143
144 // Container is running and healthy — clear backoff if stable long enough.
145 m.mu.Lock()
146 rec, exists := m.history[domainName]
147 if exists {
148 if rec.lastSeen.IsZero() {
149 rec.lastSeen = now
150 }
151 if now.Sub(rec.lastSeen) >= stableRunningDuration {
152 // Container has been running for 5+ min — clear crash history.
153 delete(m.history, domainName)
154 m.mu.Unlock()
155 log.Info().Str("domain", domainName).Msg("monitor: container stable, cleared crash history")
156 return
157 }
158 }
159 m.mu.Unlock()
160}
161
162func (m *Monitor) handleCrash(ctx context.Context, log zerowrap.Logger, domainName, containerID string, exitCode int, now time.Time) {
163 m.mu.Lock()
164 rec := m.history[domainName]
165 if rec == nil {
166 rec = &restartRecord{}
167 m.history[domainName] = rec
168 }
169
170 // Check backoff.
171 if now.Before(rec.backoffEnd) {
172 m.mu.Unlock()
173 log.Debug().Str("domain", domainName).
174 Time("backoff_until", rec.backoffEnd).
175 Msg("monitor: in crash loop backoff, skipping restart")
176 return
177 }
178
179 // Record this crash attempt.
180 rec.attempts = append(rec.attempts, now)
181 rec.consecutive++
182 rec.lastSeen = time.Time{} // reset stable timer
183
184 // Prune old attempts outside the crash loop window.
185 cutoff := now.Add(-crashLoopWindow)
186 pruned := rec.attempts[:0]
187 for _, t := range rec.attempts {
188 if t.After(cutoff) {
189 pruned = append(pruned, t)
190 }
191 }
192 rec.attempts = pruned
193
194 // Check for crash loop.
195 if len(rec.attempts) >= crashLoopThreshold {
196 backoff := m.computeBackoff(rec.consecutive)
197 rec.backoffEnd = now.Add(backoff)
198 m.mu.Unlock()
199
200 log.Error().Str("domain", domainName).
201 Int("consecutive", rec.consecutive).
202 Dur("backoff", backoff).
203 Msg("monitor: crash loop detected, backing off")
204
205 // Record crash loop metric
206 if m.service.metrics != nil {
207 attrs := metric.WithAttributes(attribute.String("domain", domainName))
208 m.service.metrics.ContainerCrashLoops.Add(ctx, 1, attrs)
209 }
210 return
211 }
212 m.mu.Unlock()
213
214 // Verify container ID still matches tracked state (deploy may have replaced it).
215 m.service.mu.RLock()
216 current := m.service.containers[domainName]
217 m.service.mu.RUnlock()
218 if current == nil || current.ID != containerID {
219 log.Debug().Str("domain", domainName).Msg("monitor: container replaced by deploy, skipping restart")
220 return
221 }
222
223 log.Warn().Str("domain", domainName).Str("container_id", containerID).
224 Int("exit_code", exitCode).
225 Msg("monitor: restarting crashed container")
226
227 if err := m.service.runtime.StartContainer(ctx, containerID); err != nil {
228 log.Warn().Err(err).Str("domain", domainName).Msg("monitor: failed to restart container")
229 return
230 }
231
232 // Record restart metric only on successful restart
233 if m.service.metrics != nil {
234 attrs := metric.WithAttributes(
235 attribute.String("domain", domainName),
236 attribute.String("source", "monitor"),
237 )
238 m.service.metrics.ContainerRestarts.Add(ctx, 1, attrs)
239 }
240}
241
242func (m *Monitor) computeBackoff(consecutive int) time.Duration {
243 // Exponential backoff: 1min, 2min, 4min, 8min, cap 15min.
244 base := time.Minute
245 shift := consecutive - crashLoopThreshold
246 if shift < 0 {
247 shift = 0
248 }
249 if shift > 4 {
250 shift = 4 // cap shift to avoid overflow (max 16min, then capped to backoffCap)
251 }
252 backoff := base << uint(shift) // #nosec G115 -- shift is bounded [0,4]
253 if backoff > backoffCap {
254 backoff = backoffCap
255 }
256 return backoff
257}