feat(monitor): add background container monitor and non-blocking AutoStart

Replace blocking healthcheck-based AutoStart with a non-blocking approach:
- AutoStart now skips readiness checks (WithSkipReadiness context flag),
  deploying all routes concurrently with a semaphore of 4
- New background Monitor polls tracked containers every 15s, restarting
  crashed (non-zero exit) and unhealthy containers automatically
- Crash loop detection: 3 restarts within 5min triggers exponential
  backoff (1min→2min→4min→8min→cap 15min), cleared after 5min stable
- Shutdown no longer stops containers; they persist across Gordon restarts
  and are reattached via SyncContainers + AutoStart on next boot
- Remove retryReadiness, healthTimeoutError, HealthTimeoutMultiplier
- Fix pg_dump to use POSTGRES_DB env var instead of shell-quoted db name
- Add tilde expansion for backup storage root directory

commit: 17304b43267d4b80466c57fb51479b379f189419
author: brice · 8mo ago · 2026-02-08 13:44 UTC
committer: brice · 8mo ago · 2026-02-08 13:52 UTC
parent: 44248c0
signature: unsigned

11 files · +696 -221

M internal/adapters/out/docker/runtime.go +7 -6


@@ -350,12 +350,13 @@ func (r Runtime) InspectContainer(ctx context.Context, containerID string) (do
350	350		name := strings.TrimPrefix(resp.Name, "/")
351	351
352	352		return &domain.Container{
353		-	ID: resp.ID,
354		-	Image: resp.Config.Image,
355		-	Name: name,
356		-	Status: resp.State.Status,
357		-	Ports: ports,
358		-	Labels: resp.Config.Labels,
	353	+	ID: resp.ID,
	354	+	Image: resp.Config.Image,
	355	+	Name: name,
	356	+	Status: resp.State.Status,
	357	+	ExitCode: resp.State.ExitCode,
	358	+	Ports: ports,
	359	+	Labels: resp.Config.Labels,
359	360		}, nil
360	361		}
361	362


@@ -350,12 +350,13 @@ func (r Runtime) InspectContainer(ctx context.Context, containerID string) (do
350		name := strings.TrimPrefix(resp.Name, "/")	350		name := strings.TrimPrefix(resp.Name, "/")
351			351
352		return &domain.Container{	352		return &domain.Container{
353	-	ID: resp.ID,	353	+	ID: resp.ID,
354	-	Image: resp.Config.Image,	354	+	Image: resp.Config.Image,
355	-	Name: name,	355	+	Name: name,
356	-	Status: resp.State.Status,	356	+	Status: resp.State.Status,
357	-	Ports: ports,	357	+	ExitCode: resp.State.ExitCode,
358	-	Labels: resp.Config.Labels,	358	+	Ports: ports,
			359	+	Labels: resp.Config.Labels,
359		}, nil	360		}, nil
360		}	361		}
361			362

M internal/adapters/out/filesystem/backup_storage.go +13 -0


@@ -27,6 +27,8 @@ type BackupStorage struct {
27	27
28	28		// NewBackupStorage creates a new filesystem backup storage.
29	29		func NewBackupStorage(rootDir string, log zerowrap.Logger) (*BackupStorage, error) {
	30	+	rootDir = expandTilde(rootDir)
	31	+
30	32		if err := os.MkdirAll(rootDir, 0750); err != nil {
31	33		return nil, fmt.Errorf("failed to create backup directory: %w", err)
32	34		}

@@ -34,6 +36,17 @@ func NewBackupStorage(rootDir string, log zerowrap.Logger) (*BackupStorage, erro
34	36		return &BackupStorage{rootDir: rootDir, log: log}, nil
35	37		}
36	38
	39	+	// expandTilde replaces a leading "~/" with the user's home directory.
	40	+	func expandTilde(path string) string {
	41	+	if !strings.HasPrefix(path, "~/") {
	42	+	return path
	43	+	}
	44	+	if home, err := os.UserHomeDir(); err == nil {
	45	+	return filepath.Join(home, path[2:])
	46	+	}
	47	+	return path
	48	+	}
	49	+
37	50		// Store saves backup data and returns the absolute storage path.
38	51		func (s *BackupStorage) Store(_ context.Context, domainName, dbName string, schedule domain.BackupSchedule, timestamp time.Time, data io.Reader) (string, error) {
39	52		domainPart := sanitizeBackupPathComponent(domainName)


@@ -27,6 +27,8 @@ type BackupStorage struct {
27		27
28	// NewBackupStorage creates a new filesystem backup storage.	28		// NewBackupStorage creates a new filesystem backup storage.
29	func NewBackupStorage(rootDir string, log zerowrap.Logger) (*BackupStorage, error) {	29		func NewBackupStorage(rootDir string, log zerowrap.Logger) (*BackupStorage, error) {
		30	+	rootDir = expandTilde(rootDir)
		31	+
30	if err := os.MkdirAll(rootDir, 0750); err != nil {	32		if err := os.MkdirAll(rootDir, 0750); err != nil {
31	return nil, fmt.Errorf("failed to create backup directory: %w", err)	33		return nil, fmt.Errorf("failed to create backup directory: %w", err)
32	}	34		}

@@ -34,6 +36,17 @@ func NewBackupStorage(rootDir string, log zerowrap.Logger) (*BackupStorage, erro
34	return &BackupStorage{rootDir: rootDir, log: log}, nil	36		return &BackupStorage{rootDir: rootDir, log: log}, nil
35	}	37		}
36		38
		39	+	// expandTilde replaces a leading "~/" with the user's home directory.
		40	+	func expandTilde(path string) string {
		41	+	if !strings.HasPrefix(path, "~/") {
		42	+	return path
		43	+	}
		44	+	if home, err := os.UserHomeDir(); err == nil {
		45	+	return filepath.Join(home, path[2:])
		46	+	}
		47	+	return path
		48	+	}
		49	+
37	// Store saves backup data and returns the absolute storage path.	50		// Store saves backup data and returns the absolute storage path.
38	func (s *BackupStorage) Store(_ context.Context, domainName, dbName string, schedule domain.BackupSchedule, timestamp time.Time, data io.Reader) (string, error) {	51		func (s *BackupStorage) Store(_ context.Context, domainName, dbName string, schedule domain.BackupSchedule, timestamp time.Time, data io.Reader) (string, error) {
39	domainPart := sanitizeBackupPathComponent(domainName)	52		domainPart := sanitizeBackupPathComponent(domainName)

M internal/app/run.go +5 -0


@@ -1254,6 +1254,9 @@ func syncAndAutoStart(ctx context.Context, svc *services, log zerowrap.Logger) {
1254	1254		log.Warn().Err(err).Msg("failed to auto-start containers")
1255	1255		}
1256	1256		}
	1257	+
	1258	+	// Start background monitor to restart crashed containers.
	1259	+	svc.containerSvc.StartMonitor(ctx)
1257	1260		}
1258	1261
1259	1262		// createHTTPHandlers creates HTTP handlers with middleware.

@@ -1662,6 +1665,8 @@ func gracefulShutdown(registrySrv, proxySrv http.Server, containerSvc containe
1662	1665		log.Warn().Err(err).Msg("proxy server shutdown error")
1663	1666		}
1664	1667
	1668	+	containerSvc.StopMonitor()
	1669	+
1665	1670		if err := containerSvc.Shutdown(shutdownCtx); err != nil {
1666	1671		log.Warn().Err(err).Msg("error during container shutdown")
1667	1672		}


@@ -1254,6 +1254,9 @@ func syncAndAutoStart(ctx context.Context, svc *services, log zerowrap.Logger) {
1254	log.Warn().Err(err).Msg("failed to auto-start containers")	1254		log.Warn().Err(err).Msg("failed to auto-start containers")
1255	}	1255		}
1256	}	1256		}
		1257	+
		1258	+	// Start background monitor to restart crashed containers.
		1259	+	svc.containerSvc.StartMonitor(ctx)
1257	}	1260		}
1258		1261
1259	// createHTTPHandlers creates HTTP handlers with middleware.	1262		// createHTTPHandlers creates HTTP handlers with middleware.

@@ -1662,6 +1665,8 @@ func gracefulShutdown(registrySrv, proxySrv http.Server, containerSvc containe
1662	log.Warn().Err(err).Msg("proxy server shutdown error")	1665		log.Warn().Err(err).Msg("proxy server shutdown error")
1663	}	1666		}
1664		1667
		1668	+	containerSvc.StopMonitor()
		1669	+
1665	if err := containerSvc.Shutdown(shutdownCtx); err != nil {	1670		if err := containerSvc.Shutdown(shutdownCtx); err != nil {
1666	log.Warn().Err(err).Msg("error during container shutdown")	1671		log.Warn().Err(err).Msg("error during container shutdown")
1667	}	1672		}

M internal/domain/container.go +7 -6


@@ -4,12 +4,13 @@ package domain
4	4
5	5		// Container represents a running container in the system.
6	6		type Container struct {
7		-	ID string
8		-	Image string
9		-	Name string
10		-	Status string
11		-	Ports []int
12		-	Labels map[string]string
	7	+	ID string
	8	+	Image string
	9	+	Name string
	10	+	Status string
	11	+	ExitCode int
	12	+	Ports []int
	13	+	Labels map[string]string
13	14		}
14	15
15	16		// NetworkInfo represents network configuration and state.


@@ -4,12 +4,13 @@ package domain
4			4
5		// Container represents a running container in the system.	5		// Container represents a running container in the system.
6		type Container struct {	6		type Container struct {
7	-	ID string	7	+	ID string
8	-	Image string	8	+	Image string
9	-	Name string	9	+	Name string
10	-	Status string	10	+	Status string
11	-	Ports []int	11	+	ExitCode int
12	-	Labels map[string]string	12	+	Ports []int
			13	+	Labels map[string]string
13		}	14		}
14			15
15		// NetworkInfo represents network configuration and state.	16		// NetworkInfo represents network configuration and state.

M internal/domain/event.go +10 -12


@@ -83,20 +83,18 @@ func WithInternalDeploy(ctx context.Context) context.Context {
83	83		}
84	84
85	85		const (
86		-	// ContextKeyHealthTimeoutMultiplier overrides the health timeout with a multiplier.
87		-	ContextKeyHealthTimeoutMultiplier contextKey = "health_timeout_multiplier"
	86	+	// ContextKeySkipReadiness indicates that readiness checks should be skipped
	87	+	// (e.g., during AutoStart where the background monitor handles crash recovery).
	88	+	ContextKeySkipReadiness contextKey = "skip_readiness"
88	89		)
89	90
90		-	// WithHealthTimeoutMultiplier returns a context that scales the health timeout.
91		-	func WithHealthTimeoutMultiplier(ctx context.Context, multiplier float64) context.Context {
92		-	return context.WithValue(ctx, ContextKeyHealthTimeoutMultiplier, multiplier)
	91	+	// WithSkipReadiness returns a context that skips readiness checks on deploy.
	92	+	func WithSkipReadiness(ctx context.Context) context.Context {
	93	+	return context.WithValue(ctx, ContextKeySkipReadiness, true)
93	94		}
94	95
95		-	// HealthTimeoutMultiplier returns the timeout multiplier from ctx, or 1.0 if unset.
96		-	func HealthTimeoutMultiplier(ctx context.Context) float64 {
97		-	v, ok := ctx.Value(ContextKeyHealthTimeoutMultiplier).(float64)
98		-	if !ok \|\| v <= 0 {
99		-	return 1.0
100		-	}
101		-	return v
	96	+	// IsSkipReadiness checks if the context indicates readiness should be skipped.
	97	+	func IsSkipReadiness(ctx context.Context) bool {
	98	+	v, ok := ctx.Value(ContextKeySkipReadiness).(bool)
	99	+	return ok && v
102	100		}


@@ -83,20 +83,18 @@ func WithInternalDeploy(ctx context.Context) context.Context {
83		}	83		}
84			84
85		const (	85		const (
86	-	// ContextKeyHealthTimeoutMultiplier overrides the health timeout with a multiplier.	86	+	// ContextKeySkipReadiness indicates that readiness checks should be skipped
87	-	ContextKeyHealthTimeoutMultiplier contextKey = "health_timeout_multiplier"	87	+	// (e.g., during AutoStart where the background monitor handles crash recovery).
			88	+	ContextKeySkipReadiness contextKey = "skip_readiness"
88		)	89		)
89			90
90	-	// WithHealthTimeoutMultiplier returns a context that scales the health timeout.	91	+	// WithSkipReadiness returns a context that skips readiness checks on deploy.
91	-	func WithHealthTimeoutMultiplier(ctx context.Context, multiplier float64) context.Context {	92	+	func WithSkipReadiness(ctx context.Context) context.Context {
92	-	return context.WithValue(ctx, ContextKeyHealthTimeoutMultiplier, multiplier)	93	+	return context.WithValue(ctx, ContextKeySkipReadiness, true)
93		}	94		}
94			95
95	-	// HealthTimeoutMultiplier returns the timeout multiplier from ctx, or 1.0 if unset.	96	+	// IsSkipReadiness checks if the context indicates readiness should be skipped.
96	-	func HealthTimeoutMultiplier(ctx context.Context) float64 {	97	+	func IsSkipReadiness(ctx context.Context) bool {
97	-	v, ok := ctx.Value(ContextKeyHealthTimeoutMultiplier).(float64)	98	+	v, ok := ctx.Value(ContextKeySkipReadiness).(bool)
98	-	if !ok \|\| v <= 0 {	99	+	return ok && v
99	-	return 1.0
100	-	}
101	-	return v
102		}	100		}

M internal/usecase/backup/service.go +2 -6


@@ -291,18 +291,14 @@ func selectDatabase(dbs []domain.DBInfo, requested string) (domain.DBInfo, error
291	291		return domain.DBInfo{}, fmt.Errorf("database %q not found for domain", requested)
292	292		}
293	293
294		-	func postgresDumpCommand(dbName string) string {
295		-	return fmt.Sprintf("PGDATABASE=%s pg_dump -Fc --dbname=\"$PGDATABASE\" --username=\"${POSTGRES_USER:-postgres}\"", shellQuote(dbName))
	294	+	func postgresDumpCommand(_ string) string {
	295	+	return "pg_dump -Fc --dbname=\"${POSTGRES_DB:-postgres}\" --username=\"${POSTGRES_USER:-postgres}\""
296	296		}
297	297
298	298		func pgDumpToPathCommand(path, dbName string) string {
299	299		return fmt.Sprintf("%s > %q", postgresDumpCommand(dbName), path)
300	300		}
301	301
302		-	func shellQuote(value string) string {
303		-	return "'" + strings.ReplaceAll(value, "'", "'\"'\"'") + "'"
304		-	}
305		-
306	302		func (s *Service) cleanupDumpFile(containerID, dumpPath string) {
307	303		cleanupCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
308	304		defer cancel()

 	return domain.DBInfo{}, fmt.Errorf("database %q not found for domain", requested)
 }
-func postgresDumpCommand(dbName string) string {
+func postgresDumpCommand(_ string) string {
-	return fmt.Sprintf("PGDATABASE=%s pg_dump -Fc --dbname=\"$PGDATABASE\" --username=\"${POSTGRES_USER:-postgres}\"", shellQuote(dbName))
+	return "pg_dump -Fc --dbname=\"${POSTGRES_DB:-postgres}\" --username=\"${POSTGRES_USER:-postgres}\""
 }
 func pgDumpToPathCommand(path, dbName string) string {
 	return fmt.Sprintf("%s > %q", postgresDumpCommand(dbName), path)
 }
-func shellQuote(value string) string {
-	return "'" + strings.ReplaceAll(value, "'", "'\"'\"'") + "'"
-}
 func (s *Service) cleanupDumpFile(containerID, dumpPath string) {
 	cleanupCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()

M internal/usecase/backup/service_test.go +5 -4


@@ -50,7 +50,7 @@ func TestService_RunBackup_Postgres(t *testing.T) {
50	50		return false
51	51		}
52	52		return bytes.Contains([]byte(cmd[2]), []byte("pg_dump -Fc")) &&
53		-	bytes.Contains([]byte(cmd[2]), []byte("PGDATABASE='postgres'")) &&
	53	+	bytes.Contains([]byte(cmd[2]), []byte("${POSTGRES_DB:-postgres}")) &&
54	54		bytes.Contains([]byte(cmd[2]), []byte(" > "))
55	55		})).Return(&outiface.ExecResult{ExitCode: 0, Stdout: []byte("backup-data")}, nil)
56	56

@@ -125,9 +125,10 @@ func TestSelectDatabaseAutoSelectsOnlyDatabaseWhenUnspecified(t *testing.T) {
125	125		assert.Equal(t, "postgres", db.Name)
126	126		}
127	127
128		-	func TestPostgresDumpCommandQuotesDatabaseName(t *testing.T) {
129		-	cmd := postgresDumpCommand("customer data")
130		-	assert.Contains(t, cmd, "PGDATABASE='customer data'")
	128	+	func TestPostgresDumpCommandUsesEnvVar(t *testing.T) {
	129	+	cmd := postgresDumpCommand("ignored")
	130	+	assert.Contains(t, cmd, "${POSTGRES_DB:-postgres}")
	131	+	assert.Contains(t, cmd, "${POSTGRES_USER:-postgres}")
131	132		}
132	133
133	134		func TestServiceStatusReturnsWhenContextCancelledDuringSemaphoreAcquire(t *testing.T) {

 			return false
 		}
 		return bytes.Contains([]byte(cmd[2]), []byte("pg_dump -Fc")) &&
-			bytes.Contains([]byte(cmd[2]), []byte("PGDATABASE='postgres'")) &&
+			bytes.Contains([]byte(cmd[2]), []byte("${POSTGRES_DB:-postgres}")) &&
 			bytes.Contains([]byte(cmd[2]), []byte(" > "))
 	})).Return(&outiface.ExecResult{ExitCode: 0, Stdout: []byte("backup-data")}, nil)
 	assert.Equal(t, "postgres", db.Name)
 }
-func TestPostgresDumpCommandQuotesDatabaseName(t *testing.T) {
+func TestPostgresDumpCommandUsesEnvVar(t *testing.T) {
-	cmd := postgresDumpCommand("customer data")
+	cmd := postgresDumpCommand("ignored")
-	assert.Contains(t, cmd, "PGDATABASE='customer data'")
+	assert.Contains(t, cmd, "${POSTGRES_DB:-postgres}")
+	assert.Contains(t, cmd, "${POSTGRES_USER:-postgres}")
 }
 func TestServiceStatusReturnsWhenContextCancelledDuringSemaphoreAcquire(t *testing.T) {

A internal/usecase/container/monitor.go +239 -0

@@ -0,0 +1,239 @@
	1	+	package container
	2	+
	3	+	import (
	4	+	"context"
	5	+	"sync"
	6	+	"time"
	7	+
	8	+	"github.com/bnema/zerowrap"
	9	+
	10	+	"github.com/bnema/gordon/internal/domain"
	11	+	)
	12	+
	13	+	const (
	14	+	monitorDefaultInterval = 15 * time.Second
	15	+	crashLoopThreshold = 3
	16	+	crashLoopWindow = 5 * time.Minute
	17	+	backoffCap = 15 * time.Minute
	18	+	stableRunningDuration = 5 * time.Minute
	19	+	)
	20	+
	21	+	// restartRecord tracks restart attempts for crash loop detection.
	22	+	type restartRecord struct {
	23	+	attempts []time.Time
	24	+	backoffEnd time.Time
	25	+	consecutive int
	26	+	lastSeen time.Time // last time the container was seen running
	27	+	}
	28	+
	29	+	// Monitor watches tracked containers and restarts crashed ones.
	30	+	type Monitor struct {
	31	+	service *Service
	32	+	stopCh chan struct{}
	33	+	stopped chan struct{}
	34	+	interval time.Duration
	35	+	mu sync.Mutex
	36	+	history map[string]*restartRecord // keyed by domain
	37	+	}
	38	+
	39	+	// newMonitor creates a new container monitor.
	40	+	func newMonitor(service Service) Monitor {
	41	+	return &Monitor{
	42	+	service: service,
	43	+	stopCh: make(chan struct{}),
	44	+	stopped: make(chan struct{}),
	45	+	interval: monitorDefaultInterval,
	46	+	history: make(map[string]*restartRecord),
	47	+	}
	48	+	}
	49	+
	50	+	// Start begins the background monitoring loop.
	51	+	func (m *Monitor) Start(ctx context.Context) {
	52	+	log := zerowrap.FromCtx(ctx)
	53	+	log.Info().Dur("interval", m.interval).Msg("container monitor started")
	54	+
	55	+	go m.run(ctx)
	56	+	}
	57	+
	58	+	// Stop signals the monitor to stop and waits for it to finish.
	59	+	func (m *Monitor) Stop() {
	60	+	close(m.stopCh)
	61	+	<-m.stopped
	62	+	}
	63	+
	64	+	func (m *Monitor) run(ctx context.Context) {
	65	+	defer close(m.stopped)
	66	+
	67	+	ticker := time.NewTicker(m.interval)
	68	+	defer ticker.Stop()
	69	+
	70	+	for {
	71	+	select {
	72	+	case <-m.stopCh:
	73	+	return
	74	+	case <-ctx.Done():
	75	+	return
	76	+	case <-ticker.C:
	77	+	m.check(ctx)
	78	+	}
	79	+	}
	80	+	}
	81	+
	82	+	func (m *Monitor) check(ctx context.Context) {
	83	+	log := zerowrap.FromCtx(ctx)
	84	+
	85	+	// Snapshot tracked containers under lock.
	86	+	m.service.mu.RLock()
	87	+	snapshot := make(map[string]*domain.Container, len(m.service.containers))
	88	+	for d, c := range m.service.containers {
	89	+	snapshot[d] = c
	90	+	}
	91	+	m.service.mu.RUnlock()
	92	+
	93	+	now := time.Now()
	94	+
	95	+	for domainName, tracked := range snapshot {
	96	+	m.checkContainer(ctx, log, domainName, tracked, now)
	97	+	}
	98	+	}
	99	+
	100	+	func (m Monitor) checkContainer(ctx context.Context, log zerowrap.Logger, domainName string, tracked domain.Container, now time.Time) {
	101	+	if tracked == nil {
	102	+	return
	103	+	}
	104	+
	105	+	inspected, err := m.service.runtime.InspectContainer(ctx, tracked.ID)
	106	+	if err != nil {
	107	+	log.Debug().Err(err).Str("domain", domainName).Str("container_id", tracked.ID).
	108	+	Msg("monitor: failed to inspect container, may have been replaced by deploy")
	109	+	return
	110	+	}
	111	+
	112	+	switch {
	113	+	case inspected.Status == string(domain.ContainerStatusRunning):
	114	+	m.handleRunning(ctx, log, domainName, tracked.ID, now)
	115	+
	116	+	case inspected.Status == string(domain.ContainerStatusExited):
	117	+	if inspected.ExitCode == 0 {
	118	+	// Graceful stop (exit 0): respect user intent, don't restart.
	119	+	log.Debug().Str("domain", domainName).Msg("monitor: container exited gracefully (code 0), skipping")
	120	+	return
	121	+	}
	122	+	// Crashed (non-zero exit): restart if not in backoff.
	123	+	m.handleCrash(ctx, log, domainName, tracked.ID, now)
	124	+	}
	125	+	}
	126	+
	127	+	func (m *Monitor) handleRunning(ctx context.Context, log zerowrap.Logger, domainName, containerID string, now time.Time) {
	128	+	// Check Docker health status for running containers.
	129	+	healthStatus, hasHealthcheck, err := m.service.runtime.GetContainerHealthStatus(ctx, containerID)
	130	+	if err != nil {
	131	+	return
	132	+	}
	133	+
	134	+	if hasHealthcheck && healthStatus == "unhealthy" {
	135	+	log.Warn().Str("domain", domainName).Msg("monitor: container is unhealthy, restarting")
	136	+	if err := m.service.runtime.RestartContainer(ctx, containerID); err != nil {
	137	+	log.Warn().Err(err).Str("domain", domainName).Msg("monitor: failed to restart unhealthy container")
	138	+	}
	139	+	return
	140	+	}
	141	+
	142	+	// Container is running and healthy — clear backoff if stable long enough.
	143	+	m.mu.Lock()
	144	+	rec, exists := m.history[domainName]
	145	+	if exists {
	146	+	if rec.lastSeen.IsZero() {
	147	+	rec.lastSeen = now
	148	+	}
	149	+	if now.Sub(rec.lastSeen) >= stableRunningDuration {
	150	+	// Container has been running for 5+ min — clear crash history.
	151	+	delete(m.history, domainName)
	152	+	m.mu.Unlock()
	153	+	log.Info().Str("domain", domainName).Msg("monitor: container stable, cleared crash history")
	154	+	return
	155	+	}
	156	+	}
	157	+	m.mu.Unlock()
	158	+	}
	159	+
	160	+	func (m *Monitor) handleCrash(ctx context.Context, log zerowrap.Logger, domainName, containerID string, now time.Time) {
	161	+	m.mu.Lock()
	162	+	rec := m.history[domainName]
	163	+	if rec == nil {
	164	+	rec = &restartRecord{}
	165	+	m.history[domainName] = rec
	166	+	}
	167	+
	168	+	// Check backoff.
	169	+	if now.Before(rec.backoffEnd) {
	170	+	m.mu.Unlock()
	171	+	log.Debug().Str("domain", domainName).
	172	+	Time("backoff_until", rec.backoffEnd).
	173	+	Msg("monitor: in crash loop backoff, skipping restart")
	174	+	return
	175	+	}
	176	+
	177	+	// Record this crash attempt.
	178	+	rec.attempts = append(rec.attempts, now)
	179	+	rec.consecutive++
	180	+	rec.lastSeen = time.Time{} // reset stable timer
	181	+
	182	+	// Prune old attempts outside the crash loop window.
	183	+	cutoff := now.Add(-crashLoopWindow)
	184	+	pruned := rec.attempts[:0]
	185	+	for _, t := range rec.attempts {
	186	+	if t.After(cutoff) {
	187	+	pruned = append(pruned, t)
	188	+	}
	189	+	}
	190	+	rec.attempts = pruned
	191	+
	192	+	// Check for crash loop.
	193	+	if len(rec.attempts) >= crashLoopThreshold {
	194	+	backoff := m.computeBackoff(rec.consecutive)
	195	+	rec.backoffEnd = now.Add(backoff)
	196	+	m.mu.Unlock()
	197	+
	198	+	log.Error().Str("domain", domainName).
	199	+	Int("consecutive", rec.consecutive).
	200	+	Dur("backoff", backoff).
	201	+	Msg("monitor: crash loop detected, backing off")
	202	+	return
	203	+	}
	204	+	m.mu.Unlock()
	205	+
	206	+	// Verify container ID still matches tracked state (deploy may have replaced it).
	207	+	m.service.mu.RLock()
	208	+	current := m.service.containers[domainName]
	209	+	m.service.mu.RUnlock()
	210	+	if current == nil \|\| current.ID != containerID {
	211	+	log.Debug().Str("domain", domainName).Msg("monitor: container replaced by deploy, skipping restart")
	212	+	return
	213	+	}
	214	+
	215	+	log.Warn().Str("domain", domainName).Str("container_id", containerID).
	216	+	Int("exit_code", -1). // exact code already logged by inspect
	217	+	Msg("monitor: restarting crashed container")
	218	+
	219	+	if err := m.service.runtime.StartContainer(ctx, containerID); err != nil {
	220	+	log.Warn().Err(err).Str("domain", domainName).Msg("monitor: failed to restart container")
	221	+	}
	222	+	}
	223	+
	224	+	func (m *Monitor) computeBackoff(consecutive int) time.Duration {
	225	+	// Exponential backoff: 1min, 2min, 4min, 8min, cap 15min.
	226	+	base := time.Minute
	227	+	shift := consecutive - crashLoopThreshold
	228	+	if shift < 0 {
	229	+	shift = 0
	230	+	}
	231	+	if shift > 4 {
	232	+	shift = 4 // cap shift to avoid overflow (max 16min, then capped to backoffCap)
	233	+	}
	234	+	backoff := base << uint(shift) // #nosec G115 -- shift is bounded [0,4]
	235	+	if backoff > backoffCap {
	236	+	backoff = backoffCap
	237	+	}
	238	+	return backoff
	239	+	}

@@ -0,0 +1,239 @@
	1	+	package container
	2	+
	3	+	import (
	4	+	"context"
	5	+	"sync"
	6	+	"time"
	7	+
	8	+	"github.com/bnema/zerowrap"
	9	+
	10	+	"github.com/bnema/gordon/internal/domain"
	11	+	)
	12	+
	13	+	const (
	14	+	monitorDefaultInterval = 15 * time.Second
	15	+	crashLoopThreshold = 3
	16	+	crashLoopWindow = 5 * time.Minute
	17	+	backoffCap = 15 * time.Minute
	18	+	stableRunningDuration = 5 * time.Minute
	19	+	)
	20	+
	21	+	// restartRecord tracks restart attempts for crash loop detection.
	22	+	type restartRecord struct {
	23	+	attempts []time.Time
	24	+	backoffEnd time.Time
	25	+	consecutive int
	26	+	lastSeen time.Time // last time the container was seen running
	27	+	}
	28	+
	29	+	// Monitor watches tracked containers and restarts crashed ones.
	30	+	type Monitor struct {
	31	+	service *Service
	32	+	stopCh chan struct{}
	33	+	stopped chan struct{}
	34	+	interval time.Duration
	35	+	mu sync.Mutex
	36	+	history map[string]*restartRecord // keyed by domain
	37	+	}
	38	+
	39	+	// newMonitor creates a new container monitor.
	40	+	func newMonitor(service Service) Monitor {
	41	+	return &Monitor{
	42	+	service: service,
	43	+	stopCh: make(chan struct{}),
	44	+	stopped: make(chan struct{}),
	45	+	interval: monitorDefaultInterval,
	46	+	history: make(map[string]*restartRecord),
	47	+	}
	48	+	}
	49	+
	50	+	// Start begins the background monitoring loop.
	51	+	func (m *Monitor) Start(ctx context.Context) {
	52	+	log := zerowrap.FromCtx(ctx)
	53	+	log.Info().Dur("interval", m.interval).Msg("container monitor started")
	54	+
	55	+	go m.run(ctx)
	56	+	}
	57	+
	58	+	// Stop signals the monitor to stop and waits for it to finish.
	59	+	func (m *Monitor) Stop() {
	60	+	close(m.stopCh)
	61	+	<-m.stopped
	62	+	}
	63	+
	64	+	func (m *Monitor) run(ctx context.Context) {
	65	+	defer close(m.stopped)
	66	+
	67	+	ticker := time.NewTicker(m.interval)
	68	+	defer ticker.Stop()
	69	+
	70	+	for {
	71	+	select {
	72	+	case <-m.stopCh:
	73	+	return
	74	+	case <-ctx.Done():
	75	+	return
	76	+	case <-ticker.C:
	77	+	m.check(ctx)
	78	+	}
	79	+	}
	80	+	}
	81	+
	82	+	func (m *Monitor) check(ctx context.Context) {
	83	+	log := zerowrap.FromCtx(ctx)
	84	+
	85	+	// Snapshot tracked containers under lock.
	86	+	m.service.mu.RLock()
	87	+	snapshot := make(map[string]*domain.Container, len(m.service.containers))
	88	+	for d, c := range m.service.containers {
	89	+	snapshot[d] = c
	90	+	}
	91	+	m.service.mu.RUnlock()
	92	+
	93	+	now := time.Now()
	94	+
	95	+	for domainName, tracked := range snapshot {
	96	+	m.checkContainer(ctx, log, domainName, tracked, now)
	97	+	}
	98	+	}
	99	+
	100	+	func (m Monitor) checkContainer(ctx context.Context, log zerowrap.Logger, domainName string, tracked domain.Container, now time.Time) {
	101	+	if tracked == nil {
	102	+	return
	103	+	}
	104	+
	105	+	inspected, err := m.service.runtime.InspectContainer(ctx, tracked.ID)
	106	+	if err != nil {
	107	+	log.Debug().Err(err).Str("domain", domainName).Str("container_id", tracked.ID).
	108	+	Msg("monitor: failed to inspect container, may have been replaced by deploy")
	109	+	return
	110	+	}
	111	+
	112	+	switch {
	113	+	case inspected.Status == string(domain.ContainerStatusRunning):
	114	+	m.handleRunning(ctx, log, domainName, tracked.ID, now)
	115	+
	116	+	case inspected.Status == string(domain.ContainerStatusExited):
	117	+	if inspected.ExitCode == 0 {
	118	+	// Graceful stop (exit 0): respect user intent, don't restart.
	119	+	log.Debug().Str("domain", domainName).Msg("monitor: container exited gracefully (code 0), skipping")
	120	+	return
	121	+	}
	122	+	// Crashed (non-zero exit): restart if not in backoff.
	123	+	m.handleCrash(ctx, log, domainName, tracked.ID, now)
	124	+	}
	125	+	}
	126	+
	127	+	func (m *Monitor) handleRunning(ctx context.Context, log zerowrap.Logger, domainName, containerID string, now time.Time) {
	128	+	// Check Docker health status for running containers.
	129	+	healthStatus, hasHealthcheck, err := m.service.runtime.GetContainerHealthStatus(ctx, containerID)
	130	+	if err != nil {
	131	+	return
	132	+	}
	133	+
	134	+	if hasHealthcheck && healthStatus == "unhealthy" {
	135	+	log.Warn().Str("domain", domainName).Msg("monitor: container is unhealthy, restarting")
	136	+	if err := m.service.runtime.RestartContainer(ctx, containerID); err != nil {
	137	+	log.Warn().Err(err).Str("domain", domainName).Msg("monitor: failed to restart unhealthy container")
	138	+	}
	139	+	return
	140	+	}
	141	+
	142	+	// Container is running and healthy — clear backoff if stable long enough.
	143	+	m.mu.Lock()
	144	+	rec, exists := m.history[domainName]
	145	+	if exists {
	146	+	if rec.lastSeen.IsZero() {
	147	+	rec.lastSeen = now
	148	+	}
	149	+	if now.Sub(rec.lastSeen) >= stableRunningDuration {
	150	+	// Container has been running for 5+ min — clear crash history.
	151	+	delete(m.history, domainName)
	152	+	m.mu.Unlock()
	153	+	log.Info().Str("domain", domainName).Msg("monitor: container stable, cleared crash history")
	154	+	return
	155	+	}
	156	+	}
	157	+	m.mu.Unlock()
	158	+	}
	159	+
	160	+	func (m *Monitor) handleCrash(ctx context.Context, log zerowrap.Logger, domainName, containerID string, now time.Time) {
	161	+	m.mu.Lock()
	162	+	rec := m.history[domainName]
	163	+	if rec == nil {
	164	+	rec = &restartRecord{}
	165	+	m.history[domainName] = rec
	166	+	}
	167	+
	168	+	// Check backoff.
	169	+	if now.Before(rec.backoffEnd) {
	170	+	m.mu.Unlock()
	171	+	log.Debug().Str("domain", domainName).
	172	+	Time("backoff_until", rec.backoffEnd).
	173	+	Msg("monitor: in crash loop backoff, skipping restart")
	174	+	return
	175	+	}
	176	+
	177	+	// Record this crash attempt.
	178	+	rec.attempts = append(rec.attempts, now)
	179	+	rec.consecutive++
	180	+	rec.lastSeen = time.Time{} // reset stable timer
	181	+
	182	+	// Prune old attempts outside the crash loop window.
	183	+	cutoff := now.Add(-crashLoopWindow)
	184	+	pruned := rec.attempts[:0]
	185	+	for _, t := range rec.attempts {
	186	+	if t.After(cutoff) {
	187	+	pruned = append(pruned, t)
	188	+	}
	189	+	}
	190	+	rec.attempts = pruned
	191	+
	192	+	// Check for crash loop.
	193	+	if len(rec.attempts) >= crashLoopThreshold {
	194	+	backoff := m.computeBackoff(rec.consecutive)
	195	+	rec.backoffEnd = now.Add(backoff)
	196	+	m.mu.Unlock()
	197	+
	198	+	log.Error().Str("domain", domainName).
	199	+	Int("consecutive", rec.consecutive).
	200	+	Dur("backoff", backoff).
	201	+	Msg("monitor: crash loop detected, backing off")
	202	+	return
	203	+	}
	204	+	m.mu.Unlock()
	205	+
	206	+	// Verify container ID still matches tracked state (deploy may have replaced it).
	207	+	m.service.mu.RLock()
	208	+	current := m.service.containers[domainName]
	209	+	m.service.mu.RUnlock()
	210	+	if current == nil \|\| current.ID != containerID {
	211	+	log.Debug().Str("domain", domainName).Msg("monitor: container replaced by deploy, skipping restart")
	212	+	return
	213	+	}
	214	+
	215	+	log.Warn().Str("domain", domainName).Str("container_id", containerID).
	216	+	Int("exit_code", -1). // exact code already logged by inspect
	217	+	Msg("monitor: restarting crashed container")
	218	+
	219	+	if err := m.service.runtime.StartContainer(ctx, containerID); err != nil {
	220	+	log.Warn().Err(err).Str("domain", domainName).Msg("monitor: failed to restart container")
	221	+	}
	222	+	}
	223	+
	224	+	func (m *Monitor) computeBackoff(consecutive int) time.Duration {
	225	+	// Exponential backoff: 1min, 2min, 4min, 8min, cap 15min.
	226	+	base := time.Minute
	227	+	shift := consecutive - crashLoopThreshold
	228	+	if shift < 0 {
	229	+	shift = 0
	230	+	}
	231	+	if shift > 4 {
	232	+	shift = 4 // cap shift to avoid overflow (max 16min, then capped to backoffCap)
	233	+	}
	234	+	backoff := base << uint(shift) // #nosec G115 -- shift is bounded [0,4]
	235	+	if backoff > backoffCap {
	236	+	backoff = backoffCap
	237	+	}
	238	+	return backoff
	239	+	}

A internal/usecase/container/monitor_test.go +324 -0

@@ -0,0 +1,324 @@
	1	+	package container
	2	+
	3	+	import (
	4	+	"context"
	5	+	"testing"
	6	+	"time"
	7	+
	8	+	"github.com/bnema/zerowrap"
	9	+	"github.com/stretchr/testify/assert"
	10	+	"github.com/stretchr/testify/mock"
	11	+
	12	+	"github.com/bnema/gordon/internal/boundaries/out/mocks"
	13	+	"github.com/bnema/gordon/internal/domain"
	14	+	)
	15	+
	16	+	func newTestService(runtime mocks.MockContainerRuntime) Service {
	17	+	return &Service{
	18	+	runtime: runtime,
	19	+	config: Config{},
	20	+	containers: make(map[string]*domain.Container),
	21	+	attachments: make(map[string][]string),
	22	+	}
	23	+	}
	24	+
	25	+	func monitorTestContext() context.Context {
	26	+	return zerowrap.WithCtx(context.Background(), zerowrap.Default())
	27	+	}
	28	+
	29	+	func TestMonitor_RestartsCrashedContainer(t *testing.T) {
	30	+	runtime := mocks.NewMockContainerRuntime(t)
	31	+	svc := newTestService(runtime)
	32	+
	33	+	// Track a container.
	34	+	svc.containers["app.example.com"] = &domain.Container{
	35	+	ID: "ctr-1",
	36	+	Image: "myapp:latest",
	37	+	}
	38	+
	39	+	// Inspect returns exited with non-zero exit code.
	40	+	runtime.EXPECT().InspectContainer(mock.Anything, "ctr-1").Return(&domain.Container{
	41	+	ID: "ctr-1",
	42	+	Status: string(domain.ContainerStatusExited),
	43	+	ExitCode: 1,
	44	+	}, nil)
	45	+
	46	+	// Monitor should restart it.
	47	+	runtime.EXPECT().StartContainer(mock.Anything, "ctr-1").Return(nil)
	48	+
	49	+	m := newMonitor(svc)
	50	+	m.check(monitorTestContext())
	51	+
	52	+	runtime.AssertExpectations(t)
	53	+	}
	54	+
	55	+	func TestMonitor_SkipsGracefulExit(t *testing.T) {
	56	+	runtime := mocks.NewMockContainerRuntime(t)
	57	+	svc := newTestService(runtime)
	58	+
	59	+	svc.containers["app.example.com"] = &domain.Container{
	60	+	ID: "ctr-1",
	61	+	Image: "myapp:latest",
	62	+	}
	63	+
	64	+	// Exited with code 0 — graceful stop.
	65	+	runtime.EXPECT().InspectContainer(mock.Anything, "ctr-1").Return(&domain.Container{
	66	+	ID: "ctr-1",
	67	+	Status: string(domain.ContainerStatusExited),
	68	+	ExitCode: 0,
	69	+	}, nil)
	70	+
	71	+	// Should NOT call StartContainer.
	72	+
	73	+	m := newMonitor(svc)
	74	+	m.check(monitorTestContext())
	75	+
	76	+	runtime.AssertExpectations(t)
	77	+	runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
	78	+	}
	79	+
	80	+	func TestMonitor_SkipsRunningContainer(t *testing.T) {
	81	+	runtime := mocks.NewMockContainerRuntime(t)
	82	+	svc := newTestService(runtime)
	83	+
	84	+	svc.containers["app.example.com"] = &domain.Container{
	85	+	ID: "ctr-1",
	86	+	Image: "myapp:latest",
	87	+	}
	88	+
	89	+	runtime.EXPECT().InspectContainer(mock.Anything, "ctr-1").Return(&domain.Container{
	90	+	ID: "ctr-1",
	91	+	Status: string(domain.ContainerStatusRunning),
	92	+	}, nil)
	93	+
	94	+	// Running container — check health status.
	95	+	runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "ctr-1").Return("healthy", true, nil)
	96	+
	97	+	m := newMonitor(svc)
	98	+	m.check(monitorTestContext())
	99	+
	100	+	runtime.AssertExpectations(t)
	101	+	runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
	102	+	}
	103	+
	104	+	func TestMonitor_RestartsUnhealthyContainer(t *testing.T) {
	105	+	runtime := mocks.NewMockContainerRuntime(t)
	106	+	svc := newTestService(runtime)
	107	+
	108	+	svc.containers["app.example.com"] = &domain.Container{
	109	+	ID: "ctr-1",
	110	+	Image: "myapp:latest",
	111	+	}
	112	+
	113	+	runtime.EXPECT().InspectContainer(mock.Anything, "ctr-1").Return(&domain.Container{
	114	+	ID: "ctr-1",
	115	+	Status: string(domain.ContainerStatusRunning),
	116	+	}, nil)
	117	+
	118	+	// Running but Docker health says unhealthy.
	119	+	runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "ctr-1").Return("unhealthy", true, nil)
	120	+
	121	+	// Should restart the unhealthy container.
	122	+	runtime.EXPECT().RestartContainer(mock.Anything, "ctr-1").Return(nil)
	123	+
	124	+	m := newMonitor(svc)
	125	+	m.check(monitorTestContext())
	126	+
	127	+	runtime.AssertExpectations(t)
	128	+	}
	129	+
	130	+	func TestMonitor_CrashLoopBackoff(t *testing.T) {
	131	+	runtime := mocks.NewMockContainerRuntime(t)
	132	+	svc := newTestService(runtime)
	133	+
	134	+	svc.containers["app.example.com"] = &domain.Container{
	135	+	ID: "ctr-1",
	136	+	Image: "myapp:latest",
	137	+	}
	138	+
	139	+	m := newMonitor(svc)
	140	+	ctx := monitorTestContext()
	141	+	now := time.Now()
	142	+
	143	+	// Simulate 3 crashes within the crash loop window.
	144	+	m.mu.Lock()
	145	+	m.history["app.example.com"] = &restartRecord{
	146	+	attempts: []time.Time{now.Add(-2 * time.Minute), now.Add(-1 * time.Minute)},
	147	+	consecutive: 2,
	148	+	}
	149	+	m.mu.Unlock()
	150	+
	151	+	// Third crash — this should trigger backoff.
	152	+	runtime.EXPECT().InspectContainer(mock.Anything, "ctr-1").Return(&domain.Container{
	153	+	ID: "ctr-1",
	154	+	Status: string(domain.ContainerStatusExited),
	155	+	ExitCode: 137,
	156	+	}, nil)
	157	+
	158	+	// Should NOT call StartContainer because crash loop is detected.
	159	+	m.check(ctx)
	160	+
	161	+	runtime.AssertExpectations(t)
	162	+	runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
	163	+
	164	+	// Verify backoff was set.
	165	+	m.mu.Lock()
	166	+	rec := m.history["app.example.com"]
	167	+	assert.False(t, rec.backoffEnd.IsZero(), "backoff should be set after crash loop detection")
	168	+	assert.True(t, rec.backoffEnd.After(time.Now()), "backoff end should be in the future")
	169	+	m.mu.Unlock()
	170	+	}
	171	+
	172	+	func TestMonitor_BackoffPreventsRestart(t *testing.T) {
	173	+	runtime := mocks.NewMockContainerRuntime(t)
	174	+	svc := newTestService(runtime)
	175	+
	176	+	svc.containers["app.example.com"] = &domain.Container{
	177	+	ID: "ctr-1",
	178	+	Image: "myapp:latest",
	179	+	}
	180	+
	181	+	m := newMonitor(svc)
	182	+
	183	+	// Set backoff into the future.
	184	+	m.mu.Lock()
	185	+	m.history["app.example.com"] = &restartRecord{
	186	+	backoffEnd: time.Now().Add(5 * time.Minute),
	187	+	consecutive: 3,
	188	+	}
	189	+	m.mu.Unlock()
	190	+
	191	+	// Container is crashed.
	192	+	runtime.EXPECT().InspectContainer(mock.Anything, "ctr-1").Return(&domain.Container{
	193	+	ID: "ctr-1",
	194	+	Status: string(domain.ContainerStatusExited),
	195	+	ExitCode: 1,
	196	+	}, nil)
	197	+
	198	+	// Should NOT restart — in backoff.
	199	+	m.check(monitorTestContext())
	200	+
	201	+	runtime.AssertExpectations(t)
	202	+	runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
	203	+	}
	204	+
	205	+	func TestMonitor_ClearsCrashHistoryAfterStableRunning(t *testing.T) {
	206	+	runtime := mocks.NewMockContainerRuntime(t)
	207	+	svc := newTestService(runtime)
	208	+
	209	+	svc.containers["app.example.com"] = &domain.Container{
	210	+	ID: "ctr-1",
	211	+	Image: "myapp:latest",
	212	+	}
	213	+
	214	+	m := newMonitor(svc)
	215	+
	216	+	// Container has been running for longer than stableRunningDuration.
	217	+	m.mu.Lock()
	218	+	m.history["app.example.com"] = &restartRecord{
	219	+	attempts: []time.Time{time.Now().Add(-10 * time.Minute)},
	220	+	consecutive: 2,
	221	+	lastSeen: time.Now().Add(-6 * time.Minute), // 6 min ago > stableRunningDuration (5 min)
	222	+	}
	223	+	m.mu.Unlock()
	224	+
	225	+	runtime.EXPECT().InspectContainer(mock.Anything, "ctr-1").Return(&domain.Container{
	226	+	ID: "ctr-1",
	227	+	Status: string(domain.ContainerStatusRunning),
	228	+	}, nil)
	229	+
	230	+	runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "ctr-1").Return("healthy", true, nil)
	231	+
	232	+	m.check(monitorTestContext())
	233	+
	234	+	// History should be cleared.
	235	+	m.mu.Lock()
	236	+	_, exists := m.history["app.example.com"]
	237	+	m.mu.Unlock()
	238	+	assert.False(t, exists, "crash history should be cleared after stable running")
	239	+
	240	+	runtime.AssertExpectations(t)
	241	+	}
	242	+
	243	+	func TestMonitor_SkipsReplacedContainer(t *testing.T) {
	244	+	runtime := mocks.NewMockContainerRuntime(t)
	245	+	svc := newTestService(runtime)
	246	+
	247	+	// Track the OLD container ID.
	248	+	svc.containers["app.example.com"] = &domain.Container{
	249	+	ID: "ctr-NEW",
	250	+	Image: "myapp:latest",
	251	+	}
	252	+
	253	+	m := newMonitor(svc)
	254	+
	255	+	// Monitor snapshot captured "ctr-OLD" but a deploy replaced it with "ctr-NEW".
	256	+	// We simulate this by directly calling checkContainer with the old ID.
	257	+	ctx := monitorTestContext()
	258	+	log := zerowrap.FromCtx(ctx)
	259	+
	260	+	// Inspect the old container — it's exited.
	261	+	runtime.EXPECT().InspectContainer(mock.Anything, "ctr-OLD").Return(&domain.Container{
	262	+	ID: "ctr-OLD",
	263	+	Status: string(domain.ContainerStatusExited),
	264	+	ExitCode: 1,
	265	+	}, nil)
	266	+
	267	+	// Should NOT restart — tracked container has been replaced.
	268	+	oldTracked := &domain.Container{ID: "ctr-OLD", Image: "myapp:latest"}
	269	+	m.checkContainer(ctx, log, "app.example.com", oldTracked, time.Now())
	270	+
	271	+	runtime.AssertExpectations(t)
	272	+	runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
	273	+	}
	274	+
	275	+	func TestMonitor_ComputeBackoff(t *testing.T) {
	276	+	m := &Monitor{}
	277	+
	278	+	tests := []struct {
	279	+	consecutive int
	280	+	expected time.Duration
	281	+	}{
	282	+	{3, 1 * time.Minute}, // first backoff
	283	+	{4, 2 * time.Minute}, // second
	284	+	{5, 4 * time.Minute}, // third
	285	+	{6, 8 * time.Minute}, // fourth
	286	+	{7, 15 * time.Minute}, // capped
	287	+	{10, 15 * time.Minute}, // still capped
	288	+	}
	289	+
	290	+	for _, tt := range tests {
	291	+	got := m.computeBackoff(tt.consecutive)
	292	+	assert.Equal(t, tt.expected, got, "consecutive=%d", tt.consecutive)
	293	+	}
	294	+	}
	295	+
	296	+	func TestMonitor_StartStop(t *testing.T) {
	297	+	runtime := mocks.NewMockContainerRuntime(t)
	298	+	svc := newTestService(runtime)
	299	+
	300	+	m := newMonitor(svc)
	301	+	m.interval = 50 * time.Millisecond // fast ticking for test
	302	+
	303	+	ctx, cancel := context.WithCancel(monitorTestContext())
	304	+	defer cancel()
	305	+
	306	+	m.Start(ctx)
	307	+
	308	+	// Let it tick a couple times.
	309	+	time.Sleep(150 * time.Millisecond)
	310	+
	311	+	// Stop should return promptly.
	312	+	done := make(chan struct{})
	313	+	go func() {
	314	+	m.Stop()
	315	+	close(done)
	316	+	}()
	317	+
	318	+	select {
	319	+	case <-done:
	320	+	// OK
	321	+	case <-time.After(2 * time.Second):
	322	+	t.Fatal("monitor.Stop() did not return within 2 seconds")
	323	+	}
	324	+	}

@@ -0,0 +1,324 @@
	1	+	package container
	2	+
	3	+	import (
	4	+	"context"
	5	+	"testing"
	6	+	"time"
	7	+
	8	+	"github.com/bnema/zerowrap"
	9	+	"github.com/stretchr/testify/assert"
	10	+	"github.com/stretchr/testify/mock"
	11	+
	12	+	"github.com/bnema/gordon/internal/boundaries/out/mocks"
	13	+	"github.com/bnema/gordon/internal/domain"
	14	+	)
	15	+
	16	+	func newTestService(runtime mocks.MockContainerRuntime) Service {
	17	+	return &Service{
	18	+	runtime: runtime,
	19	+	config: Config{},
	20	+	containers: make(map[string]*domain.Container),
	21	+	attachments: make(map[string][]string),
	22	+	}
	23	+	}
	24	+
	25	+	func monitorTestContext() context.Context {
	26	+	return zerowrap.WithCtx(context.Background(), zerowrap.Default())
	27	+	}
	28	+
	29	+	func TestMonitor_RestartsCrashedContainer(t *testing.T) {
	30	+	runtime := mocks.NewMockContainerRuntime(t)
	31	+	svc := newTestService(runtime)
	32	+
	33	+	// Track a container.
	34	+	svc.containers["app.example.com"] = &domain.Container{
	35	+	ID: "ctr-1",
	36	+	Image: "myapp:latest",
	37	+	}
	38	+
	39	+	// Inspect returns exited with non-zero exit code.
	40	+	runtime.EXPECT().InspectContainer(mock.Anything, "ctr-1").Return(&domain.Container{
	41	+	ID: "ctr-1",
	42	+	Status: string(domain.ContainerStatusExited),
	43	+	ExitCode: 1,
	44	+	}, nil)
	45	+
	46	+	// Monitor should restart it.
	47	+	runtime.EXPECT().StartContainer(mock.Anything, "ctr-1").Return(nil)
	48	+
	49	+	m := newMonitor(svc)
	50	+	m.check(monitorTestContext())
	51	+
	52	+	runtime.AssertExpectations(t)
	53	+	}
	54	+
	55	+	func TestMonitor_SkipsGracefulExit(t *testing.T) {
	56	+	runtime := mocks.NewMockContainerRuntime(t)
	57	+	svc := newTestService(runtime)
	58	+
	59	+	svc.containers["app.example.com"] = &domain.Container{
	60	+	ID: "ctr-1",
	61	+	Image: "myapp:latest",
	62	+	}
	63	+
	64	+	// Exited with code 0 — graceful stop.
	65	+	runtime.EXPECT().InspectContainer(mock.Anything, "ctr-1").Return(&domain.Container{
	66	+	ID: "ctr-1",
	67	+	Status: string(domain.ContainerStatusExited),
	68	+	ExitCode: 0,
	69	+	}, nil)
	70	+
	71	+	// Should NOT call StartContainer.
	72	+
	73	+	m := newMonitor(svc)
	74	+	m.check(monitorTestContext())
	75	+
	76	+	runtime.AssertExpectations(t)
	77	+	runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
	78	+	}
	79	+
	80	+	func TestMonitor_SkipsRunningContainer(t *testing.T) {
	81	+	runtime := mocks.NewMockContainerRuntime(t)
	82	+	svc := newTestService(runtime)
	83	+
	84	+	svc.containers["app.example.com"] = &domain.Container{
	85	+	ID: "ctr-1",
	86	+	Image: "myapp:latest",
	87	+	}
	88	+
	89	+	runtime.EXPECT().InspectContainer(mock.Anything, "ctr-1").Return(&domain.Container{
	90	+	ID: "ctr-1",
	91	+	Status: string(domain.ContainerStatusRunning),
	92	+	}, nil)
	93	+
	94	+	// Running container — check health status.
	95	+	runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "ctr-1").Return("healthy", true, nil)
	96	+
	97	+	m := newMonitor(svc)
	98	+	m.check(monitorTestContext())
	99	+
	100	+	runtime.AssertExpectations(t)
	101	+	runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
	102	+	}
	103	+
	104	+	func TestMonitor_RestartsUnhealthyContainer(t *testing.T) {
	105	+	runtime := mocks.NewMockContainerRuntime(t)
	106	+	svc := newTestService(runtime)
	107	+
	108	+	svc.containers["app.example.com"] = &domain.Container{
	109	+	ID: "ctr-1",
	110	+	Image: "myapp:latest",
	111	+	}
	112	+
	113	+	runtime.EXPECT().InspectContainer(mock.Anything, "ctr-1").Return(&domain.Container{
	114	+	ID: "ctr-1",
	115	+	Status: string(domain.ContainerStatusRunning),
	116	+	}, nil)
	117	+
	118	+	// Running but Docker health says unhealthy.
	119	+	runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "ctr-1").Return("unhealthy", true, nil)
	120	+
	121	+	// Should restart the unhealthy container.
	122	+	runtime.EXPECT().RestartContainer(mock.Anything, "ctr-1").Return(nil)
	123	+
	124	+	m := newMonitor(svc)
	125	+	m.check(monitorTestContext())
	126	+
	127	+	runtime.AssertExpectations(t)
	128	+	}
	129	+
	130	+	func TestMonitor_CrashLoopBackoff(t *testing.T) {
	131	+	runtime := mocks.NewMockContainerRuntime(t)
	132	+	svc := newTestService(runtime)
	133	+
	134	+	svc.containers["app.example.com"] = &domain.Container{
	135	+	ID: "ctr-1",
	136	+	Image: "myapp:latest",
	137	+	}
	138	+
	139	+	m := newMonitor(svc)
	140	+	ctx := monitorTestContext()
	141	+	now := time.Now()
	142	+
	143	+	// Simulate 3 crashes within the crash loop window.
	144	+	m.mu.Lock()
	145	+	m.history["app.example.com"] = &restartRecord{
	146	+	attempts: []time.Time{now.Add(-2 * time.Minute), now.Add(-1 * time.Minute)},
	147	+	consecutive: 2,
	148	+	}
	149	+	m.mu.Unlock()
	150	+
	151	+	// Third crash — this should trigger backoff.
	152	+	runtime.EXPECT().InspectContainer(mock.Anything, "ctr-1").Return(&domain.Container{
	153	+	ID: "ctr-1",
	154	+	Status: string(domain.ContainerStatusExited),
	155	+	ExitCode: 137,
	156	+	}, nil)
	157	+
	158	+	// Should NOT call StartContainer because crash loop is detected.
	159	+	m.check(ctx)
	160	+
	161	+	runtime.AssertExpectations(t)
	162	+	runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
	163	+
	164	+	// Verify backoff was set.
	165	+	m.mu.Lock()
	166	+	rec := m.history["app.example.com"]
	167	+	assert.False(t, rec.backoffEnd.IsZero(), "backoff should be set after crash loop detection")
	168	+	assert.True(t, rec.backoffEnd.After(time.Now()), "backoff end should be in the future")
	169	+	m.mu.Unlock()
	170	+	}
	171	+
	172	+	func TestMonitor_BackoffPreventsRestart(t *testing.T) {
	173	+	runtime := mocks.NewMockContainerRuntime(t)
	174	+	svc := newTestService(runtime)
	175	+
	176	+	svc.containers["app.example.com"] = &domain.Container{
	177	+	ID: "ctr-1",
	178	+	Image: "myapp:latest",
	179	+	}
	180	+
	181	+	m := newMonitor(svc)
	182	+
	183	+	// Set backoff into the future.
	184	+	m.mu.Lock()
	185	+	m.history["app.example.com"] = &restartRecord{
	186	+	backoffEnd: time.Now().Add(5 * time.Minute),
	187	+	consecutive: 3,
	188	+	}
	189	+	m.mu.Unlock()
	190	+
	191	+	// Container is crashed.
	192	+	runtime.EXPECT().InspectContainer(mock.Anything, "ctr-1").Return(&domain.Container{
	193	+	ID: "ctr-1",
	194	+	Status: string(domain.ContainerStatusExited),
	195	+	ExitCode: 1,
	196	+	}, nil)
	197	+
	198	+	// Should NOT restart — in backoff.
	199	+	m.check(monitorTestContext())
	200	+
	201	+	runtime.AssertExpectations(t)
	202	+	runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
	203	+	}
	204	+
	205	+	func TestMonitor_ClearsCrashHistoryAfterStableRunning(t *testing.T) {
	206	+	runtime := mocks.NewMockContainerRuntime(t)
	207	+	svc := newTestService(runtime)
	208	+
	209	+	svc.containers["app.example.com"] = &domain.Container{
	210	+	ID: "ctr-1",
	211	+	Image: "myapp:latest",
	212	+	}
	213	+
	214	+	m := newMonitor(svc)
	215	+
	216	+	// Container has been running for longer than stableRunningDuration.
	217	+	m.mu.Lock()
	218	+	m.history["app.example.com"] = &restartRecord{
	219	+	attempts: []time.Time{time.Now().Add(-10 * time.Minute)},
	220	+	consecutive: 2,
	221	+	lastSeen: time.Now().Add(-6 * time.Minute), // 6 min ago > stableRunningDuration (5 min)
	222	+	}
	223	+	m.mu.Unlock()
	224	+
	225	+	runtime.EXPECT().InspectContainer(mock.Anything, "ctr-1").Return(&domain.Container{
	226	+	ID: "ctr-1",
	227	+	Status: string(domain.ContainerStatusRunning),
	228	+	}, nil)
	229	+
	230	+	runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "ctr-1").Return("healthy", true, nil)
	231	+
	232	+	m.check(monitorTestContext())
	233	+
	234	+	// History should be cleared.
	235	+	m.mu.Lock()
	236	+	_, exists := m.history["app.example.com"]
	237	+	m.mu.Unlock()
	238	+	assert.False(t, exists, "crash history should be cleared after stable running")
	239	+
	240	+	runtime.AssertExpectations(t)
	241	+	}
	242	+
	243	+	func TestMonitor_SkipsReplacedContainer(t *testing.T) {
	244	+	runtime := mocks.NewMockContainerRuntime(t)
	245	+	svc := newTestService(runtime)
	246	+
	247	+	// Track the OLD container ID.
	248	+	svc.containers["app.example.com"] = &domain.Container{
	249	+	ID: "ctr-NEW",
	250	+	Image: "myapp:latest",
	251	+	}
	252	+
	253	+	m := newMonitor(svc)
	254	+
	255	+	// Monitor snapshot captured "ctr-OLD" but a deploy replaced it with "ctr-NEW".
	256	+	// We simulate this by directly calling checkContainer with the old ID.
	257	+	ctx := monitorTestContext()
	258	+	log := zerowrap.FromCtx(ctx)
	259	+
	260	+	// Inspect the old container — it's exited.
	261	+	runtime.EXPECT().InspectContainer(mock.Anything, "ctr-OLD").Return(&domain.Container{
	262	+	ID: "ctr-OLD",
	263	+	Status: string(domain.ContainerStatusExited),
	264	+	ExitCode: 1,
	265	+	}, nil)
	266	+
	267	+	// Should NOT restart — tracked container has been replaced.
	268	+	oldTracked := &domain.Container{ID: "ctr-OLD", Image: "myapp:latest"}
	269	+	m.checkContainer(ctx, log, "app.example.com", oldTracked, time.Now())
	270	+
	271	+	runtime.AssertExpectations(t)
	272	+	runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
	273	+	}
	274	+
	275	+	func TestMonitor_ComputeBackoff(t *testing.T) {
	276	+	m := &Monitor{}
	277	+
	278	+	tests := []struct {
	279	+	consecutive int
	280	+	expected time.Duration
	281	+	}{
	282	+	{3, 1 * time.Minute}, // first backoff
	283	+	{4, 2 * time.Minute}, // second
	284	+	{5, 4 * time.Minute}, // third
	285	+	{6, 8 * time.Minute}, // fourth
	286	+	{7, 15 * time.Minute}, // capped
	287	+	{10, 15 * time.Minute}, // still capped
	288	+	}
	289	+
	290	+	for _, tt := range tests {
	291	+	got := m.computeBackoff(tt.consecutive)
	292	+	assert.Equal(t, tt.expected, got, "consecutive=%d", tt.consecutive)
	293	+	}
	294	+	}
	295	+
	296	+	func TestMonitor_StartStop(t *testing.T) {
	297	+	runtime := mocks.NewMockContainerRuntime(t)
	298	+	svc := newTestService(runtime)
	299	+
	300	+	m := newMonitor(svc)
	301	+	m.interval = 50 * time.Millisecond // fast ticking for test
	302	+
	303	+	ctx, cancel := context.WithCancel(monitorTestContext())
	304	+	defer cancel()
	305	+
	306	+	m.Start(ctx)
	307	+
	308	+	// Let it tick a couple times.
	309	+	time.Sleep(150 * time.Millisecond)
	310	+
	311	+	// Stop should return promptly.
	312	+	done := make(chan struct{})
	313	+	go func() {
	314	+	m.Stop()
	315	+	close(done)
	316	+	}()
	317	+
	318	+	select {
	319	+	case <-done:
	320	+	// OK
	321	+	case <-time.After(2 * time.Second):
	322	+	t.Fatal("monitor.Stop() did not return within 2 seconds")
	323	+	}
	324	+	}

M internal/usecase/container/service.go +75 -154


@@ -56,28 +56,6 @@ const (
56	56		internalPullMaxAttempts = 3
57	57		)
58	58
59		-	// healthTimeoutError is returned when a container's health check times out.
60		-	// It carries the last observed health status to distinguish retryable (starting)
61		-	// from fatal (unhealthy) timeouts.
62		-	type healthTimeoutError struct {
63		-	lastStatus string
64		-	timeout time.Duration
65		-	}
66		-
67		-	func (e *healthTimeoutError) Error() string {
68		-	return fmt.Sprintf("container healthcheck timeout after %s (last status: %s)", e.timeout, e.lastStatus)
69		-	}
70		-
71		-	// isHealthTimeoutStarting returns true if err is a health timeout where the
72		-	// container was still in "starting" state (i.e., not unhealthy, just slow).
73		-	func isHealthTimeoutStarting(err error) bool {
74		-	var hte *healthTimeoutError
75		-	if errors.As(err, &hte) {
76		-	return hte.lastStatus == "starting"
77		-	}
78		-	return false
79		-	}
80		-
81	59		// Service implements the ContainerService interface.
82	60		type Service struct {
83	61		runtime out.ContainerRuntime

@@ -91,6 +69,7 @@ type Service struct {
91	69		attachments map[string][]string // ownerDomain → []containerIDs
92	70		mu sync.RWMutex
93	71		deployMu sync.Map // per-domain deploy locks (domain → *domainDeployLock)
	72	+	monitor *Monitor
94	73		}
95	74
96	75		// domainDeployLock is a context-aware mutex using a buffered channel.

@@ -369,13 +348,11 @@ func (s *Service) createStartedContainer(ctx context.Context, route domain.Route
369	348		s.runtime.RemoveContainer(ctx, newContainer.ID, true)
370	349		return nil, log.WrapErr(err, "failed to start container")
371	350		}
372		-	if err := s.waitForReady(ctx, newContainer.ID); err != nil {
373		-	// If health timed out but the container is still starting (not unhealthy),
374		-	// keep it running for potential retry by the caller.
375		-	if !isHealthTimeoutStarting(err) {
	351	+	if !domain.IsSkipReadiness(ctx) {
	352	+	if err := s.waitForReady(ctx, newContainer.ID); err != nil {
376	353		s.cleanupFailedContainer(ctx, newContainer.ID)
	354	+	return nil, log.WrapErr(err, "container failed readiness check")
377	355		}
378		-	return nil, log.WrapErr(err, "container failed readiness check")
379	356		}
380	357
381	358		inspected, err := s.runtime.InspectContainer(ctx, newContainer.ID)

@@ -831,6 +808,8 @@ func (s *Service) SyncContainers(ctx context.Context) error {
831	808		}
832	809
833	810		// AutoStart starts containers for the provided routes that aren't running.
	811	+	// It skips readiness checks to avoid blocking boot; the background monitor
	812	+	// handles crash recovery.
834	813		func (s *Service) AutoStart(ctx context.Context, routes []domain.Route) error {
835	814		ctx = zerowrap.CtxWithFields(ctx, map[string]any{
836	815		zerowrap.FieldLayer: "usecase",

@@ -840,47 +819,55 @@ func (s *Service) AutoStart(ctx context.Context, routes []domain.Route) error {
840	819
841	820		log.Info().Int("route_count", len(routes)).Msg("auto-starting containers for configured routes")
842	821
843		-	var started, skipped, errCount int
844		-	var retryable []domain.Route
845		-
	822	+	// Filter out routes that already have a running container.
	823	+	var pending []domain.Route
	824	+	skipped := 0
846	825		for _, route := range routes {
847	826		if _, exists := s.Get(ctx, route.Domain); exists {
848	827		log.Debug().Str("domain", route.Domain).Msg("container already running, skipping")
849	828		skipped++
850		-	continue
	829	+	} else {
	830	+	pending = append(pending, route)
851	831		}
	832	+	}
852	833
853		-	log.Info().
854		-	Str("domain", route.Domain).
855		-	Str("image", route.Image).
856		-	Msg("auto-starting container for route")
857		-
858		-	if _, err := s.Deploy(ctx, route); err != nil {
859		-	if isHealthTimeoutStarting(err) {
860		-	log.Warn().Str("domain", route.Domain).Msg("health check still starting, will retry")
861		-	retryable = append(retryable, route)
862		-	} else {
863		-	log.Warn().Err(err).Str("domain", route.Domain).Msg("failed to auto-start container")
864		-	errCount++
865		-	}
866		-	continue
867		-	}
	834	+	if len(pending) == 0 {
	835	+	log.Info().Int("skipped", skipped).Msg("auto-start completed, all containers already running")
	836	+	return nil
	837	+	}
868	838
869		-	started++
	839	+	// Deploy all pending routes concurrently with readiness checks skipped.
	840	+	deployCtx := domain.WithSkipReadiness(ctx)
	841	+
	842	+	type result struct {
	843	+	route domain.Route
	844	+	err error
870	845		}
	846	+	results := make(chan result, len(pending))
	847	+	sem := make(chan struct{}, 4) // limit concurrency
871	848
872		-	// Retry routes whose containers are still starting with extended timeout.
873		-	if len(retryable) > 0 {
874		-	log.Info().Int("retry_count", len(retryable)).Msg("retrying slow-starting containers with extended timeout")
875		-	retryCtx := domain.WithHealthTimeoutMultiplier(ctx, 2.0)
	849	+	for _, route := range pending {
	850	+	sem <- struct{}{}
	851	+	go func(r domain.Route) {
	852	+	defer func() { <-sem }()
876	853
877		-	for _, route := range retryable {
878		-	// Container is still running from first attempt, try readiness again.
879		-	if _, err := s.retryReadiness(retryCtx, route); err != nil {
880		-	log.Warn().Err(err).Str("domain", route.Domain).Msg("failed to auto-start container after retry")
881		-	errCount++
882		-	continue
	854	+	log.Info().Str("domain", r.Domain).Str("image", r.Image).Msg("auto-starting container for route")
	855	+
	856	+	if _, err := s.Deploy(deployCtx, r); err != nil {
	857	+	results <- result{route: r, err: err}
	858	+	return
883	859		}
	860	+	results <- result{route: r}
	861	+	}(route)
	862	+	}
	863	+
	864	+	var started, errCount int
	865	+	for range len(pending) {
	866	+	res := <-results
	867	+	if res.err != nil {
	868	+	log.Warn().Err(res.err).Str("domain", res.route.Domain).Msg("failed to auto-start container")
	869	+	errCount++
	870	+	} else {
884	871		started++
885	872		}
886	873		}

@@ -897,54 +884,6 @@ func (s *Service) AutoStart(ctx context.Context, routes []domain.Route) error {
897	884		return nil
898	885		}
899	886
900		-	// retryReadiness re-checks readiness for a container that was left running
901		-	// after a health timeout during auto-start. If readiness succeeds, the
902		-	// container is activated. If it fails, the container is cleaned up.
903		-	func (s Service) retryReadiness(ctx context.Context, route domain.Route) (domain.Container, error) {
904		-	log := zerowrap.FromCtx(ctx)
905		-
906		-	unlock, err := s.acquireDomainDeployLock(ctx, route.Domain)
907		-	if err != nil {
908		-	return nil, err
909		-	}
910		-	defer unlock()
911		-
912		-	// Find the container that was left running from the first attempt.
913		-	containers, err := s.runtime.ListContainers(ctx, false)
914		-	if err != nil {
915		-	return nil, err
916		-	}
917		-
918		-	var containerID string
919		-	for _, c := range containers {
920		-	if c.Labels != nil && c.Labels[domain.LabelRoute] == route.Domain && c.Status == string(domain.ContainerStatusRunning) {
921		-	containerID = c.ID
922		-	break
923		-	}
924		-	}
925		-	if containerID == "" {
926		-	// Container is no longer running, do a full deploy.
927		-	return s.Deploy(ctx, route)
928		-	}
929		-
930		-	log.Info().Str("domain", route.Domain).Str("container_id", containerID).Msg("retrying readiness check")
931		-
932		-	if err := s.waitForReady(ctx, containerID); err != nil {
933		-	s.cleanupFailedContainer(ctx, containerID)
934		-	return nil, log.WrapErr(err, "container failed readiness check on retry")
935		-	}
936		-
937		-	inspected, err := s.runtime.InspectContainer(ctx, containerID)
938		-	if err != nil {
939		-	s.cleanupFailedContainer(ctx, containerID)
940		-	return nil, log.WrapErr(err, "failed to inspect container after retry")
941		-	}
942		-
943		-	s.activateDeployedContainer(ctx, route.Domain, inspected)
944		-
945		-	return inspected, nil
946		-	}
947		-
948	887		// Shutdown gracefully shuts down all managed containers.
949	888		func (s *Service) Shutdown(ctx context.Context) error {
950	889		ctx = zerowrap.CtxWithFields(ctx, map[string]any{

@@ -954,9 +893,8 @@ func (s *Service) Shutdown(ctx context.Context) error {
954	893		log := zerowrap.FromCtx(ctx)
955	894		log.Info().Msg("shutting down container manager...")
956	895
957		-	if err := s.stopAllManagedContainers(ctx); err != nil {
958		-	log.WrapErr(err, "failed to stop all containers during shutdown")
959		-	}
	896	+	// Containers are left running across Gordon restarts.
	897	+	// SyncContainers + AutoStart will pick them back up on next boot.
960	898
961	899		// Close log writer to stop all log collection
962	900		if s.logWriter != nil {

@@ -969,6 +907,33 @@ func (s *Service) Shutdown(ctx context.Context) error {
969	907		return nil
970	908		}
971	909
	910	+	// StartMonitor begins background monitoring of tracked containers.
	911	+	// The monitor restarts crashed containers and detects crash loops.
	912	+	// Safe to call multiple times; subsequent calls are no-ops.
	913	+	func (s *Service) StartMonitor(ctx context.Context) {
	914	+	s.mu.Lock()
	915	+	if s.monitor != nil {
	916	+	s.mu.Unlock()
	917	+	return
	918	+	}
	919	+	s.monitor = newMonitor(s)
	920	+	m := s.monitor
	921	+	s.mu.Unlock()
	922	+	m.Start(ctx)
	923	+	}
	924	+
	925	+	// StopMonitor stops the background container monitor.
	926	+	// Safe to call multiple times; subsequent calls are no-ops.
	927	+	func (s *Service) StopMonitor() {
	928	+	s.mu.Lock()
	929	+	m := s.monitor
	930	+	s.monitor = nil
	931	+	s.mu.Unlock()
	932	+	if m != nil {
	933	+	m.Stop()
	934	+	}
	935	+	}
	936	+
972	937		// UpdateConfig updates the service configuration.
973	938		func (s *Service) UpdateConfig(config Config) {
974	939		s.mu.Lock()

@@ -1816,46 +1781,6 @@ func (s *Service) deployAttachedService(ctx context.Context, ownerDomain, servic
1816	1781		return nil
1817	1782		}
1818	1783
1819		-	func (s *Service) stopAllManagedContainers(ctx context.Context) error {
1820		-	log := zerowrap.FromCtx(ctx)
1821		-
1822		-	s.mu.RLock()
1823		-	containers := make(map[string]*domain.Container, len(s.containers))
1824		-	maps.Copy(containers, s.containers)
1825		-	s.mu.RUnlock()
1826		-
1827		-	if len(containers) == 0 {
1828		-	log.Info().Msg("no managed containers to stop")
1829		-	return nil
1830		-	}
1831		-
1832		-	log.Info().Int(zerowrap.FieldCount, len(containers)).Msg("stopping all managed containers...")
1833		-
1834		-	errorCount := 0
1835		-	for d, c := range containers {
1836		-	log.Info().Str("domain", d).Str(zerowrap.FieldEntityID, c.ID).Msg("stopping managed container")
1837		-
1838		-	if err := s.runtime.StopContainer(ctx, c.ID); err != nil {
1839		-	log.WrapErrWithFields(err, "failed to stop container", map[string]any{"domain": d, zerowrap.FieldEntityID: c.ID})
1840		-	errorCount++
1841		-	continue
1842		-	}
1843		-
1844		-	s.mu.Lock()
1845		-	delete(s.containers, d)
1846		-	s.mu.Unlock()
1847		-
1848		-	log.Info().Str("domain", d).Str(zerowrap.FieldEntityID, c.ID).Msg("container stopped successfully")
1849		-	}
1850		-
1851		-	if errorCount > 0 {
1852		-	return fmt.Errorf("failed to stop %d containers", errorCount)
1853		-	}
1854		-
1855		-	log.Info().Msg("all managed containers stopped successfully")
1856		-	return nil
1857		-	}
1858		-
1859	1784		// Utility functions
1860	1785
1861	1786		func normalizeImageRef(image string) string {

@@ -2062,14 +1987,10 @@ func (s *Service) waitForHealthy(ctx context.Context, containerID string, timeou
2062	1987		if timeout == 0 {
2063	1988		timeout = 90 * time.Second
2064	1989		}
2065		-	// Apply multiplier from context (used for auto-start retries).
2066		-	if m := domain.HealthTimeoutMultiplier(ctx); m > 1.0 {
2067		-	timeout = time.Duration(float64(timeout) * m)
2068		-	}
2069	1990		waitCtx, cancel := context.WithTimeout(ctx, timeout)
2070	1991		defer cancel()
2071	1992
2072		-	lastStatus := "starting"
	1993	+	var lastStatus string
2073	1994		for {
2074	1995		status, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(waitCtx, containerID)
2075	1996		if err != nil {

@@ -2088,7 +2009,7 @@ func (s *Service) waitForHealthy(ctx context.Context, containerID string, timeou
2088	2009		select {
2089	2010		case <-time.After(time.Second):
2090	2011		case <-waitCtx.Done():
2091		-	return &healthTimeoutError{lastStatus: lastStatus, timeout: timeout}
	2012	+	return fmt.Errorf("container healthcheck timeout after %s (last status: %s)", timeout, lastStatus)
2092	2013		}
2093	2014		}
2094	2015		}


@@ -56,28 +56,6 @@ const (
56		internalPullMaxAttempts = 3	56		internalPullMaxAttempts = 3
57		)	57		)
58			58
59	-	// healthTimeoutError is returned when a container's health check times out.
60	-	// It carries the last observed health status to distinguish retryable (starting)
61	-	// from fatal (unhealthy) timeouts.
62	-	type healthTimeoutError struct {
63	-	lastStatus string
64	-	timeout time.Duration
65	-	}
66	-
67	-	func (e *healthTimeoutError) Error() string {
68	-	return fmt.Sprintf("container healthcheck timeout after %s (last status: %s)", e.timeout, e.lastStatus)
69	-	}
70	-
71	-	// isHealthTimeoutStarting returns true if err is a health timeout where the
72	-	// container was still in "starting" state (i.e., not unhealthy, just slow).
73	-	func isHealthTimeoutStarting(err error) bool {
74	-	var hte *healthTimeoutError
75	-	if errors.As(err, &hte) {
76	-	return hte.lastStatus == "starting"
77	-	}
78	-	return false
79	-	}
80	-
81		// Service implements the ContainerService interface.	59		// Service implements the ContainerService interface.
82		type Service struct {	60		type Service struct {
83		runtime out.ContainerRuntime	61		runtime out.ContainerRuntime

@@ -91,6 +69,7 @@ type Service struct {
91		attachments map[string][]string // ownerDomain → []containerIDs	69		attachments map[string][]string // ownerDomain → []containerIDs
92		mu sync.RWMutex	70		mu sync.RWMutex
93		deployMu sync.Map // per-domain deploy locks (domain → *domainDeployLock)	71		deployMu sync.Map // per-domain deploy locks (domain → *domainDeployLock)
			72	+	monitor *Monitor
94		}	73		}
95			74
96		// domainDeployLock is a context-aware mutex using a buffered channel.	75		// domainDeployLock is a context-aware mutex using a buffered channel.

@@ -369,13 +348,11 @@ func (s *Service) createStartedContainer(ctx context.Context, route domain.Route
369		s.runtime.RemoveContainer(ctx, newContainer.ID, true)	348		s.runtime.RemoveContainer(ctx, newContainer.ID, true)
370		return nil, log.WrapErr(err, "failed to start container")	349		return nil, log.WrapErr(err, "failed to start container")
371		}	350		}
372	-	if err := s.waitForReady(ctx, newContainer.ID); err != nil {	351	+	if !domain.IsSkipReadiness(ctx) {
373	-	// If health timed out but the container is still starting (not unhealthy),	352	+	if err := s.waitForReady(ctx, newContainer.ID); err != nil {
374	-	// keep it running for potential retry by the caller.
375	-	if !isHealthTimeoutStarting(err) {
376		s.cleanupFailedContainer(ctx, newContainer.ID)	353		s.cleanupFailedContainer(ctx, newContainer.ID)
			354	+	return nil, log.WrapErr(err, "container failed readiness check")
377		}	355		}
378	-	return nil, log.WrapErr(err, "container failed readiness check")
379		}	356		}
380			357
381		inspected, err := s.runtime.InspectContainer(ctx, newContainer.ID)	358		inspected, err := s.runtime.InspectContainer(ctx, newContainer.ID)

@@ -831,6 +808,8 @@ func (s *Service) SyncContainers(ctx context.Context) error {
831		}	808		}
832			809
833		// AutoStart starts containers for the provided routes that aren't running.	810		// AutoStart starts containers for the provided routes that aren't running.
			811	+	// It skips readiness checks to avoid blocking boot; the background monitor
			812	+	// handles crash recovery.
834		func (s *Service) AutoStart(ctx context.Context, routes []domain.Route) error {	813		func (s *Service) AutoStart(ctx context.Context, routes []domain.Route) error {
835		ctx = zerowrap.CtxWithFields(ctx, map[string]any{	814		ctx = zerowrap.CtxWithFields(ctx, map[string]any{
836		zerowrap.FieldLayer: "usecase",	815		zerowrap.FieldLayer: "usecase",

@@ -840,47 +819,55 @@ func (s *Service) AutoStart(ctx context.Context, routes []domain.Route) error {
840			819
841		log.Info().Int("route_count", len(routes)).Msg("auto-starting containers for configured routes")	820		log.Info().Int("route_count", len(routes)).Msg("auto-starting containers for configured routes")
842			821
843	-	var started, skipped, errCount int	822	+	// Filter out routes that already have a running container.
844	-	var retryable []domain.Route	823	+	var pending []domain.Route
845	-		824	+	skipped := 0
846		for _, route := range routes {	825		for _, route := range routes {
847		if _, exists := s.Get(ctx, route.Domain); exists {	826		if _, exists := s.Get(ctx, route.Domain); exists {
848		log.Debug().Str("domain", route.Domain).Msg("container already running, skipping")	827		log.Debug().Str("domain", route.Domain).Msg("container already running, skipping")
849		skipped++	828		skipped++
850	-	continue	829	+	} else {
			830	+	pending = append(pending, route)
851		}	831		}
			832	+	}
852			833
853	-	log.Info().	834	+	if len(pending) == 0 {
854	-	Str("domain", route.Domain).	835	+	log.Info().Int("skipped", skipped).Msg("auto-start completed, all containers already running")
855	-	Str("image", route.Image).	836	+	return nil
856	-	Msg("auto-starting container for route")	837	+	}
857	-
858	-	if _, err := s.Deploy(ctx, route); err != nil {
859	-	if isHealthTimeoutStarting(err) {
860	-	log.Warn().Str("domain", route.Domain).Msg("health check still starting, will retry")
861	-	retryable = append(retryable, route)
862	-	} else {
863	-	log.Warn().Err(err).Str("domain", route.Domain).Msg("failed to auto-start container")
864	-	errCount++
865	-	}
866	-	continue
867	-	}
868			838
869	-	started++	839	+	// Deploy all pending routes concurrently with readiness checks skipped.
			840	+	deployCtx := domain.WithSkipReadiness(ctx)
			841	+
			842	+	type result struct {
			843	+	route domain.Route
			844	+	err error
870		}	845		}
			846	+	results := make(chan result, len(pending))
			847	+	sem := make(chan struct{}, 4) // limit concurrency
871			848
872	-	// Retry routes whose containers are still starting with extended timeout.	849	+	for _, route := range pending {
873	-	if len(retryable) > 0 {	850	+	sem <- struct{}{}
874	-	log.Info().Int("retry_count", len(retryable)).Msg("retrying slow-starting containers with extended timeout")	851	+	go func(r domain.Route) {
875	-	retryCtx := domain.WithHealthTimeoutMultiplier(ctx, 2.0)	852	+	defer func() { <-sem }()
876			853
877	-	for _, route := range retryable {	854	+	log.Info().Str("domain", r.Domain).Str("image", r.Image).Msg("auto-starting container for route")
878	-	// Container is still running from first attempt, try readiness again.	855	+
879	-	if _, err := s.retryReadiness(retryCtx, route); err != nil {	856	+	if _, err := s.Deploy(deployCtx, r); err != nil {
880	-	log.Warn().Err(err).Str("domain", route.Domain).Msg("failed to auto-start container after retry")	857	+	results <- result{route: r, err: err}
881	-	errCount++	858	+	return
882	-	continue
883		}	859		}
			860	+	results <- result{route: r}
			861	+	}(route)
			862	+	}
			863	+
			864	+	var started, errCount int
			865	+	for range len(pending) {
			866	+	res := <-results
			867	+	if res.err != nil {
			868	+	log.Warn().Err(res.err).Str("domain", res.route.Domain).Msg("failed to auto-start container")
			869	+	errCount++
			870	+	} else {
884		started++	871		started++
885		}	872		}
886		}	873		}

@@ -897,54 +884,6 @@ func (s *Service) AutoStart(ctx context.Context, routes []domain.Route) error {
897		return nil	884		return nil
898		}	885		}
899			886
900	-	// retryReadiness re-checks readiness for a container that was left running
901	-	// after a health timeout during auto-start. If readiness succeeds, the
902	-	// container is activated. If it fails, the container is cleaned up.
903	-	func (s Service) retryReadiness(ctx context.Context, route domain.Route) (domain.Container, error) {
904	-	log := zerowrap.FromCtx(ctx)
905	-
906	-	unlock, err := s.acquireDomainDeployLock(ctx, route.Domain)
907	-	if err != nil {
908	-	return nil, err
909	-	}
910	-	defer unlock()
911	-
912	-	// Find the container that was left running from the first attempt.
913	-	containers, err := s.runtime.ListContainers(ctx, false)
914	-	if err != nil {
915	-	return nil, err
916	-	}
917	-
918	-	var containerID string
919	-	for _, c := range containers {
920	-	if c.Labels != nil && c.Labels[domain.LabelRoute] == route.Domain && c.Status == string(domain.ContainerStatusRunning) {
921	-	containerID = c.ID
922	-	break
923	-	}
924	-	}
925	-	if containerID == "" {
926	-	// Container is no longer running, do a full deploy.
927	-	return s.Deploy(ctx, route)
928	-	}
929	-
930	-	log.Info().Str("domain", route.Domain).Str("container_id", containerID).Msg("retrying readiness check")
931	-
932	-	if err := s.waitForReady(ctx, containerID); err != nil {
933	-	s.cleanupFailedContainer(ctx, containerID)
934	-	return nil, log.WrapErr(err, "container failed readiness check on retry")
935	-	}
936	-
937	-	inspected, err := s.runtime.InspectContainer(ctx, containerID)
938	-	if err != nil {
939	-	s.cleanupFailedContainer(ctx, containerID)
940	-	return nil, log.WrapErr(err, "failed to inspect container after retry")
941	-	}
942	-
943	-	s.activateDeployedContainer(ctx, route.Domain, inspected)
944	-
945	-	return inspected, nil
946	-	}
947	-
948		// Shutdown gracefully shuts down all managed containers.	887		// Shutdown gracefully shuts down all managed containers.
949		func (s *Service) Shutdown(ctx context.Context) error {	888		func (s *Service) Shutdown(ctx context.Context) error {
950		ctx = zerowrap.CtxWithFields(ctx, map[string]any{	889		ctx = zerowrap.CtxWithFields(ctx, map[string]any{

@@ -954,9 +893,8 @@ func (s *Service) Shutdown(ctx context.Context) error {
954		log := zerowrap.FromCtx(ctx)	893		log := zerowrap.FromCtx(ctx)
955		log.Info().Msg("shutting down container manager...")	894		log.Info().Msg("shutting down container manager...")
956			895
957	-	if err := s.stopAllManagedContainers(ctx); err != nil {	896	+	// Containers are left running across Gordon restarts.
958	-	log.WrapErr(err, "failed to stop all containers during shutdown")	897	+	// SyncContainers + AutoStart will pick them back up on next boot.
959	-	}
960			898
961		// Close log writer to stop all log collection	899		// Close log writer to stop all log collection
962		if s.logWriter != nil {	900		if s.logWriter != nil {

@@ -969,6 +907,33 @@ func (s *Service) Shutdown(ctx context.Context) error {
969		return nil	907		return nil
970		}	908		}
971			909
			910	+	// StartMonitor begins background monitoring of tracked containers.
			911	+	// The monitor restarts crashed containers and detects crash loops.
			912	+	// Safe to call multiple times; subsequent calls are no-ops.
			913	+	func (s *Service) StartMonitor(ctx context.Context) {
			914	+	s.mu.Lock()
			915	+	if s.monitor != nil {
			916	+	s.mu.Unlock()
			917	+	return
			918	+	}
			919	+	s.monitor = newMonitor(s)
			920	+	m := s.monitor
			921	+	s.mu.Unlock()
			922	+	m.Start(ctx)
			923	+	}
			924	+
			925	+	// StopMonitor stops the background container monitor.
			926	+	// Safe to call multiple times; subsequent calls are no-ops.
			927	+	func (s *Service) StopMonitor() {
			928	+	s.mu.Lock()
			929	+	m := s.monitor
			930	+	s.monitor = nil
			931	+	s.mu.Unlock()
			932	+	if m != nil {
			933	+	m.Stop()
			934	+	}
			935	+	}
			936	+
972		// UpdateConfig updates the service configuration.	937		// UpdateConfig updates the service configuration.
973		func (s *Service) UpdateConfig(config Config) {	938		func (s *Service) UpdateConfig(config Config) {
974		s.mu.Lock()	939		s.mu.Lock()

@@ -1816,46 +1781,6 @@ func (s *Service) deployAttachedService(ctx context.Context, ownerDomain, servic
1816		return nil	1781		return nil
1817		}	1782		}
1818			1783
1819	-	func (s *Service) stopAllManagedContainers(ctx context.Context) error {
1820	-	log := zerowrap.FromCtx(ctx)
1821	-
1822	-	s.mu.RLock()
1823	-	containers := make(map[string]*domain.Container, len(s.containers))
1824	-	maps.Copy(containers, s.containers)
1825	-	s.mu.RUnlock()
1826	-
1827	-	if len(containers) == 0 {
1828	-	log.Info().Msg("no managed containers to stop")
1829	-	return nil
1830	-	}
1831	-
1832	-	log.Info().Int(zerowrap.FieldCount, len(containers)).Msg("stopping all managed containers...")
1833	-
1834	-	errorCount := 0
1835	-	for d, c := range containers {
1836	-	log.Info().Str("domain", d).Str(zerowrap.FieldEntityID, c.ID).Msg("stopping managed container")
1837	-
1838	-	if err := s.runtime.StopContainer(ctx, c.ID); err != nil {
1839	-	log.WrapErrWithFields(err, "failed to stop container", map[string]any{"domain": d, zerowrap.FieldEntityID: c.ID})
1840	-	errorCount++
1841	-	continue
1842	-	}
1843	-
1844	-	s.mu.Lock()
1845	-	delete(s.containers, d)
1846	-	s.mu.Unlock()
1847	-
1848	-	log.Info().Str("domain", d).Str(zerowrap.FieldEntityID, c.ID).Msg("container stopped successfully")
1849	-	}
1850	-
1851	-	if errorCount > 0 {
1852	-	return fmt.Errorf("failed to stop %d containers", errorCount)
1853	-	}
1854	-
1855	-	log.Info().Msg("all managed containers stopped successfully")
1856	-	return nil
1857	-	}
1858	-
1859		// Utility functions	1784		// Utility functions
1860			1785
1861		func normalizeImageRef(image string) string {	1786		func normalizeImageRef(image string) string {

@@ -2062,14 +1987,10 @@ func (s *Service) waitForHealthy(ctx context.Context, containerID string, timeou
2062		if timeout == 0 {	1987		if timeout == 0 {
2063		timeout = 90 * time.Second	1988		timeout = 90 * time.Second
2064		}	1989		}
2065	-	// Apply multiplier from context (used for auto-start retries).
2066	-	if m := domain.HealthTimeoutMultiplier(ctx); m > 1.0 {
2067	-	timeout = time.Duration(float64(timeout) * m)
2068	-	}
2069		waitCtx, cancel := context.WithTimeout(ctx, timeout)	1990		waitCtx, cancel := context.WithTimeout(ctx, timeout)
2070		defer cancel()	1991		defer cancel()
2071			1992
2072	-	lastStatus := "starting"	1993	+	var lastStatus string
2073		for {	1994		for {
2074		status, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(waitCtx, containerID)	1995		status, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(waitCtx, containerID)
2075		if err != nil {	1996		if err != nil {

@@ -2088,7 +2009,7 @@ func (s *Service) waitForHealthy(ctx context.Context, containerID string, timeou
2088		select {	2009		select {
2089		case <-time.After(time.Second):	2010		case <-time.After(time.Second):
2090		case <-waitCtx.Done():	2011		case <-waitCtx.Done():
2091	-	return &healthTimeoutError{lastStatus: lastStatus, timeout: timeout}	2012	+	return fmt.Errorf("container healthcheck timeout after %s (last status: %s)", timeout, lastStatus)
2092		}	2013		}
2093		}	2014		}
2094		}	2015		}

M internal/usecase/container/service_test.go +9 -33


@@ -630,35 +630,13 @@ func TestService_Shutdown(t *testing.T) {
630	630		svc.containers["app1.example.com"] = &domain.Container{ID: "container-1"}
631	631		svc.containers["app2.example.com"] = &domain.Container{ID: "container-2"}
632	632
633		-	runtime.EXPECT().StopContainer(mock.Anything, "container-1").Return(nil)
634		-	runtime.EXPECT().StopContainer(mock.Anything, "container-2").Return(nil)
635		-
	633	+	// Shutdown no longer stops containers — they are left running for
	634	+	// SyncContainers + AutoStart to pick back up on next boot.
636	635		err := svc.Shutdown(ctx)
637	636
638	637		assert.NoError(t, err)
639		-	assert.Empty(t, svc.containers)
640		-	}
641		-
642		-	func TestService_Shutdown_PartialFailure(t *testing.T) {
643		-	runtime := mocks.NewMockContainerRuntime(t)
644		-	envLoader := mocks.NewMockEnvLoader(t)
645		-	eventBus := mocks.NewMockEventPublisher(t)
646		-
647		-	svc := NewService(runtime, envLoader, eventBus, nil, Config{})
648		-	ctx := testContext()
649		-
650		-	svc.containers["app1.example.com"] = &domain.Container{ID: "container-1"}
651		-	svc.containers["app2.example.com"] = &domain.Container{ID: "container-2"}
652		-
653		-	runtime.EXPECT().StopContainer(mock.Anything, "container-1").Return(nil)
654		-	runtime.EXPECT().StopContainer(mock.Anything, "container-2").Return(errors.New("stop failed"))
655		-
656		-	err := svc.Shutdown(ctx)
657		-
658		-	// Shutdown logs errors but always returns nil for graceful degradation
659		-	assert.NoError(t, err)
660		-	// One container should still be tracked (the one that failed to stop)
661		-	assert.Len(t, svc.containers, 1)
	638	+	// Containers remain tracked (not stopped).
	639	+	assert.Len(t, svc.containers, 2)
662	640		}
663	641
664	642		func TestService_Restart_NotFound(t *testing.T) {

@@ -1196,14 +1174,14 @@ func TestService_AutoStart_StartsNewContainers(t *testing.T) {
1196	1174		{Domain: "app2.example.com", Image: "myapp2:latest"},
1197	1175		}
1198	1176
1199		-	// Setup mocks for first route deployment
	1177	+	// Setup mocks for route deployments
1200	1178		runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil).Times(2)
1201	1179		runtime.EXPECT().ListImages(mock.Anything).Return([]string{"myapp1:latest", "myapp2:latest"}, nil).Times(2)
1202	1180		runtime.EXPECT().GetImageExposedPorts(mock.Anything, mock.AnythingOfType("string")).Return([]int{8080}, nil).Times(2)
1203	1181		envLoader.EXPECT().LoadEnv(mock.Anything, mock.AnythingOfType("string")).Return([]string{}, nil).Times(2)
1204	1182		runtime.EXPECT().InspectImageEnv(mock.Anything, mock.AnythingOfType("string")).Return([]string{}, nil).Times(2)
1205	1183
1206		-	// Create and start containers
	1184	+	// Create and start containers — readiness is skipped so no IsContainerRunning calls.
1207	1185		runtime.EXPECT().CreateContainer(mock.Anything, mock.AnythingOfType("*domain.ContainerConfig")).Return(&domain.Container{
1208	1186		ID: "container-1", Name: "gordon-app1.example.com", Status: "created",
1209	1187		}, nil).Once()

@@ -1211,7 +1189,6 @@ func TestService_AutoStart_StartsNewContainers(t *testing.T) {
1211	1189		ID: "container-2", Name: "gordon-app2.example.com", Status: "created",
1212	1190		}, nil).Once()
1213	1191		runtime.EXPECT().StartContainer(mock.Anything, mock.AnythingOfType("string")).Return(nil).Times(2)
1214		-	runtime.EXPECT().IsContainerRunning(mock.Anything, mock.AnythingOfType("string")).Return(true, nil).Times(4)
1215	1192		runtime.EXPECT().InspectContainer(mock.Anything, mock.AnythingOfType("string")).Return(&domain.Container{
1216	1193		Status: "running",
1217	1194		}, nil).Times(2)

@@ -1245,7 +1222,7 @@ func TestService_AutoStart_SkipsExistingContainers(t *testing.T) {
1245	1222		{Domain: "app2.example.com", Image: "myapp2:latest"}, // New route
1246	1223		}
1247	1224
1248		-	// Only deploy for app2 (app1 is skipped)
	1225	+	// Only deploy for app2 (app1 is skipped). Readiness is skipped — no IsContainerRunning.
1249	1226		runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil).Once()
1250	1227		runtime.EXPECT().ListImages(mock.Anything).Return([]string{"myapp2:latest"}, nil).Once()
1251	1228		runtime.EXPECT().GetImageExposedPorts(mock.Anything, "myapp2:latest").Return([]int{8080}, nil).Once()

@@ -1255,7 +1232,6 @@ func TestService_AutoStart_SkipsExistingContainers(t *testing.T) {
1255	1232		ID: "container-2", Status: "created",
1256	1233		}, nil).Once()
1257	1234		runtime.EXPECT().StartContainer(mock.Anything, "container-2").Return(nil).Once()
1258		-	runtime.EXPECT().IsContainerRunning(mock.Anything, "container-2").Return(true, nil).Times(2)
1259	1235		runtime.EXPECT().InspectContainer(mock.Anything, "container-2").Return(&domain.Container{
1260	1236		Status: "running",
1261	1237		}, nil).Once()

@@ -1313,7 +1289,8 @@ func TestService_AutoStart_UsesInternalDeployContext(t *testing.T) {
1313	1289		}
1314	1290
1315	1291		// Key assertion: PullImage should be called with localhost:5000 rewrite,
1316		-	// NOT the original reg.example.com/myapp:latest
	1292	+	// NOT the original reg.example.com/myapp:latest.
	1293	+	// Readiness is skipped — no IsContainerRunning calls.
1317	1294		runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil)
1318	1295		runtime.EXPECT().PullImage(mock.Anything, "localhost:5000/myapp:latest").Return(nil)
1319	1296		runtime.EXPECT().TagImage(mock.Anything, "localhost:5000/myapp:latest", "reg.example.com/myapp:latest").Return(nil)

@@ -1325,7 +1302,6 @@ func TestService_AutoStart_UsesInternalDeployContext(t *testing.T) {
1325	1302		ID: "container-1", Name: "gordon-app1.example.com", Status: "created",
1326	1303		}, nil)
1327	1304		runtime.EXPECT().StartContainer(mock.Anything, "container-1").Return(nil)
1328		-	runtime.EXPECT().IsContainerRunning(mock.Anything, "container-1").Return(true, nil).Times(2)
1329	1305		runtime.EXPECT().InspectContainer(mock.Anything, "container-1").Return(&domain.Container{
1330	1306		Status: "running",
1331	1307		}, nil)

 	svc.containers["app1.example.com"] = &domain.Container{ID: "container-1"}
 	svc.containers["app2.example.com"] = &domain.Container{ID: "container-2"}
-	runtime.EXPECT().StopContainer(mock.Anything, "container-1").Return(nil)
+	// Shutdown no longer stops containers — they are left running for
-	runtime.EXPECT().StopContainer(mock.Anything, "container-2").Return(nil)
+	// SyncContainers + AutoStart to pick back up on next boot.
 	err := svc.Shutdown(ctx)
 	assert.NoError(t, err)
-	assert.Empty(t, svc.containers)
+	// Containers remain tracked (not stopped).
-}
+	assert.Len(t, svc.containers, 2)
-func TestService_Shutdown_PartialFailure(t *testing.T) {
-	runtime := mocks.NewMockContainerRuntime(t)
-	envLoader := mocks.NewMockEnvLoader(t)
-	eventBus := mocks.NewMockEventPublisher(t)
-	svc := NewService(runtime, envLoader, eventBus, nil, Config{})
-	ctx := testContext()
-	svc.containers["app1.example.com"] = &domain.Container{ID: "container-1"}
-	svc.containers["app2.example.com"] = &domain.Container{ID: "container-2"}
-	runtime.EXPECT().StopContainer(mock.Anything, "container-1").Return(nil)
-	runtime.EXPECT().StopContainer(mock.Anything, "container-2").Return(errors.New("stop failed"))
-	err := svc.Shutdown(ctx)
-	// Shutdown logs errors but always returns nil for graceful degradation
-	assert.NoError(t, err)
-	// One container should still be tracked (the one that failed to stop)
-	assert.Len(t, svc.containers, 1)
 }
 func TestService_Restart_NotFound(t *testing.T) {
 		{Domain: "app2.example.com", Image: "myapp2:latest"},
 	}
-	// Setup mocks for first route deployment
+	// Setup mocks for route deployments
 	runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil).Times(2)
 	runtime.EXPECT().ListImages(mock.Anything).Return([]string{"myapp1:latest", "myapp2:latest"}, nil).Times(2)
 	runtime.EXPECT().GetImageExposedPorts(mock.Anything, mock.AnythingOfType("string")).Return([]int{8080}, nil).Times(2)
 	envLoader.EXPECT().LoadEnv(mock.Anything, mock.AnythingOfType("string")).Return([]string{}, nil).Times(2)
 	runtime.EXPECT().InspectImageEnv(mock.Anything, mock.AnythingOfType("string")).Return([]string{}, nil).Times(2)
-	// Create and start containers
+	// Create and start containers — readiness is skipped so no IsContainerRunning calls.
 	runtime.EXPECT().CreateContainer(mock.Anything, mock.AnythingOfType("*domain.ContainerConfig")).Return(&domain.Container{
 		ID: "container-1", Name: "gordon-app1.example.com", Status: "created",
 	}, nil).Once()
 		ID: "container-2", Name: "gordon-app2.example.com", Status: "created",
 	}, nil).Once()
 	runtime.EXPECT().StartContainer(mock.Anything, mock.AnythingOfType("string")).Return(nil).Times(2)
-	runtime.EXPECT().IsContainerRunning(mock.Anything, mock.AnythingOfType("string")).Return(true, nil).Times(4)
 	runtime.EXPECT().InspectContainer(mock.Anything, mock.AnythingOfType("string")).Return(&domain.Container{
 		Status: "running",
 	}, nil).Times(2)
 		{Domain: "app2.example.com", Image: "myapp2:latest"}, // New route
 	}
-	// Only deploy for app2 (app1 is skipped)
+	// Only deploy for app2 (app1 is skipped). Readiness is skipped — no IsContainerRunning.
 	runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil).Once()
 	runtime.EXPECT().ListImages(mock.Anything).Return([]string{"myapp2:latest"}, nil).Once()
 	runtime.EXPECT().GetImageExposedPorts(mock.Anything, "myapp2:latest").Return([]int{8080}, nil).Once()
 		ID: "container-2", Status: "created",
 	}, nil).Once()
 	runtime.EXPECT().StartContainer(mock.Anything, "container-2").Return(nil).Once()
-	runtime.EXPECT().IsContainerRunning(mock.Anything, "container-2").Return(true, nil).Times(2)
 	runtime.EXPECT().InspectContainer(mock.Anything, "container-2").Return(&domain.Container{
 		Status: "running",
 	}, nil).Once()
 	}
 	// Key assertion: PullImage should be called with localhost:5000 rewrite,
-	// NOT the original reg.example.com/myapp:latest
+	// NOT the original reg.example.com/myapp:latest.
+	// Readiness is skipped — no IsContainerRunning calls.
 	runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil)
 	runtime.EXPECT().PullImage(mock.Anything, "localhost:5000/myapp:latest").Return(nil)
 	runtime.EXPECT().TagImage(mock.Anything, "localhost:5000/myapp:latest", "reg.example.com/myapp:latest").Return(nil)
 		ID: "container-1", Name: "gordon-app1.example.com", Status: "created",
 	}, nil)
 	runtime.EXPECT().StartContainer(mock.Anything, "container-1").Return(nil)
-	runtime.EXPECT().IsContainerRunning(mock.Anything, "container-1").Return(true, nil).Times(2)
 	runtime.EXPECT().InspectContainer(mock.Anything, "container-1").Return(&domain.Container{
 		Status: "running",
 	}, nil)