service.go RawHistoryFolder

1// Package container implements the container management use case.
2package container
3
4import (
5	"context"
6	"errors"
7	"fmt"
8	"maps"
9	"slices"
10	"strconv"
11	"strings"
12	"sync"
13	"time"
14
15	"github.com/bnema/zerowrap"
16	"go.opentelemetry.io/otel"
17	"go.opentelemetry.io/otel/attribute"
18	"go.opentelemetry.io/otel/codes"
19	"go.opentelemetry.io/otel/metric"
20	"go.opentelemetry.io/otel/trace"
21
22	"github.com/bnema/gordon/internal/adapters/out/telemetry"
23	"github.com/bnema/gordon/internal/boundaries/out"
24	"github.com/bnema/gordon/internal/domain"
25)
26
27// Config holds configuration needed by the container service.
28type Config struct {
29	RegistryAuthEnabled        bool
30	RegistryDomain             string
31	RegistryPort               int
32	ServiceTokenUsername       string
33	ServiceToken               string
34	InternalRegistryUsername   string
35	InternalRegistryPassword   string
36	PullPolicy                 string
37	VolumeAutoCreate           bool
38	VolumePrefix               string
39	VolumePreserve             bool
40	NetworkIsolation           bool
41	NetworkPrefix              string
42	NetworkGroups              map[string][]string
43	Attachments                map[string][]string
44	ReadinessDelay             time.Duration // Delay after container starts before considering it ready
45	ReadinessMode              string        // Readiness strategy: auto, docker-health, delay
46	HealthTimeout              time.Duration // Max wait for health-based readiness
47	DrainDelay                 time.Duration // Grace period after cache invalidation before stopping old container
48	DrainDelayConfigured       bool          // True when deploy.drain_delay was explicitly configured
49	DrainMode                  string        // Drain strategy: auto, inflight, delay
50	DrainTimeout               time.Duration // Max wait for in-flight requests to drain
51	StabilizationDelay         time.Duration // Post-switch monitoring window (default 2s)
52	TCPProbeTimeout            time.Duration // TCP probe timeout (default 30s)
53	HTTPProbeTimeout           time.Duration // HTTP probe timeout (default 60s)
54	AttachmentReadinessTimeout time.Duration // Max wait for attachment readiness (default 30s)
55	DefaultMemoryLimit         int64         // Default memory limit in bytes for containers (0 = no limit)
56	DefaultNanoCPUs            int64         // Default CPU quota in nanoseconds for containers (0 = no limit)
57	DefaultPidsLimit           int64         // Default max PIDs for containers (0 = no limit)
58}
59
60var tracer = otel.Tracer("gordon.container")
61
62const (
63	PullPolicyAlways       = "always"
64	PullPolicyIfNotPresent = "if-not-present"
65	PullPolicyIfTagChanged = "if-tag-changed"
66
67	// readinessRecoveryWindow is an additional grace window used when a container
68	// is briefly not running at the end of readiness delay. This avoids false
69	// negatives during short startup flaps.
70	readinessRecoveryWindow = 30 * time.Second
71	internalPullMaxAttempts = 3
72)
73
74// Service implements the ContainerService interface.
75type Service struct {
76	runtime          out.ContainerRuntime
77	envLoader        out.EnvLoader
78	eventBus         out.EventPublisher
79	logWriter        out.ContainerLogWriter
80	cacheInvalidator out.ProxyCacheInvalidator
81	drainWaiter      out.ProxyDrainWaiter
82	config           Config
83	configProvider   AttachmentConfigProvider // live config reads for attachments/networks (may be nil)
84	metrics          *telemetry.Metrics
85	containers       map[string]*domain.Container
86	attachments      map[string][]string // ownerDomain → []containerIDs
87	managedCount     int64               // tracks UpDownCounter value for delta computation
88	mu               sync.RWMutex
89	deployMu         sync.Map       // per-domain deploy locks (domain → *domainDeployLock)
90	cleanupWg        sync.WaitGroup // tracks background old-container cleanup goroutines
91	monitor          *Monitor
92}
93
94// domainDeployLock is a context-aware mutex using a buffered channel.
95// It allows Lock() to be interrupted by context cancellation.
96type domainDeployLock struct {
97	ch chan struct{}
98}
99
100// newDomainDeployLock creates a new lock that is initially unlocked.
101func newDomainDeployLock() *domainDeployLock {
102	l := &domainDeployLock{ch: make(chan struct{}, 1)}
103	l.ch <- struct{}{}
104	return l
105}
106
107// Lock acquires the lock or returns an error if the context is cancelled.
108func (l *domainDeployLock) Lock(ctx context.Context) error {
109	// Fast path: honor already-canceled contexts before blocking.
110	if err := ctx.Err(); err != nil {
111		return err
112	}
113	select {
114	case <-ctx.Done():
115		return ctx.Err()
116	case <-l.ch:
117		// Re-check context after acquisition: if canceled while we were
118		// in the select, release the token so the lock isn't consumed.
119		if err := ctx.Err(); err != nil {
120			l.ch <- struct{}{}
121			return err
122		}
123		return nil
124	}
125}
126
127// Unlock releases the lock.
128func (l *domainDeployLock) Unlock() {
129	select {
130	case l.ch <- struct{}{}:
131	default:
132		// This should never happen if Lock/Unlock are used correctly.
133		// Panic to detect double-unlocks or other misuse early.
134		panic("domainDeployLock: double unlock detected")
135	}
136}
137
138// NewService creates a new container service.
139func NewService(
140	runtime out.ContainerRuntime,
141	envLoader out.EnvLoader,
142	eventBus out.EventPublisher,
143	logWriter out.ContainerLogWriter,
144	config Config,
145	configProvider AttachmentConfigProvider,
146) *Service {
147	return &Service{
148		runtime:        runtime,
149		envLoader:      envLoader,
150		eventBus:       eventBus,
151		logWriter:      logWriter,
152		config:         config,
153		configProvider: configProvider,
154		containers:     make(map[string]*domain.Container),
155		attachments:    make(map[string][]string),
156	}
157}
158
159// SetMetrics sets the telemetry metrics for the container service.
160func (s *Service) SetMetrics(m *telemetry.Metrics) {
161	s.metrics = m
162}
163
164// SetProxyCacheInvalidator sets the proxy cache invalidator for synchronous
165// cache invalidation during zero-downtime deployments. This must be called
166// after construction because the proxy service is created after the container
167// service during application initialization.
168func (s *Service) SetProxyCacheInvalidator(inv out.ProxyCacheInvalidator) {
169	s.mu.Lock()
170	s.cacheInvalidator = inv
171	s.mu.Unlock()
172}
173
174// SetProxyDrainWaiter sets the proxy in-flight drain waiter used during
175// zero-downtime replacement before stopping the old container.
176func (s *Service) SetProxyDrainWaiter(waiter out.ProxyDrainWaiter) {
177	s.mu.Lock()
178	s.drainWaiter = waiter
179	s.mu.Unlock()
180}
181
182// domainDeployMu returns a per-domain lock for serializing deploy operations.
183func (s *Service) domainDeployMu(domain string) *domainDeployLock {
184	v, _ := s.deployMu.LoadOrStore(domain, newDomainDeployLock())
185	// Type assertion is safe: LoadOrStore always stores *domainDeployLock values,
186	// and any concurrent LoadOrStore calls will also store *domainDeployLock.
187	return v.(*domainDeployLock)
188}
189
190// acquireDomainDeployLock acquires the per-domain deploy lock with context cancellation support.
191// Returns an unlock function that must be called to release the lock.
192func (s *Service) acquireDomainDeployLock(ctx context.Context, domain string) (func(), error) {
193	lock := s.domainDeployMu(domain)
194	if err := lock.Lock(ctx); err != nil {
195		return nil, err
196	}
197	return func() { lock.Unlock() }, nil
198}
199
200// buildContainerConfig constructs the container configuration for deployment.
201func (s *Service) deploymentContainerName(containerDomain string, existing *domain.Container) string {
202	canonicalName := fmt.Sprintf("gordon-%s", containerDomain)
203	if existing == nil {
204		return canonicalName
205	}
206
207	// Alternate temporary names if the tracked container was left with a temp suffix.
208	// This prevents name collisions after interrupted zero-downtime deploys.
209	newName := canonicalName + "-new"
210	nextName := canonicalName + "-next"
211	switch existing.Name {
212	case newName:
213		return nextName
214	case nextName:
215		return newName
216	default:
217		return newName
218	}
219}
220
221// containerConfigInput groups the parameters for building a container configuration.
222type containerConfigInput struct {
223	Domain       string
224	Image        string
225	ImageRef     string
226	ExposedPorts []int
227	EnvVars      []string
228	EnvHash      string
229	Volumes      map[string]string
230	NetworkName  string
231	ImageLabels  map[string]string
232	Existing     *domain.Container
233}
234
235func (s *Service) buildContainerConfig(in containerConfigInput) *domain.ContainerConfig {
236	s.mu.RLock()
237	cfg := s.config
238	s.mu.RUnlock()
239
240	containerName := s.deploymentContainerName(in.Domain, in.Existing)
241	labels := map[string]string{
242		domain.LabelDomain:  in.Domain,
243		domain.LabelEnvHash: in.EnvHash,
244		domain.LabelImage:   in.Image,
245		domain.LabelManaged: "true",
246		domain.LabelRoute:   in.Domain,
247	}
248
249	// Propagate proxy/port labels from image so readiness probes can find them
250	for _, key := range []string{domain.LabelProxyPort, domain.LabelPort, domain.LabelHealth} {
251		if v, ok := in.ImageLabels[key]; ok && v != "" {
252			labels[key] = v
253		}
254	}
255
256	// Ensure the label-specified proxy port is included in exposed ports
257	// so Docker creates a host port binding for it.
258	ports := in.ExposedPorts
259	if portStr, ok := labels[domain.LabelProxyPort]; ok {
260		if port, err := strconv.Atoi(portStr); err == nil && port > 0 && port <= 65535 {
261			if !slices.Contains(ports, port) {
262				ports = append(ports, port)
263			}
264		}
265	}
266
267	return &domain.ContainerConfig{
268		Image:       in.ImageRef,
269		Name:        containerName,
270		Ports:       ports,
271		Env:         in.EnvVars,
272		Volumes:     in.Volumes,
273		NetworkMode: in.NetworkName,
274		Hostname:    in.Domain,
275		Labels:      labels,
276		AutoRemove:  false,
277		MemoryLimit: cfg.DefaultMemoryLimit,
278		NanoCPUs:    cfg.DefaultNanoCPUs,
279		PidsLimit:   cfg.DefaultPidsLimit,
280	}
281}
282
283// Deploy creates and starts a container for the given route.
284// Implements zero-downtime deployment: new container starts before old one stops.
285func (s *Service) Deploy(ctx context.Context, route domain.Route) (*domain.Container, error) {
286	ctx, span := tracer.Start(ctx, "container.deploy",
287		trace.WithAttributes(
288			attribute.String("domain", route.Domain),
289			attribute.String("image", route.Image),
290		))
291	defer span.End()
292
293	deployStart := time.Now()
294
295	// Serialize deploys for the same domain to prevent race conditions
296	// (e.g. multiple image.pushed events + explicit deploy call from CLI).
297	unlock, err := s.acquireDomainDeployLock(ctx, route.Domain)
298	if err != nil {
299		return nil, err
300	}
301	defer unlock()
302
303	// Enrich context with use case fields for all downstream logs
304	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
305		zerowrap.FieldLayer:   "usecase",
306		zerowrap.FieldUseCase: "Deploy",
307		"domain":              route.Domain,
308	})
309	log := zerowrap.FromCtx(ctx)
310
311	// Record deploy metrics and trace status
312	defer s.recordDeployMetrics(ctx, span, route, deployStart, &err)
313
314	existing, hasExisting := s.resolveExistingContainer(ctx, route.Domain)
315
316	resources, err := s.prepareDeployResources(ctx, route, existing)
317	if err != nil {
318		return nil, err
319	}
320
321	// Skip redundant deploy: if the existing container is already running
322	// the exact same image (by Docker image ID), return it immediately.
323	// This prevents the double-deploy caused by the event-based deploy
324	// (triggered by image.pushed) racing with the explicit CLI deploy call.
325	if hasExisting {
326		existingForSkip := existing
327		if existingForSkip.ImageID == "" && existingForSkip.Image != "" && normalizeImageRef(existingForSkip.Image) == normalizeImageRef(route.Image) {
328			existingForSkip = s.containerForRedundantCheck(ctx, existingForSkip)
329		}
330		if existingForSkip.ImageID != "" {
331			if skip, container := s.skipRedundantDeploy(ctx, existingForSkip, resources.actualImageRef, resources.envHash); skip {
332				return container, nil
333			}
334		}
335	}
336
337	newContainer, err := s.createStartedContainer(ctx, route, existing, resources)
338	if err != nil {
339		return nil, err
340	}
341
342	invalidated := s.activateDeployedContainer(ctx, route.Domain, newContainer)
343
344	// Post-switch stabilization: verify new container stays running
345	if hasExisting {
346		stable, stabilizeErr := s.stabilizeNewContainer(ctx, route.Domain, newContainer, existing)
347		if stabilizeErr != nil {
348			// Both old and new containers are dead; assign to named return so
349			// the deferred recordDeployMetrics and span see the failure.
350			err = stabilizeErr
351			return nil, err
352		}
353		if !stable {
354			// Rollback performed — old container is restored
355			return existing, nil
356		}
357	}
358
359	// Finalize old container in the background — the new container is already
360	// serving traffic, so there's no reason to block the deploy response while
361	// waiting for the old container to stop (which can take 20s if the app
362	// doesn't handle SIGTERM).
363	s.cleanupWg.Add(1)
364	go func() {
365		defer s.cleanupWg.Done()
366		s.finalizePreviousContainer(context.WithoutCancel(ctx), route.Domain, existing, hasExisting, invalidated, newContainer.ID)
367	}()
368
369	// Start container log collection (non-blocking, errors don't fail deployment)
370	s.startLogCollection(ctx, newContainer.ID, route.Domain)
371
372	log.Info().
373		Str("image", route.Image).
374		Str(zerowrap.FieldEntityID, newContainer.ID).
375		Ints("ports", newContainer.Ports).
376		Str("network", resources.networkName).
377		Bool("zero_downtime", hasExisting).
378		Msg("container deployed successfully")
379
380	return newContainer, nil
381}
382
383// recordDeployMetrics records span error status and deploy metrics at the end of a Deploy call.
384// It is called via defer and receives a pointer to the named return error so it can observe
385// the final error value after all deferred functions have run.
386func (s *Service) recordDeployMetrics(ctx context.Context, span trace.Span, route domain.Route, start time.Time, errPtr *error) {
387	err := *errPtr
388	if err != nil {
389		span.RecordError(err)
390		span.SetStatus(codes.Error, err.Error())
391	}
392	if s.metrics == nil {
393		return
394	}
395	attrs := metric.WithAttributes(
396		attribute.String("domain", route.Domain),
397		attribute.String("image", route.Image),
398	)
399	s.metrics.DeployTotal.Add(ctx, 1, attrs)
400	s.metrics.DeployDuration.Record(ctx, time.Since(start).Seconds(), attrs)
401	if err != nil {
402		s.metrics.DeployErrors.Add(ctx, 1, attrs)
403	}
404}
405
406// skipRedundantDeploy checks whether the existing container is already running
407// the same image (by Docker image ID) as the one we are about to deploy.
408// When a push triggers both an event-based deploy and an explicit CLI deploy,
409// the second one arrives after the first has already completed; this avoids
410// a full redundant create-start-readiness cycle.
411func (s *Service) skipRedundantDeploy(ctx context.Context, existing *domain.Container, actualImageRef, envHash string) (bool, *domain.Container) {
412	log := zerowrap.FromCtx(ctx)
413
414	newImageID, err := s.runtime.GetImageID(ctx, actualImageRef)
415	if err != nil {
416		log.Debug().Err(err).Msg("cannot resolve image ID for redundancy check, proceeding with deploy")
417		return false, nil
418	}
419
420	if existing.ImageID != newImageID {
421		return false, nil
422	}
423
424	// Verify the container is actually still running before skipping.
425	running, err := s.runtime.IsContainerRunning(ctx, existing.ID)
426	if err != nil || !running {
427		return false, nil
428	}
429
430	if existing.Labels != nil {
431		existingEnvHash, hasEnvHash := existing.Labels[domain.LabelEnvHash]
432		if !hasEnvHash || existingEnvHash != envHash {
433			log.Info().
434				Str("container_id", existing.ID).
435				Bool("has_env_hash", hasEnvHash).
436				Msg("env changed, proceeding with deploy despite same image")
437			return false, nil
438		}
439	} else {
440		return false, nil
441	}
442
443	log.Info().
444		Str("container_id", existing.ID).
445		Str("image_id", newImageID).
446		Msg("skipping redundant deploy: container already running this image")
447
448	return true, existing
449}
450
451func (s *Service) containerForRedundantCheck(ctx context.Context, existing *domain.Container) *domain.Container {
452	if existing == nil || existing.ImageID != "" || existing.ID == "" {
453		return existing
454	}
455
456	log := zerowrap.FromCtx(ctx)
457	inspected, err := s.runtime.InspectContainer(ctx, existing.ID)
458	if err != nil {
459		log.Debug().Err(err).Str("container_id", existing.ID).Msg("cannot inspect existing container for redundancy check")
460		return existing
461	}
462	if inspected == nil || inspected.ImageID == "" {
463		return existing
464	}
465
466	existingCopy := *existing
467	existingCopy.ImageID = inspected.ImageID
468	return &existingCopy
469}
470
471type deployResources struct {
472	networkName    string
473	actualImageRef string
474	exposedPorts   []int
475	imageLabels    map[string]string
476	envVars        []string
477	envHash        string
478	volumes        map[string]string
479}
480
481// resolveExistingContainer returns the currently running container for a domain.
482// It first checks the in-memory map (fast path). If not found, it queries the
483// runtime for a running container with matching name and managed label. This
484// handles cases where Gordon restarted and in-memory state is stale.
485//
486// The slow path checks canonical, -new, and -next names because a previous
487// deploy may have been interrupted (e.g. eventbus timeout, Gordon restart)
488// leaving the active container under a temp name. Canonical is preferred when
489// multiple matches exist.
490func (s *Service) resolveExistingContainer(ctx context.Context, domainName string) (*domain.Container, bool) {
491	// Fast path: check in-memory state
492	s.mu.RLock()
493	container, ok := s.containers[domainName]
494	s.mu.RUnlock()
495	if ok {
496		return container, true
497	}
498
499	// Slow path: query runtime for running containers
500	log := zerowrap.FromCtx(ctx)
501	canonicalName := fmt.Sprintf("gordon-%s", domainName)
502	candidateNames := map[string]bool{
503		canonicalName:           true,
504		canonicalName + "-new":  true,
505		canonicalName + "-next": true,
506	}
507
508	running, err := s.runtime.ListContainers(ctx, false)
509	if err != nil {
510		log.Warn().Err(err).Msg("failed to list running containers for existing container resolution")
511		return nil, false
512	}
513
514	// Prefer canonical name; fall back to any running managed temp container.
515	var best *domain.Container
516	for _, c := range running {
517		if !candidateNames[c.Name] || c.Labels[domain.LabelManaged] != "true" {
518			continue
519		}
520		if best == nil || c.Name == canonicalName {
521			best = c
522		}
523	}
524
525	if best != nil {
526		log.Info().
527			Str("container_id", best.ID).
528			Str("container_name", best.Name).
529			Msg("resolved existing container from runtime (in-memory state was stale)")
530
531		// Update in-memory state so subsequent lookups are fast
532		s.mu.Lock()
533		if _, alreadyTracked := s.containers[domainName]; !alreadyTracked {
534			s.managedCount++
535		}
536		s.containers[domainName] = best
537		s.mu.Unlock()
538
539		return best, true
540	}
541
542	return nil, false
543}
544
545func (s *Service) prepareDeployResources(ctx context.Context, route domain.Route, existing *domain.Container) (*deployResources, error) {
546	log := zerowrap.FromCtx(ctx)
547
548	existingID := ""
549	if existing != nil {
550		existingID = existing.ID
551	}
552	if err := s.cleanupOrphanedContainers(ctx, route.Domain, existingID); err != nil {
553		log.WrapErr(err, "failed to cleanup orphaned containers")
554	}
555
556	imageRef := s.buildImageRef(route.Image)
557	actualImageRef, err := s.ensureImage(ctx, imageRef)
558	if err != nil {
559		return nil, err
560	}
561
562	networkName := s.getNetworkForApp(route.Domain)
563	if err := s.createNetworkIfNeeded(ctx, networkName); err != nil {
564		return nil, log.WrapErr(err, "failed to create network")
565	}
566	if err := s.deployAttachments(ctx, route.Domain, networkName); err != nil {
567		return nil, log.WrapErr(err, "failed to deploy attachments")
568	}
569
570	exposedPorts, err := s.runtime.GetImageExposedPorts(ctx, actualImageRef)
571	if err != nil {
572		log.WrapErr(err, "failed to get exposed ports, using defaults")
573		exposedPorts = []int{80, 8080, 3000}
574	}
575
576	imageLabels, err := s.runtime.GetImageLabels(ctx, actualImageRef)
577	if err != nil {
578		log.Warn().Err(err).Msg("failed to get image labels, skipping label propagation")
579		imageLabels = nil
580	}
581
582	envVars, err := s.loadEnvironment(ctx, route.Env, route.Domain, actualImageRef)
583	if err != nil {
584		return nil, err
585	}
586	envHash := hashEnvironment(envVars)
587
588	volumes, err := s.setupVolumes(ctx, route.Domain, actualImageRef)
589	if err != nil {
590		return nil, err
591	}
592
593	return &deployResources{
594		networkName:    networkName,
595		actualImageRef: actualImageRef,
596		exposedPorts:   exposedPorts,
597		imageLabels:    imageLabels,
598		envVars:        envVars,
599		envHash:        envHash,
600		volumes:        volumes,
601	}, nil
602}
603
604func (s *Service) createStartedContainer(ctx context.Context, route domain.Route, existing *domain.Container, resources *deployResources) (*domain.Container, error) {
605	ctx, span := tracer.Start(ctx, "container.create_and_start")
606	defer span.End()
607
608	log := zerowrap.FromCtx(ctx)
609
610	containerConfig := s.buildContainerConfig(containerConfigInput{
611		Domain:       route.Domain,
612		Image:        route.Image,
613		ImageRef:     resources.actualImageRef,
614		ExposedPorts: resources.exposedPorts,
615		EnvVars:      resources.envVars,
616		EnvHash:      resources.envHash,
617		Volumes:      resources.volumes,
618		NetworkName:  resources.networkName,
619		ImageLabels:  resources.imageLabels,
620		Existing:     existing,
621	})
622
623	newContainer, err := s.runtime.CreateContainer(ctx, containerConfig)
624	if err != nil {
625		return nil, log.WrapErr(err, "failed to create container")
626	}
627	if err := s.runtime.StartContainer(ctx, newContainer.ID); err != nil {
628		s.runtime.RemoveContainer(ctx, newContainer.ID, true)
629		return nil, log.WrapErr(err, "failed to start container")
630	}
631	if !domain.IsSkipReadiness(ctx) {
632		if err := s.waitForReady(ctx, newContainer.ID, containerConfig); err != nil {
633			s.cleanupFailedContainer(ctx, newContainer.ID)
634			return nil, log.WrapErr(err, "container failed readiness check")
635		}
636	}
637
638	inspected, err := s.runtime.InspectContainer(ctx, newContainer.ID)
639	if err != nil {
640		s.cleanupFailedContainer(ctx, newContainer.ID)
641		return nil, log.WrapErr(err, "failed to inspect started container")
642	}
643
644	return inspected, nil
645}
646
647func (s *Service) activateDeployedContainer(ctx context.Context, domainName string, container *domain.Container) bool {
648	s.mu.Lock()
649	_, wasTracked := s.containers[domainName]
650	s.containers[domainName] = container
651	if !wasTracked {
652		s.managedCount++
653	}
654	s.mu.Unlock()
655
656	// Track managed container count (only increment for new domains, not replacements)
657	if !wasTracked && s.metrics != nil {
658		s.metrics.ManagedContainers.Add(ctx, 1)
659	}
660
661	s.publishContainerDeployed(ctx, domainName, container.ID)
662
663	s.mu.RLock()
664	inv := s.cacheInvalidator
665	s.mu.RUnlock()
666	if inv != nil {
667		inv.InvalidateTarget(ctx, domainName)
668		return true
669	}
670
671	return false
672}
673
674// stabilizeNewContainer monitors the new container briefly after traffic switch.
675// If it crashes during this window, rolls back to old container.
676// Returns true if stabilization succeeded, false if rollback was performed.
677// Returns an error only when both new and old containers are dead.
678func (s *Service) stabilizeNewContainer(ctx context.Context, domainName string, newContainer, oldContainer *domain.Container) (bool, error) {
679	log := zerowrap.FromCtx(ctx)
680
681	if oldContainer == nil {
682		return true, nil
683	}
684
685	s.mu.RLock()
686	delay := s.config.StabilizationDelay
687	s.mu.RUnlock()
688	if delay == 0 {
689		delay = 2 * time.Second
690	}
691
692	// Brief stabilization: verify new container is still running after delay
693	select {
694	case <-time.After(delay):
695	case <-ctx.Done():
696		return true, nil
697	}
698
699	running, err := s.runtime.IsContainerRunning(ctx, newContainer.ID)
700	if err != nil || !running {
701		log.Error().
702			Str("new_container", newContainer.ID).
703			Str("old_container", oldContainer.ID).
704			Msg("new container crashed during stabilization, rolling back to old")
705
706		// Verify old container is still running before restoring it.
707		// If both old and new are dead (e.g., OOM), returning a dead
708		// container as "existing" would leave the domain in a broken state.
709		oldRunning, oldErr := s.runtime.IsContainerRunning(ctx, oldContainer.ID)
710		if oldErr != nil || !oldRunning {
711			log.Error().
712				Str("old_container", oldContainer.ID).
713				Msg("old container is also not running, cannot rollback")
714
715			// Cleanup failed new container
716			if stopErr := s.runtime.StopContainer(ctx, newContainer.ID); stopErr != nil {
717				log.WrapErrWithFields(stopErr, "failed to stop failed new container during rollback", map[string]any{zerowrap.FieldEntityID: newContainer.ID})
718			}
719			if removeErr := s.runtime.RemoveContainer(ctx, newContainer.ID, true); removeErr != nil {
720				log.WrapErrWithFields(removeErr, "failed to remove failed new container during rollback", map[string]any{zerowrap.FieldEntityID: newContainer.ID})
721			}
722
723			return false, fmt.Errorf("stabilization failed: new container crashed and old container %s is also not running", oldContainer.ID)
724		}
725
726		// Rollback: restore old container as tracked
727		s.mu.Lock()
728		s.containers[domainName] = oldContainer
729		s.mu.Unlock()
730
731		// Re-invalidate proxy cache to point back to old
732		s.mu.RLock()
733		inv := s.cacheInvalidator
734		s.mu.RUnlock()
735		if inv != nil {
736			inv.InvalidateTarget(ctx, domainName)
737		}
738
739		// Cleanup failed new container
740		if stopErr := s.runtime.StopContainer(ctx, newContainer.ID); stopErr != nil {
741			log.WrapErrWithFields(stopErr, "failed to stop failed new container during rollback", map[string]any{zerowrap.FieldEntityID: newContainer.ID})
742		}
743		if removeErr := s.runtime.RemoveContainer(ctx, newContainer.ID, true); removeErr != nil {
744			log.WrapErrWithFields(removeErr, "failed to remove failed new container during rollback", map[string]any{zerowrap.FieldEntityID: newContainer.ID})
745		}
746
747		return false, nil
748	}
749
750	return true, nil
751}
752
753func (s *Service) finalizePreviousContainer(ctx context.Context, domainName string, existing *domain.Container, hasExisting, invalidated bool, newContainerID string) {
754	if !hasExisting {
755		return
756	}
757
758	if invalidated {
759		s.waitForDrain(ctx, existing.ID)
760	}
761
762	s.cleanupOldContainer(ctx, existing, newContainerID, domainName)
763}
764
765func (s *Service) waitForDrain(ctx context.Context, oldContainerID string) {
766	log := zerowrap.FromCtx(ctx)
767
768	s.mu.RLock()
769	cfg := s.config
770	waiter := s.drainWaiter
771	s.mu.RUnlock()
772
773	drainMode := cfg.DrainMode
774	if drainMode == "" {
775		drainMode = "delay"
776	}
777
778	shouldUseInFlight := (drainMode == "inflight" || drainMode == "auto") && waiter != nil
779	if shouldUseInFlight {
780		timeout := cfg.DrainTimeout
781		if timeout == 0 {
782			timeout = 30 * time.Second
783		}
784		drained := waiter.WaitForNoInFlight(ctx, oldContainerID, timeout)
785		if !drained {
786			log.Warn().
787				Str("old_container_id", oldContainerID).
788				Dur("drain_timeout", timeout).
789				Msg("drain wait timed out; old container may still have in-flight traffic")
790		}
791		return
792	}
793
794	drainDelay := 2 * time.Second
795	if cfg.DrainDelayConfigured {
796		drainDelay = cfg.DrainDelay
797	}
798	if drainDelay <= 0 {
799		return
800	}
801	select {
802	case <-time.After(drainDelay):
803	case <-ctx.Done():
804	}
805}
806
807// restartContainerWithRecovery restarts a container, reconciling stale state if needed.
808// It returns the (possibly refreshed) container and updated attachment IDs.
809func (s *Service) restartContainerWithRecovery(ctx context.Context, domainName string, container *domain.Container, attachmentIDs []string) (*domain.Container, []string, error) {
810	log := zerowrap.FromCtx(ctx)
811
812	if err := s.runtime.RestartContainer(ctx, container.ID); err != nil {
813		if !isContainerNotFoundError(err) {
814			return nil, nil, log.WrapErr(err, "failed to restart container")
815		}
816
817		// In-memory container ID can become stale after external runtime changes.
818		// Re-sync and retry once with the latest tracked container.
819		log.Warn().
820			Err(err).
821			Str(zerowrap.FieldEntityID, container.ID).
822			Msg("tracked container missing during restart, attempting state reconciliation")
823
824		if syncErr := s.SyncContainers(ctx); syncErr != nil {
825			log.Warn().Err(syncErr).Msg("failed to sync container state during restart recovery")
826		}
827
828		s.mu.RLock()
829		refreshed, refreshedExists := s.containers[domainName]
830		attachmentIDs = append([]string{}, s.attachments[domainName]...)
831		s.mu.RUnlock()
832
833		if !refreshedExists || refreshed == nil {
834			return nil, nil, domain.ErrContainerNotFound
835		}
836		if err := s.runtime.RestartContainer(ctx, refreshed.ID); err != nil {
837			return nil, nil, log.WrapErr(err, "failed to restart container after state reconciliation")
838		}
839		return refreshed, attachmentIDs, nil
840	}
841
842	return container, attachmentIDs, nil
843}
844
845// Restart restarts a running container for the given domain.
846func (s *Service) Restart(ctx context.Context, domainName string, withAttachments bool) error {
847	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
848		zerowrap.FieldLayer:   "usecase",
849		zerowrap.FieldUseCase: "Restart",
850		"domain":              domainName,
851	})
852	log := zerowrap.FromCtx(ctx)
853
854	s.mu.RLock()
855	container, exists := s.containers[domainName]
856	attachments := s.attachments[domainName]
857	var attachmentIDs []string
858	if len(attachments) > 0 {
859		attachmentIDs = make([]string, len(attachments))
860		copy(attachmentIDs, attachments)
861	}
862	s.mu.RUnlock()
863
864	if !exists || container == nil {
865		return domain.ErrContainerNotFound
866	}
867
868	// When attachments are requested, sync runtime state first to get accurate
869	// attachment tracking, then check if configured attachments are deployed.
870	// If fewer attachment containers are tracked than configured, fail early with
871	// guidance rather than restarting the main container into a broken state
872	// (e.g., missing database hostname).
873	if withAttachments {
874		if syncErr := s.SyncContainers(ctx); syncErr != nil {
875			log.Warn().Err(syncErr).Msg("failed to sync container state before attachment check")
876		}
877
878		// Re-read attachment IDs after sync
879		s.mu.RLock()
880		if freshAttachments := s.attachments[domainName]; len(freshAttachments) > 0 {
881			attachmentIDs = make([]string, len(freshAttachments))
882			copy(attachmentIDs, freshAttachments)
883		} else {
884			attachmentIDs = nil
885		}
886		s.mu.RUnlock()
887
888		configuredAttachments := s.resolveAttachmentsForDomain(domainName)
889		if len(configuredAttachments) > 0 && len(attachmentIDs) < len(configuredAttachments) {
890			missing := len(configuredAttachments) - len(attachmentIDs)
891			return fmt.Errorf("%w: domain %q has %d configured attachment(s) but only %d deployed (%d missing)",
892				domain.ErrAttachmentNotDeployed, domainName, len(configuredAttachments), len(attachmentIDs), missing)
893		}
894	}
895
896	// Restart the main container (with stale-state recovery)
897	container, attachmentIDs, err := s.restartContainerWithRecovery(ctx, domainName, container, attachmentIDs)
898	if err != nil {
899		return err
900	}
901	log.Info().Str(zerowrap.FieldEntityID, container.ID).Msg("container restarted")
902
903	// Record restart metric
904	if s.metrics != nil {
905		s.metrics.ContainerRestarts.Add(ctx, 1, metric.WithAttributes(
906			attribute.String("domain", domainName),
907			attribute.String("source", "api"),
908		))
909	}
910
911	// Restart attachments if requested
912	if withAttachments && len(attachmentIDs) > 0 {
913		for _, attachID := range attachmentIDs {
914			if err := s.runtime.RestartContainer(ctx, attachID); err != nil {
915				log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to restart attachment")
916				continue // Don't fail the whole operation for one attachment
917			}
918			log.Info().Str("attachment_id", attachID).Msg("attachment restarted")
919		}
920	}
921
922	return nil
923}
924
925// Stop stops a running container.
926func (s *Service) Stop(ctx context.Context, containerID string) error {
927	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
928		zerowrap.FieldLayer:    "usecase",
929		zerowrap.FieldUseCase:  "Stop",
930		zerowrap.FieldEntityID: containerID,
931	})
932	log := zerowrap.FromCtx(ctx)
933
934	// Stop log collection before stopping container
935	if s.logWriter != nil {
936		if err := s.logWriter.StopLogging(containerID); err != nil {
937			log.Warn().Err(err).Msg("failed to stop container log collection")
938		}
939	}
940
941	if err := s.runtime.StopContainer(ctx, containerID); err != nil {
942		return log.WrapErr(err, "failed to stop container")
943	}
944
945	log.Info().Msg("container stopped")
946	return nil
947}
948
949// Remove removes a container.
950func (s *Service) Remove(ctx context.Context, containerID string, force bool) error {
951	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
952		zerowrap.FieldLayer:    "usecase",
953		zerowrap.FieldUseCase:  "Remove",
954		zerowrap.FieldEntityID: containerID,
955	})
956	log := zerowrap.FromCtx(ctx)
957
958	// Stop log collection before removing container
959	if s.logWriter != nil {
960		if err := s.logWriter.StopLogging(containerID); err != nil {
961			log.Warn().Err(err).Msg("failed to stop container log collection")
962		}
963	}
964
965	// Find domain and attachment IDs for cleanup
966	containerDomain, attachmentIDs, cfg := s.findContainerContext(containerID)
967
968	if err := s.runtime.RemoveContainer(ctx, containerID, force); err != nil {
969		return log.WrapErr(err, "failed to remove container")
970	}
971
972	// Cleanup volumes
973	if containerDomain != "" && !cfg.VolumePreserve {
974		if err := s.cleanupVolumesForDomain(ctx, containerDomain); err != nil {
975			log.WrapErrWithFields(err, "failed to cleanup volumes", map[string]any{"domain": containerDomain})
976		}
977	}
978
979	// Clean up attachment containers
980	s.removeAttachments(ctx, attachmentIDs)
981
982	// Remove from tracking
983	s.mu.Lock()
984	var removedDomain string
985	for d, c := range s.containers {
986		if c.ID == containerID {
987			delete(s.containers, d)
988			removedDomain = d
989			s.managedCount--
990			log.Info().Str("domain", d).Msg("container removed")
991			break
992		}
993	}
994	if removedDomain != "" {
995		delete(s.attachments, removedDomain)
996	}
997	s.mu.Unlock()
998
999	// Decrement managed container count
1000	if removedDomain != "" && s.metrics != nil {
1001		s.metrics.ManagedContainers.Add(ctx, -1)
1002	}
1003
1004	// Note: We intentionally do NOT clean up deployMu entries to avoid a race condition:
1005	// If Remove deletes the mutex entry while a concurrent Deploy is acquiring the lock,
1006	// the Deploy would create a fresh mutex, breaking serialization and allowing concurrent
1007	// deploys to the same domain. The memory footprint is acceptable (one small struct per
1008	// domain ever deployed), and this is the safer choice for correctness.
1009
1010	// Cleanup network
1011	if removedDomain != "" && cfg.NetworkIsolation {
1012		networkName := s.getNetworkForApp(removedDomain)
1013		if err := s.cleanupNetworkIfEmpty(ctx, networkName); err != nil {
1014			log.WrapErrWithFields(err, "failed to cleanup network", map[string]any{"network": networkName})
1015		}
1016	}
1017
1018	return nil
1019}
1020
1021// findContainerContext looks up the domain, attachment IDs, and config snapshot for a container.
1022func (s *Service) findContainerContext(containerID string) (string, []string, Config) {
1023	s.mu.RLock()
1024	defer s.mu.RUnlock()
1025	cfg := s.config
1026	var containerDomain string
1027	for d, c := range s.containers {
1028		if c.ID == containerID {
1029			containerDomain = d
1030			break
1031		}
1032	}
1033	var attachmentIDs []string
1034	if containerDomain != "" {
1035		attachmentIDs = append(attachmentIDs, s.attachments[containerDomain]...)
1036	}
1037	return containerDomain, attachmentIDs, cfg
1038}
1039
1040// removeAttachments stops and removes attachment containers best-effort.
1041func (s *Service) removeAttachments(ctx context.Context, attachmentIDs []string) {
1042	log := zerowrap.FromCtx(ctx)
1043	for _, attachID := range attachmentIDs {
1044		if s.logWriter != nil {
1045			if err := s.logWriter.StopLogging(attachID); err != nil {
1046				log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to stop attachment log collection")
1047			}
1048		}
1049		if err := s.runtime.StopContainer(ctx, attachID); err != nil {
1050			log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to stop attachment container")
1051		}
1052		if err := s.runtime.RemoveContainer(ctx, attachID, true); err != nil {
1053			log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to remove attachment container")
1054		}
1055	}
1056}
1057
1058// Get retrieves a container by domain.
1059func (s *Service) Get(_ context.Context, domainName string) (*domain.Container, bool) {
1060	s.mu.RLock()
1061	defer s.mu.RUnlock()
1062	container, exists := s.containers[domainName]
1063	return container, exists
1064}
1065
1066// List returns all managed containers.
1067func (s *Service) List(_ context.Context) map[string]*domain.Container {
1068	s.mu.RLock()
1069	defer s.mu.RUnlock()
1070
1071	result := make(map[string]*domain.Container, len(s.containers))
1072	maps.Copy(result, s.containers)
1073	return result
1074}
1075
1076// ListRoutesWithDetails returns routes with network and attachment info.
1077// Note: Container map is copied under lock, then external calls are made without lock.
1078// If containers are removed between copy and runtime calls, errors are handled gracefully
1079// (network becomes empty string). This trade-off avoids holding locks during I/O.
1080func (s *Service) ListRoutesWithDetails(ctx context.Context) []domain.RouteInfo {
1081	s.mu.RLock()
1082	containers := make(map[string]*domain.Container, len(s.containers))
1083	maps.Copy(containers, s.containers)
1084	s.mu.RUnlock()
1085
1086	// Fetch all attachments once to avoid N+1 queries
1087	attachmentsByDomain := s.getAllAttachments(ctx)
1088
1089	results := make([]domain.RouteInfo, 0, len(containers))
1090	for domainName, container := range containers {
1091		network := ""
1092		image := ""
1093		containerID := ""
1094		status := ""
1095		if container != nil {
1096			containerID = container.ID
1097			status = container.Status
1098			image = container.Image
1099			if container.Labels != nil {
1100				if labelImage, ok := container.Labels[domain.LabelImage]; ok && labelImage != "" {
1101					image = labelImage
1102				}
1103			}
1104			// Strip registry domain prefix from image for cleaner display
1105			image = s.stripRegistryPrefix(image)
1106			if networkName, err := s.runtime.GetContainerNetwork(ctx, container.ID); err == nil {
1107				network = networkName
1108			}
1109		}
1110
1111		results = append(results, domain.RouteInfo{
1112			Domain:          domainName,
1113			Image:           image,
1114			ContainerID:     containerID,
1115			ContainerStatus: status,
1116			Network:         network,
1117			Attachments:     attachmentsByDomain[domainName],
1118		})
1119	}
1120
1121	return results
1122}
1123
1124// stripRegistryPrefix removes the configured registry domain prefix from an image reference.
1125// For example, "reg.example.com/myapp:latest" becomes "myapp:latest" if registry domain is "reg.example.com".
1126func (s *Service) stripRegistryPrefix(image string) string {
1127	s.mu.RLock()
1128	cfg := s.config
1129	s.mu.RUnlock()
1130
1131	if cfg.RegistryDomain == "" {
1132		return image
1133	}
1134	prefix := strings.TrimSuffix(cfg.RegistryDomain, "/") + "/"
1135	if strings.HasPrefix(image, prefix) {
1136		return strings.TrimPrefix(image, prefix)
1137	}
1138	return image
1139}
1140
1141// ListAttachments returns attachments for a domain.
1142func (s *Service) ListAttachments(ctx context.Context, domainName string) []domain.Attachment {
1143	return s.getAttachmentsForDomain(ctx, domainName)
1144}
1145
1146// ListNetworks returns Gordon-managed networks.
1147func (s *Service) ListNetworks(ctx context.Context) ([]*domain.NetworkInfo, error) {
1148	s.mu.RLock()
1149	cfg := s.config
1150	s.mu.RUnlock()
1151
1152	networks, err := s.runtime.ListNetworks(ctx)
1153	if err != nil {
1154		return nil, err
1155	}
1156
1157	var filtered []*domain.NetworkInfo
1158	for _, network := range networks {
1159		if strings.HasPrefix(network.Name, cfg.NetworkPrefix+"-") && network.Labels[domain.LabelManaged] == "true" {
1160			filtered = append(filtered, network)
1161		}
1162	}
1163
1164	return filtered, nil
1165}
1166
1167// HealthCheck performs health checks on all containers.
1168func (s *Service) HealthCheck(ctx context.Context) map[string]bool {
1169	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1170		zerowrap.FieldLayer:   "usecase",
1171		zerowrap.FieldUseCase: "HealthCheck",
1172	})
1173	log := zerowrap.FromCtx(ctx)
1174
1175	// Copy container IDs under lock, then release before making Docker API calls
1176	s.mu.RLock()
1177	snapshot := make(map[string]string, len(s.containers))
1178	for d, c := range s.containers {
1179		snapshot[d] = c.ID
1180	}
1181	s.mu.RUnlock()
1182
1183	health := make(map[string]bool, len(snapshot))
1184	for d, id := range snapshot {
1185		running, err := s.runtime.IsContainerRunning(ctx, id)
1186		if err != nil {
1187			log.WrapErrWithFields(err, "health check failed", map[string]any{"domain": d, zerowrap.FieldEntityID: id})
1188			health[d] = false
1189		} else {
1190			health[d] = running
1191		}
1192	}
1193	return health
1194}
1195
1196// SyncContainers synchronizes containers with runtime state.
1197func (s *Service) SyncContainers(ctx context.Context) error {
1198	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1199		zerowrap.FieldLayer:   "usecase",
1200		zerowrap.FieldUseCase: "SyncContainers",
1201	})
1202	log := zerowrap.FromCtx(ctx)
1203
1204	// List containers without holding lock to avoid blocking during Docker API call
1205	allContainers, err := s.runtime.ListContainers(ctx, false)
1206	if err != nil {
1207		return log.WrapErr(err, "failed to list containers")
1208	}
1209
1210	managed := make(map[string]*domain.Container)
1211	attachments := make(map[string][]string)
1212	for _, c := range allContainers {
1213		if c.Labels == nil || c.Labels[domain.LabelManaged] != "true" {
1214			continue
1215		}
1216		if c.Labels[domain.LabelAttachment] == "true" {
1217			owner := c.Labels[domain.LabelAttachedTo]
1218			if owner != "" {
1219				attachments[owner] = append(attachments[owner], c.ID)
1220			}
1221			continue
1222		}
1223		if d, ok := c.Labels[domain.LabelDomain]; ok {
1224			managed[d] = c
1225		}
1226	}
1227
1228	s.mu.Lock()
1229	s.containers = managed
1230	s.attachments = attachments
1231	newCount := int64(len(managed))
1232	delta := newCount - s.managedCount
1233	s.managedCount = newCount
1234	s.mu.Unlock()
1235
1236	// Report initial/delta managed container count to OTel UpDownCounter.
1237	if delta != 0 && s.metrics != nil {
1238		s.metrics.ManagedContainers.Add(ctx, delta)
1239	}
1240
1241	log.Info().Int(zerowrap.FieldCount, len(managed)).Msg("container state synchronized")
1242	return nil
1243}
1244
1245// AutoStart starts containers for the provided routes that aren't running.
1246// It skips readiness checks to avoid blocking boot; the background monitor
1247// handles crash recovery.
1248func (s *Service) AutoStart(ctx context.Context, routes []domain.Route) error {
1249	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1250		zerowrap.FieldLayer:   "usecase",
1251		zerowrap.FieldUseCase: "AutoStart",
1252	})
1253	log := zerowrap.FromCtx(ctx)
1254
1255	log.Info().Int("route_count", len(routes)).Msg("auto-starting containers for configured routes")
1256
1257	// Filter out routes that already have a running container.
1258	var pending []domain.Route
1259	skipped := 0
1260	for _, route := range routes {
1261		if _, exists := s.Get(ctx, route.Domain); exists {
1262			log.Debug().Str("domain", route.Domain).Msg("container already running, skipping")
1263			skipped++
1264		} else {
1265			pending = append(pending, route)
1266		}
1267	}
1268
1269	if len(pending) == 0 {
1270		log.Info().Int("skipped", skipped).Msg("auto-start completed, all containers already running")
1271		return nil
1272	}
1273
1274	// Deploy all pending routes concurrently with readiness checks skipped.
1275	deployCtx := domain.WithSkipReadiness(ctx)
1276
1277	type result struct {
1278		route domain.Route
1279		err   error
1280	}
1281	results := make(chan result, len(pending))
1282	sem := make(chan struct{}, 4) // limit concurrency
1283
1284	for _, route := range pending {
1285		sem <- struct{}{}
1286		go func(r domain.Route) {
1287			defer func() { <-sem }()
1288
1289			log.Info().Str("domain", r.Domain).Str("image", r.Image).Msg("auto-starting container for route")
1290
1291			if _, err := s.Deploy(deployCtx, r); err != nil {
1292				results <- result{route: r, err: err}
1293				return
1294			}
1295			results <- result{route: r}
1296		}(route)
1297	}
1298
1299	var started, errCount int
1300	for i := 0; i < len(pending); i++ {
1301		res := <-results
1302		if res.err != nil {
1303			log.Warn().Err(res.err).Str("domain", res.route.Domain).Msg("failed to auto-start container")
1304			errCount++
1305		} else {
1306			started++
1307		}
1308	}
1309
1310	log.Info().
1311		Int("started", started).
1312		Int("skipped", skipped).
1313		Int("errors", errCount).
1314		Msg("auto-start completed")
1315
1316	if errCount > 0 {
1317		return fmt.Errorf("auto-start completed with %d errors", errCount)
1318	}
1319	return nil
1320}
1321
1322// Shutdown gracefully shuts down all managed containers.
1323func (s *Service) Shutdown(ctx context.Context) error {
1324	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1325		zerowrap.FieldLayer:   "usecase",
1326		zerowrap.FieldUseCase: "Shutdown",
1327	})
1328	log := zerowrap.FromCtx(ctx)
1329	log.Info().Msg("shutting down container manager...")
1330
1331	// Wait for any in-flight background cleanup goroutines to finish
1332	s.cleanupWg.Wait()
1333
1334	// Containers are left running across Gordon restarts.
1335	// SyncContainers + AutoStart will pick them back up on next boot.
1336
1337	// Close log writer to stop all log collection
1338	if s.logWriter != nil {
1339		if err := s.logWriter.Close(); err != nil {
1340			log.Warn().Err(err).Msg("failed to close container log writer")
1341		}
1342	}
1343
1344	log.Info().Msg("container manager shutdown complete")
1345	return nil
1346}
1347
1348// WaitForCleanup blocks until all background cleanup goroutines complete.
1349// Intended for use in tests to avoid mock assertion races.
1350func (s *Service) WaitForCleanup() {
1351	s.cleanupWg.Wait()
1352}
1353
1354// StartMonitor begins background monitoring of tracked containers.
1355// The monitor restarts crashed containers and detects crash loops.
1356// Safe to call multiple times; subsequent calls are no-ops.
1357func (s *Service) StartMonitor(ctx context.Context) {
1358	s.mu.Lock()
1359	if s.monitor != nil {
1360		s.mu.Unlock()
1361		return
1362	}
1363	s.monitor = newMonitor(s)
1364	m := s.monitor
1365	m.Start(ctx)
1366	s.mu.Unlock()
1367}
1368
1369// StopMonitor stops the background container monitor.
1370// Safe to call multiple times; subsequent calls are no-ops.
1371func (s *Service) StopMonitor() {
1372	s.mu.Lock()
1373	m := s.monitor
1374	s.monitor = nil
1375	s.mu.Unlock()
1376	if m != nil {
1377		m.Stop()
1378	}
1379}
1380
1381// UpdateConfig updates the service configuration.
1382func (s *Service) UpdateConfig(config Config) {
1383	s.mu.Lock()
1384	s.config = config
1385	s.mu.Unlock()
1386}
1387
1388// UpdateAttachments updates only the attachment configuration in the service.
1389// This is called after a config reload to propagate attachment changes without restart.
1390// The incoming map is deep-copied so external callers cannot mutate service state.
1391func (s *Service) UpdateAttachments(attachments map[string][]string) {
1392	var copied map[string][]string
1393	if attachments != nil {
1394		copied = make(map[string][]string, len(attachments))
1395		for k, v := range attachments {
1396			sl := make([]string, len(v))
1397			copy(sl, v)
1398			copied[k] = sl
1399		}
1400	}
1401	s.mu.Lock()
1402	s.config.Attachments = copied
1403	s.mu.Unlock()
1404}
1405
1406// Helper methods
1407
1408// cleanupFailedContainer stops and removes a container that failed to start properly.
1409func (s *Service) cleanupFailedContainer(ctx context.Context, containerID string) {
1410	log := zerowrap.FromCtx(ctx)
1411	if err := s.runtime.StopContainer(ctx, containerID); err != nil {
1412		log.Warn().Err(err).Str(zerowrap.FieldEntityID, containerID).Msg("failed to stop container after failure")
1413	}
1414	if err := s.runtime.RemoveContainer(ctx, containerID, true); err != nil {
1415		log.Warn().Err(err).Str(zerowrap.FieldEntityID, containerID).Msg("failed to remove container after failure")
1416	}
1417}
1418
1419// cleanupOldContainer stops and removes an old container after zero-downtime switch.
1420// It also renames the new container to the canonical name.
1421func (s *Service) cleanupOldContainer(ctx context.Context, old *domain.Container, newContainerID, domainName string) {
1422	log := zerowrap.FromCtx(ctx)
1423	log.Info().Str(zerowrap.FieldEntityID, old.ID).Msg("stopping old container after zero-downtime switch")
1424
1425	if s.logWriter != nil {
1426		if err := s.logWriter.StopLogging(old.ID); err != nil {
1427			log.Warn().Err(err).Str(zerowrap.FieldEntityID, old.ID).Msg("failed to stop logging for old container")
1428		}
1429	}
1430
1431	if err := s.runtime.StopContainer(ctx, old.ID); err != nil {
1432		log.Warn().Err(err).Str(zerowrap.FieldEntityID, old.ID).Msg("failed to stop old container")
1433	}
1434
1435	if err := s.runtime.RemoveContainer(ctx, old.ID, true); err != nil {
1436		log.Warn().Err(err).Str(zerowrap.FieldEntityID, old.ID).Msg("failed to remove old container")
1437	}
1438
1439	// Rename new container to canonical name
1440	canonicalName := fmt.Sprintf("gordon-%s", domainName)
1441	if err := s.runtime.RenameContainer(ctx, newContainerID, canonicalName); err != nil {
1442		log.Warn().Err(err).Str("canonical_name", canonicalName).Msg("failed to rename container to canonical name")
1443	}
1444}
1445
1446// startLogCollection starts log collection for a container in the background.
1447// Errors are logged but don't fail the calling operation.
1448func (s *Service) startLogCollection(ctx context.Context, containerID, domainName string) {
1449	if s.logWriter == nil {
1450		return
1451	}
1452
1453	log := zerowrap.FromCtx(ctx)
1454
1455	// Use context.WithoutCancel so the log stream outlives the HTTP request.
1456	// Without this, the Docker log stream closes when the deploy response completes.
1457	bgCtx := context.WithoutCancel(ctx)
1458
1459	logStream, err := s.runtime.GetContainerLogs(bgCtx, containerID, true)
1460	if err != nil {
1461		log.Warn().Err(err).Msg("failed to get container logs for collection")
1462		return
1463	}
1464
1465	if err := s.logWriter.StartLogging(bgCtx, containerID, domainName, logStream); err != nil {
1466		log.Warn().Err(err).Msg("failed to start container log collection")
1467		logStream.Close()
1468	}
1469}
1470
1471func (s *Service) buildImageRef(image string) string {
1472	s.mu.RLock()
1473	cfg := s.config
1474	s.mu.RUnlock()
1475
1476	if !cfg.RegistryAuthEnabled || cfg.RegistryDomain == "" {
1477		return image
1478	}
1479
1480	// Normalize registry domain by trimming any trailing slash
1481	reg := strings.TrimSuffix(cfg.RegistryDomain, "/")
1482
1483	// Check if image already has the registry domain prefix
1484	prefix := reg + "/"
1485	if strings.HasPrefix(image, prefix) {
1486		return image
1487	}
1488
1489	// Check if image already has an explicit registry (host:port/ or host.domain/).
1490	// We need to detect patterns like:
1491	// - "docker.io/library/nginx:latest" (has dot and slash)
1492	// - "localhost:5001/myapp:latest" (has host:port/ pattern)
1493	// - "myregistry:5000/app:v1" (has host:port/ pattern)
1494	// - "[fd00::1]:5000/app:v1" (IPv6 with port)
1495	if hasExplicitRegistry(image) {
1496		return image
1497	}
1498
1499	return fmt.Sprintf("%s/%s", reg, image)
1500}
1501
1502// hasExplicitRegistry checks if an image reference already includes an explicit registry.
1503// Returns true for patterns like "docker.io/image", "localhost:5000/image", "localhost/image",
1504// "registry:8080/image", "[fd00::1]/image", "[fd00::1]:5000/image".
1505func hasExplicitRegistry(image string) bool {
1506	// Find the first slash which separates registry from image name
1507	slashIdx := strings.Index(image, "/")
1508	if slashIdx == -1 {
1509		return false // No slash means no registry prefix (e.g., "myapp:latest")
1510	}
1511
1512	// Extract the part before the first slash (potential registry)
1513	registryPart := image[:slashIdx]
1514
1515	// Check for bracketed IPv6 address (e.g., "[fd00::1]" or "[fd00::1]:5000")
1516	if strings.HasPrefix(registryPart, "[") {
1517		// Look for closing bracket
1518		if closeBracket := strings.Index(registryPart, "]"); closeBracket != -1 {
1519			// Valid bracketed IPv6: either ends at ] or has ]:port
1520			return true
1521		}
1522	}
1523
1524	// Check for localhost (with or without port)
1525	if registryPart == "localhost" {
1526		return true
1527	}
1528
1529	// Check for host:port pattern (e.g., "localhost:5000", "registry:8080")
1530	if colonIdx := strings.LastIndex(registryPart, ":"); colonIdx != -1 {
1531		port := registryPart[colonIdx+1:]
1532		// If everything after colon is digits, it's a port
1533		if len(port) > 0 && isNumeric(port) {
1534			return true
1535		}
1536	}
1537
1538	// Check for domain pattern (has a dot, e.g., "docker.io", "gcr.io")
1539	if strings.Contains(registryPart, ".") {
1540		return true
1541	}
1542
1543	return false
1544}
1545
1546// isNumeric checks if a string contains only digits.
1547func isNumeric(s string) bool {
1548	for _, c := range s {
1549		if c < '0' || c > '9' {
1550			return false
1551		}
1552	}
1553	return true
1554}
1555
1556func normalizePullPolicy(policy string) string {
1557	switch strings.ToLower(strings.TrimSpace(policy)) {
1558	case PullPolicyAlways:
1559		return PullPolicyAlways
1560	case PullPolicyIfTagChanged:
1561		return PullPolicyIfTagChanged
1562	case PullPolicyIfNotPresent:
1563		return PullPolicyIfNotPresent
1564	default:
1565		return PullPolicyIfNotPresent
1566	}
1567}
1568
1569func isDigestRef(imageRef string) bool {
1570	return strings.Contains(imageRef, "@sha256:")
1571}
1572
1573func (s *Service) pullRefForDeploy(ctx context.Context, imageRef string) (string, bool) {
1574	if !domain.IsInternalDeploy(ctx) {
1575		return imageRef, false
1576	}
1577	s.mu.RLock()
1578	cfg := s.config
1579	s.mu.RUnlock()
1580	return rewriteToLocalRegistry(imageRef, cfg.RegistryDomain, cfg.RegistryPort), true
1581}
1582
1583// ensureImage ensures the image is available locally, pulling if needed.
1584// Returns the image reference to use for container operations:
1585//   - For digest references (@sha256:...), returns the pullRef since Docker can't tag digests
1586//   - For tagged images, returns the original imageRef after tagging the pulled image
1587func (s *Service) ensureImage(ctx context.Context, imageRef string) (string, error) {
1588	ctx, span := tracer.Start(ctx, "container.ensure_image",
1589		trace.WithAttributes(attribute.String("image", imageRef)))
1590	defer span.End()
1591
1592	ctx = zerowrap.CtxWithField(ctx, "image", imageRef)
1593	log := zerowrap.FromCtx(ctx)
1594
1595	// Determine if this is an internal deploy and what reference to use for pulls.
1596	pullRef, isInternal := s.pullRefForDeploy(ctx, imageRef)
1597	if pullRef != imageRef {
1598		log.Info().
1599			Str("original_ref", imageRef).
1600			Str("pull_ref", pullRef).
1601			Msg("internal deploy: using localhost registry for pull")
1602	}
1603
1604	found, err := s.ensureLocalImage(ctx, imageRef, pullRef)
1605	if err != nil {
1606		return "", err
1607	}
1608	if found {
1609		return imageRef, nil
1610	}
1611
1612	// Pull image
1613	log.Info().Msg("pulling image from registry")
1614
1615	if err := s.pullImage(ctx, pullRef, isInternal); err != nil {
1616		return "", err
1617	}
1618
1619	// For digest references, we can't create a tag (Docker doesn't allow it).
1620	// In this case, use the pullRef directly since the image is available under that reference.
1621	if strings.Contains(imageRef, "@sha256:") {
1622		log.Info().
1623			Str("pull_ref", pullRef).
1624			Msg("digest reference: using pull reference for container operations")
1625		return pullRef, nil
1626	}
1627
1628	if err := s.tagImageIfNeeded(ctx, pullRef, imageRef); err != nil {
1629		return "", err
1630	}
1631
1632	// Clean up the temporary pull reference tag to avoid duplicate entries
1633	if pullRef != imageRef {
1634		if err := s.runtime.UntagImage(ctx, pullRef); err != nil {
1635			// Log but don't fail - the canonical tag is already applied
1636			log.Debug().Err(err).Str("pull_ref", pullRef).Msg("failed to remove temporary pull tag")
1637		}
1638	}
1639
1640	log.Info().Msg("image pulled successfully")
1641	return imageRef, nil
1642}
1643
1644func (s *Service) ensureLocalImage(ctx context.Context, imageRef, pullRef string) (bool, error) {
1645	log := zerowrap.FromCtx(ctx)
1646
1647	// For internal deploys (image push events), always pull fresh image
1648	// because the same tag may reference new content (e.g., latest tag updated)
1649	if domain.IsInternalDeploy(ctx) {
1650		log.Info().Msg("internal deploy detected, forcing image pull to ensure latest content")
1651		return false, nil
1652	}
1653
1654	s.mu.RLock()
1655	cfg := s.config
1656	s.mu.RUnlock()
1657
1658	pullPolicy := normalizePullPolicy(cfg.PullPolicy)
1659	switch pullPolicy {
1660	case PullPolicyAlways:
1661		log.Info().Str("pull_policy", pullPolicy).Msg("pull policy forces image pull")
1662		return false, nil
1663	case PullPolicyIfTagChanged:
1664		if !isDigestRef(imageRef) {
1665			log.Info().Str("pull_policy", pullPolicy).Msg("tag reference detected, pulling to check for updates")
1666			return false, nil
1667		}
1668	}
1669
1670	localImages, err := s.runtime.ListImages(ctx)
1671	if err != nil {
1672		log.WrapErr(err, "failed to list local images, will attempt pull")
1673		return false, nil
1674	}
1675
1676	normalizedRef := normalizeImageRef(imageRef)
1677	normalizedPullRef := normalizeImageRef(pullRef)
1678	for _, img := range localImages {
1679		normalizedImage := normalizeImageRef(img)
1680		if normalizedImage == normalizedRef {
1681			log.Info().Msg("image found locally, skipping pull")
1682			return true, nil
1683		}
1684		if normalizedImage == normalizedPullRef {
1685			if err := s.tagImageIfNeeded(ctx, pullRef, imageRef); err != nil {
1686				return false, err
1687			}
1688			// Clean up the temporary pull reference tag
1689			if pullRef != imageRef {
1690				if err := s.runtime.UntagImage(ctx, pullRef); err != nil {
1691					log.Debug().Err(err).Str("pull_ref", pullRef).Msg("failed to remove temporary pull tag")
1692				}
1693			}
1694			log.Info().Msg("image found locally, skipping pull")
1695			return true, nil
1696		}
1697	}
1698
1699	return false, nil
1700}
1701
1702func (s *Service) pullImage(ctx context.Context, pullRef string, isInternal bool) error {
1703	s.mu.RLock()
1704	cfg := s.config
1705	s.mu.RUnlock()
1706
1707	log := zerowrap.FromCtx(ctx)
1708
1709	switch {
1710	case isInternal && cfg.RegistryAuthEnabled:
1711		if cfg.InternalRegistryUsername == "" || cfg.InternalRegistryPassword == "" {
1712			return log.WrapErr(fmt.Errorf("internal registry auth not configured"), "failed to pull image for internal deploy")
1713		}
1714		if err := s.pullImageWithRetry(ctx, func(ctx context.Context) error {
1715			return s.runtime.PullImageWithAuth(ctx, pullRef, cfg.InternalRegistryUsername, cfg.InternalRegistryPassword)
1716		}, isInternal); err != nil {
1717			return log.WrapErr(err, "failed to pull image with internal auth")
1718		}
1719	case isInternal:
1720		if err := s.pullImageWithRetry(ctx, func(ctx context.Context) error {
1721			return s.runtime.PullImage(ctx, pullRef)
1722		}, isInternal); err != nil {
1723			return log.WrapErr(err, "failed to pull image")
1724		}
1725	case cfg.RegistryAuthEnabled:
1726		if cfg.ServiceTokenUsername == "" || cfg.ServiceToken == "" {
1727			return log.WrapErr(fmt.Errorf("registry service token not configured"), "failed to pull image for registry auth")
1728		}
1729		if err := s.runtime.PullImageWithAuth(ctx, pullRef, cfg.ServiceTokenUsername, cfg.ServiceToken); err != nil {
1730			return log.WrapErr(err, "failed to pull image with auth")
1731		}
1732	default:
1733		if err := s.runtime.PullImage(ctx, pullRef); err != nil {
1734			return log.WrapErr(err, "failed to pull image")
1735		}
1736	}
1737
1738	return nil
1739}
1740
1741func (s *Service) pullImageWithRetry(ctx context.Context, pullFn func(context.Context) error, isInternal bool) error {
1742	log := zerowrap.FromCtx(ctx)
1743
1744	for attempt := 1; attempt <= internalPullMaxAttempts; attempt++ {
1745		err := pullFn(ctx)
1746		if err == nil {
1747			return nil
1748		}
1749		if !isInternal || !isConnectionRefusedError(err) || attempt == internalPullMaxAttempts {
1750			return fmt.Errorf("internal image pull failed after %d attempts: %w", attempt, err)
1751		}
1752
1753		backoff := time.Duration(attempt) * time.Second
1754		log.Warn().
1755			Err(err).
1756			Int("attempt", attempt).
1757			Dur("backoff", backoff).
1758			Msg("internal image pull failed with connection refused, retrying")
1759
1760		select {
1761		case <-time.After(backoff):
1762		case <-ctx.Done():
1763			return ctx.Err()
1764		}
1765	}
1766	return nil
1767}
1768
1769func (s *Service) tagImageIfNeeded(ctx context.Context, sourceRef, targetRef string) error {
1770	if sourceRef == targetRef {
1771		return nil
1772	}
1773
1774	// Cannot create a tag with a digest reference - Docker/Podman doesn't allow it.
1775	// When using digest references (image@sha256:...), skip tagging as the image
1776	// is already available by its digest.
1777	if strings.Contains(targetRef, "@sha256:") {
1778		return nil
1779	}
1780
1781	log := zerowrap.FromCtx(ctx)
1782	if err := s.runtime.TagImage(ctx, sourceRef, targetRef); err != nil {
1783		return log.WrapErr(err, "failed to tag image from pull reference")
1784	}
1785	return nil
1786}
1787
1788func (s *Service) loadEnvironment(ctx context.Context, preResolved []string, domainName, imageRef string) ([]string, error) {
1789	log := zerowrap.FromCtx(ctx)
1790
1791	var userEnvVars []string
1792	if len(preResolved) > 0 {
1793		userEnvVars = preResolved
1794	} else {
1795		var err error
1796		userEnvVars, err = s.envLoader.LoadEnv(ctx, domainName)
1797		if err != nil {
1798			return nil, log.WrapErr(err, "failed to load environment variables")
1799		}
1800	}
1801
1802	dockerfileEnvVars, err := s.runtime.InspectImageEnv(ctx, imageRef)
1803	if err != nil {
1804		log.WrapErr(err, "failed to inspect image environment")
1805		dockerfileEnvVars = []string{}
1806	}
1807
1808	return mergeEnvironmentVariables(dockerfileEnvVars, userEnvVars), nil
1809}
1810
1811func (s *Service) setupVolumes(ctx context.Context, domainName, imageRef string) (map[string]string, error) {
1812	s.mu.RLock()
1813	cfg := s.config
1814	s.mu.RUnlock()
1815
1816	log := zerowrap.FromCtx(ctx)
1817	volumes := make(map[string]string)
1818
1819	if !cfg.VolumeAutoCreate {
1820		return volumes, nil
1821	}
1822
1823	volumePaths, err := s.runtime.InspectImageVolumes(ctx, imageRef)
1824	if err != nil {
1825		log.WrapErr(err, "failed to inspect image volumes")
1826		return volumes, nil
1827	}
1828
1829	for _, path := range volumePaths {
1830		name := generateVolumeName(cfg.VolumePrefix, domainName, path)
1831
1832		exists, err := s.runtime.VolumeExists(ctx, name)
1833		if err != nil {
1834			log.WrapErrWithFields(err, "failed to check volume", map[string]any{"volume": name})
1835			continue
1836		}
1837
1838		if !exists {
1839			if err := s.runtime.CreateVolume(ctx, name); err != nil {
1840				log.WrapErrWithFields(err, "failed to create volume", map[string]any{"volume": name})
1841				continue
1842			}
1843			log.Info().Str("volume", name).Str(zerowrap.FieldPath, path).Msg("created volume")
1844		}
1845
1846		volumes[path] = name
1847	}
1848
1849	return volumes, nil
1850}
1851
1852func (s *Service) getNetworkForApp(domainName string) string {
1853	var networkIsolation bool
1854	var networkGroups map[string][]string
1855
1856	if s.configProvider != nil {
1857		snap := s.configProvider.GetAttachmentConfig()
1858		networkGroups = snap.NetworkGroups
1859		s.mu.RLock()
1860		networkIsolation = s.config.NetworkIsolation
1861		s.mu.RUnlock()
1862	} else {
1863		s.mu.RLock()
1864		networkIsolation = s.config.NetworkIsolation
1865		networkGroups = s.config.NetworkGroups
1866		s.mu.RUnlock()
1867	}
1868
1869	if !networkIsolation {
1870		return "bridge"
1871	}
1872
1873	for groupName, domains := range networkGroups {
1874		if slices.Contains(domains, domainName) {
1875			return s.generateNetworkName(groupName)
1876		}
1877	}
1878
1879	return s.generateNetworkName(domainName)
1880}
1881
1882func (s *Service) generateNetworkName(identifier string) string {
1883	s.mu.RLock()
1884	cfg := s.config
1885	s.mu.RUnlock()
1886
1887	return fmt.Sprintf("%s-%s", cfg.NetworkPrefix, strings.ReplaceAll(identifier, ".", "-"))
1888}
1889
1890func (s *Service) createNetworkIfNeeded(ctx context.Context, networkName string) error {
1891	if networkName == "bridge" || networkName == "default" {
1892		return nil
1893	}
1894
1895	ctx = zerowrap.CtxWithField(ctx, "network", networkName)
1896	log := zerowrap.FromCtx(ctx)
1897
1898	exists, err := s.runtime.NetworkExists(ctx, networkName)
1899	if err != nil {
1900		return log.WrapErr(err, "failed to check network existence")
1901	}
1902
1903	if !exists {
1904		if err := s.runtime.CreateNetwork(ctx, networkName, map[string]string{"driver": "bridge"}); err != nil {
1905			return log.WrapErr(err, "failed to create network")
1906		}
1907		log.Info().Msg("created network for app isolation")
1908	}
1909
1910	return nil
1911}
1912
1913func (s *Service) cleanupOrphanedContainers(ctx context.Context, domainName string, skipContainerID string) error {
1914	log := zerowrap.FromCtx(ctx)
1915	expectedName := fmt.Sprintf("gordon-%s", domainName)
1916	expectedNewName := expectedName + "-new"
1917	expectedNextName := expectedName + "-next"
1918
1919	allContainers, err := s.runtime.ListContainers(ctx, true)
1920	if err != nil {
1921		return err
1922	}
1923
1924	for _, c := range allContainers {
1925		if (c.Name == expectedName || c.Name == expectedNewName || c.Name == expectedNextName) && c.ID != skipContainerID {
1926			// Skip any running container regardless of name — a running
1927			// temp container may be actively serving traffic while the
1928			// new container stabilizes, or may be from a concurrent deploy.
1929			if c.Status == "running" {
1930				log.Debug().
1931					Str(zerowrap.FieldEntityID, c.ID).
1932					Str("container_name", c.Name).
1933					Str(zerowrap.FieldStatus, c.Status).
1934					Msg("skipping running container during orphan cleanup")
1935				continue
1936			}
1937
1938			log.Info().Str(zerowrap.FieldEntityID, c.ID).Str("container_name", c.Name).Str(zerowrap.FieldStatus, c.Status).Msg("found orphaned container, removing")
1939
1940			if err := s.runtime.StopContainer(ctx, c.ID); err != nil {
1941				log.WrapErrWithFields(err, "failed to stop orphaned container", map[string]any{zerowrap.FieldEntityID: c.ID})
1942			}
1943
1944			if err := s.runtime.RemoveContainer(ctx, c.ID, true); err != nil {
1945				log.WrapErrWithFields(err, "failed to remove orphaned container", map[string]any{zerowrap.FieldEntityID: c.ID})
1946			}
1947		}
1948	}
1949
1950	return nil
1951}
1952
1953func (s *Service) cleanupVolumesForDomain(_ context.Context, _ string) error {
1954	return nil
1955}
1956
1957func (s *Service) cleanupNetworkIfEmpty(ctx context.Context, networkName string) error {
1958	if networkName == "bridge" || networkName == "default" {
1959		return nil
1960	}
1961
1962	ctx = zerowrap.CtxWithField(ctx, "network", networkName)
1963	log := zerowrap.FromCtx(ctx)
1964
1965	networks, err := s.runtime.ListNetworks(ctx)
1966	if err != nil {
1967		return log.WrapErr(err, "failed to list networks")
1968	}
1969
1970	for _, n := range networks {
1971		if n.Name == networkName && len(n.Containers) == 0 {
1972			if err := s.runtime.RemoveNetwork(ctx, networkName); err != nil {
1973				return err
1974			}
1975			log.Info().Msg("cleaned up empty network")
1976			break
1977		}
1978	}
1979
1980	return nil
1981}
1982
1983func (s *Service) deployAttachments(ctx context.Context, domainName, networkName string) error {
1984	// Collect attachments from both domain-specific and network group configs
1985	attachments := s.resolveAttachmentsForDomain(domainName)
1986	if len(attachments) == 0 {
1987		return nil
1988	}
1989
1990	log := zerowrap.FromCtx(ctx)
1991	var deployed []string // track successfully deployed attachment IDs for rollback
1992	for _, svc := range attachments {
1993		if err := s.deployAttachedService(ctx, domainName, svc, networkName); err != nil {
1994			log.WrapErrWithFields(err, "failed to deploy attachment", map[string]any{zerowrap.FieldService: svc, "domain": domainName})
1995
1996			// Rollback: clean up already-deployed attachments
1997			for _, id := range deployed {
1998				if stopErr := s.runtime.StopContainer(ctx, id); stopErr != nil {
1999					log.Warn().Err(stopErr).Str("attachment_id", id).Msg("rollback: failed to stop attachment")
2000				}
2001				if rmErr := s.runtime.RemoveContainer(ctx, id, true); rmErr != nil {
2002					log.Warn().Err(rmErr).Str("attachment_id", id).Msg("rollback: failed to remove attachment")
2003				}
2004			}
2005			// Deregister rolled-back attachments
2006			if len(deployed) > 0 {
2007				s.mu.Lock()
2008				remaining := s.attachments[domainName]
2009				filtered := remaining[:0]
2010				rollbackSet := make(map[string]bool, len(deployed))
2011				for _, id := range deployed {
2012					rollbackSet[id] = true
2013				}
2014				for _, id := range remaining {
2015					if !rollbackSet[id] {
2016						filtered = append(filtered, id)
2017					}
2018				}
2019				s.attachments[domainName] = filtered
2020				s.mu.Unlock()
2021			}
2022
2023			return fmt.Errorf("failed to deploy attachment %q (rolled back %d already-deployed)", svc, len(deployed))
2024		}
2025		// Collect IDs of attachments tracked under this domain after successful deploy
2026		s.mu.RLock()
2027		ids := s.attachments[domainName]
2028		if len(ids) > 0 {
2029			latest := ids[len(ids)-1]
2030			deployed = append(deployed, latest)
2031		}
2032		s.mu.RUnlock()
2033	}
2034	return nil
2035}
2036
2037// resolveAttachmentsForDomain returns attachments for a domain by checking:
2038// 1. Direct domain attachments (attachments[domain])
2039// 2. Network group attachments (attachments[group] where domain is in network_groups[group])
2040func (s *Service) resolveAttachmentsForDomain(domainName string) []string {
2041	var attachments map[string][]string
2042	var networkGroups map[string][]string
2043
2044	if s.configProvider != nil {
2045		snap := s.configProvider.GetAttachmentConfig()
2046		attachments = snap.Attachments
2047		networkGroups = snap.NetworkGroups
2048	} else {
2049		s.mu.RLock()
2050		attachments = s.config.Attachments
2051		networkGroups = s.config.NetworkGroups
2052		s.mu.RUnlock()
2053	}
2054
2055	seen := make(map[string]bool)
2056	var result []string
2057
2058	// First, add domain-specific attachments
2059	if domainAttachments, ok := attachments[domainName]; ok {
2060		for _, img := range domainAttachments {
2061			if !seen[img] {
2062				seen[img] = true
2063				result = append(result, img)
2064			}
2065		}
2066	}
2067
2068	// Then, find which network group this domain belongs to and add group attachments
2069	for groupName, domains := range networkGroups {
2070		if slices.Contains(domains, domainName) {
2071			if groupAttachments, ok := attachments[groupName]; ok {
2072				for _, img := range groupAttachments {
2073					if !seen[img] {
2074						seen[img] = true
2075						result = append(result, img)
2076					}
2077				}
2078			}
2079			break // Domain can only be in one network group
2080		}
2081	}
2082
2083	return result
2084}
2085
2086func (s *Service) getAttachmentsForDomain(ctx context.Context, domainName string) []domain.Attachment {
2087	containers, err := s.runtime.ListContainers(ctx, true)
2088	if err != nil {
2089		return nil
2090	}
2091
2092	return s.filterAttachments(ctx, containers, domainName)
2093}
2094
2095// getAllAttachments fetches all containers once and returns attachments grouped by domain.
2096// This avoids N+1 queries when listing multiple routes.
2097func (s *Service) getAllAttachments(ctx context.Context) map[string][]domain.Attachment {
2098	containers, err := s.runtime.ListContainers(ctx, true)
2099	if err != nil {
2100		return nil
2101	}
2102
2103	result := make(map[string][]domain.Attachment)
2104	for _, container := range containers {
2105		if container.Labels == nil {
2106			continue
2107		}
2108		if container.Labels[domain.LabelAttachment] != "true" {
2109			continue
2110		}
2111		ownerDomain := container.Labels[domain.LabelAttachedTo]
2112		if ownerDomain == "" {
2113			continue
2114		}
2115		image := container.Image
2116		serviceName := container.Name
2117		if labelImage, ok := container.Labels[domain.LabelImage]; ok && labelImage != "" {
2118			image = labelImage
2119			serviceName = extractServiceName(labelImage)
2120		}
2121		// Get the container's network for display in the routes table
2122		network := ""
2123		if networkName, err := s.runtime.GetContainerNetwork(ctx, container.ID); err == nil {
2124			network = networkName
2125		}
2126		attachment := domain.Attachment{
2127			Name:        serviceName,
2128			Image:       s.stripRegistryPrefix(image),
2129			ContainerID: container.ID,
2130			Status:      container.Status,
2131			Network:     network,
2132			Ports:       append([]int(nil), container.Ports...),
2133		}
2134		result[ownerDomain] = append(result[ownerDomain], attachment)
2135	}
2136
2137	return result
2138}
2139
2140// filterAttachments extracts attachments for a specific domain from a container list.
2141func (s *Service) filterAttachments(ctx context.Context, containers []*domain.Container, domainName string) []domain.Attachment {
2142	attachments := make([]domain.Attachment, 0)
2143	for _, container := range containers {
2144		if container.Labels == nil {
2145			continue
2146		}
2147		if container.Labels[domain.LabelAttachment] != "true" {
2148			continue
2149		}
2150		if container.Labels[domain.LabelAttachedTo] != domainName {
2151			continue
2152		}
2153		image := container.Image
2154		serviceName := container.Name
2155		if labelImage, ok := container.Labels[domain.LabelImage]; ok && labelImage != "" {
2156			image = labelImage
2157			serviceName = extractServiceName(labelImage)
2158		}
2159		// Get the container's network for display in the routes table
2160		network := ""
2161		if networkName, err := s.runtime.GetContainerNetwork(ctx, container.ID); err == nil {
2162			network = networkName
2163		}
2164		attachment := domain.Attachment{
2165			Name:        serviceName,
2166			Image:       s.stripRegistryPrefix(image),
2167			ContainerID: container.ID,
2168			Status:      container.Status,
2169			Network:     network,
2170			Ports:       append([]int(nil), container.Ports...),
2171		}
2172		attachments = append(attachments, attachment)
2173	}
2174
2175	return attachments
2176}
2177
2178func (s *Service) deployAttachedService(ctx context.Context, ownerDomain, serviceImage, networkName string) error {
2179	log := zerowrap.FromCtx(ctx)
2180
2181	// Parse service name from image (e.g., "my-postgres:latest" → "postgres")
2182	serviceName := extractServiceName(serviceImage)
2183	containerName := fmt.Sprintf("gordon-%s-%s", sanitizeName(ownerDomain), serviceName)
2184
2185	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
2186		"attachment":     serviceName,
2187		"container_name": containerName,
2188		"owner_domain":   ownerDomain,
2189	})
2190
2191	// Check if already running (idempotent)
2192	// Try new name first, then fall back to legacy name for backwards compatibility
2193	existingContainer, err := s.resolveExistingAttachment(ctx, ownerDomain, serviceName)
2194	if err != nil {
2195		return err
2196	}
2197
2198	if existingContainer != nil {
2199		shouldSkip, err := s.handleRunningAttachment(ctx, existingContainer, containerName, serviceImage)
2200		if err != nil {
2201			return err
2202		}
2203		if shouldSkip {
2204			return nil
2205		}
2206		if existingContainer.Status == string(domain.ContainerStatusRunning) {
2207			existingContainer = nil
2208		}
2209	}
2210
2211	if err := s.removeStoppedAttachment(ctx, existingContainer, containerName); err != nil {
2212		return err
2213	}
2214
2215	log.Info().Str(zerowrap.FieldService, serviceImage).Msg("deploying attached service")
2216
2217	// Ensure image (canonical ref is returned; internal pulls may use the local registry).
2218	imageRef := s.buildImageRef(serviceImage)
2219	actualImageRef, err := s.ensureImage(ctx, imageRef)
2220	if err != nil {
2221		return err
2222	}
2223
2224	// Get image metadata
2225	exposedPorts, err := s.runtime.GetImageExposedPorts(ctx, actualImageRef)
2226	if err != nil {
2227		log.WrapErr(err, "failed to get exposed ports for attachment, using defaults")
2228		exposedPorts = []int{}
2229	}
2230
2231	// Setup volumes (attachments need persistent data)
2232	volumes, err := s.setupVolumes(ctx, containerName, actualImageRef)
2233	if err != nil {
2234		log.WrapErr(err, "failed to setup volumes for attachment")
2235		volumes = make(map[string]string)
2236	}
2237
2238	// Load environment (attachment-specific env file)
2239	envVars, err := s.loadEnvironment(ctx, nil, containerName, actualImageRef)
2240	if err != nil {
2241		log.WrapErr(err, "failed to load environment for attachment")
2242		envVars = []string{}
2243	}
2244	envHash := hashEnvironment(envVars)
2245
2246	s.mu.RLock()
2247	cfg := s.config
2248	s.mu.RUnlock()
2249
2250	// Create container with attachment labels (use actual ref, track original in labels)
2251	config := &domain.ContainerConfig{
2252		Image:       actualImageRef,
2253		Name:        containerName,
2254		Hostname:    serviceName, // Internal DNS: postgres, redis, etc.
2255		Aliases:     []string{serviceName},
2256		Ports:       exposedPorts,
2257		Env:         envVars,
2258		Volumes:     volumes,
2259		NetworkMode: networkName, // Same network as main app
2260		Labels: map[string]string{
2261			domain.LabelManaged:    "true",
2262			domain.LabelAttachment: "true",
2263			domain.LabelAttachedTo: ownerDomain,
2264			domain.LabelEnvHash:    envHash,
2265			domain.LabelImage:      serviceImage,
2266		},
2267		MemoryLimit: cfg.DefaultMemoryLimit,
2268		NanoCPUs:    cfg.DefaultNanoCPUs,
2269		PidsLimit:   cfg.DefaultPidsLimit,
2270	}
2271
2272	container, err := s.runtime.CreateContainer(ctx, config)
2273	if err != nil {
2274		return log.WrapErr(err, "failed to create attachment container")
2275	}
2276
2277	// Start container
2278	if err := s.runtime.StartContainer(ctx, container.ID); err != nil {
2279		s.runtime.RemoveContainer(ctx, container.ID, true)
2280		return log.WrapErr(err, "failed to start attachment container")
2281	}
2282
2283	// Wait for attachment to be ready before proceeding
2284	if err := s.waitForAttachmentReady(ctx, container.ID, config); err != nil {
2285		log.WrapErr(err, "attachment readiness check failed, cleaning up")
2286		if stopErr := s.runtime.StopContainer(ctx, container.ID); stopErr != nil {
2287			log.Warn().Err(stopErr).Str(zerowrap.FieldEntityID, container.ID).Msg("failed to stop unready attachment")
2288		}
2289		s.runtime.RemoveContainer(ctx, container.ID, true)
2290		return fmt.Errorf("attachment %q not ready: %w", serviceName, err)
2291	}
2292
2293	// Track attachment
2294	s.mu.Lock()
2295	s.attachments[ownerDomain] = append(s.attachments[ownerDomain], container.ID)
2296	s.mu.Unlock()
2297
2298	// Start log collection for attachment
2299	s.startLogCollection(ctx, container.ID, containerName)
2300
2301	log.Info().Str(zerowrap.FieldEntityID, container.ID).Msg("attachment deployed successfully")
2302	return nil
2303}
2304
2305func (s *Service) attachmentEnvDrifted(ctx context.Context, existing *domain.Container, containerName, serviceImage string) (bool, error) {
2306	currentEnv, err := s.loadEnvironment(ctx, nil, containerName, s.buildImageRef(serviceImage))
2307	if err != nil {
2308		return false, err
2309	}
2310
2311	currentHash := hashEnvironment(currentEnv)
2312	existingHash, ok := existing.Labels[domain.LabelEnvHash]
2313	if ok && existingHash == currentHash {
2314		return false, nil
2315	}
2316
2317	return true, nil
2318}
2319
2320func (s *Service) handleRunningAttachment(ctx context.Context, existing *domain.Container, containerName, serviceImage string) (bool, error) {
2321	if existing.Status != string(domain.ContainerStatusRunning) {
2322		return false, nil
2323	}
2324
2325	log := zerowrap.FromCtx(ctx)
2326
2327	// Check image drift
2328	existingImage := existing.Labels[domain.LabelImage]
2329	imageDrifted := existingImage != "" && existingImage != serviceImage
2330
2331	// Check env drift
2332	envDrifted, err := s.attachmentEnvDrifted(ctx, existing, containerName, serviceImage)
2333	if err != nil {
2334		return false, log.WrapErr(err, "failed to check attachment env drift")
2335	}
2336
2337	if !envDrifted && !imageDrifted {
2338		log.Debug().Str("container_name", containerName).Msg("attachment already running with current env and image, skipping")
2339		return true, nil
2340	}
2341
2342	if imageDrifted {
2343		log.Info().Str("container_name", containerName).Str("existing_image", existingImage).Str("desired_image", serviceImage).Msg("attachment image changed, recreating")
2344	}
2345	if envDrifted {
2346		log.Info().Str("container_name", containerName).Msg("attachment env changed, recreating")
2347	}
2348
2349	if err := s.runtime.StopContainer(ctx, existing.ID); err != nil {
2350		return false, log.WrapErr(err, "failed to stop attachment for drift update")
2351	}
2352	if err := s.runtime.RemoveContainer(ctx, existing.ID, true); err != nil {
2353		return false, log.WrapErr(err, "failed to remove attachment for drift update")
2354	}
2355
2356	return false, nil
2357}
2358
2359func (s *Service) removeStoppedAttachment(ctx context.Context, existing *domain.Container, containerName string) error {
2360	if existing == nil {
2361		return nil
2362	}
2363
2364	log := zerowrap.FromCtx(ctx)
2365	log.Info().Str("container_name", containerName).Msg("removing stopped attachment container")
2366	if err := s.runtime.RemoveContainer(ctx, existing.ID, true); err != nil {
2367		return log.WrapErr(err, "failed to remove existing attachment container")
2368	}
2369
2370	return nil
2371}
2372
2373func (s *Service) resolveExistingAttachment(ctx context.Context, ownerDomain, serviceName string) (*domain.Container, error) {
2374	log := zerowrap.FromCtx(ctx)
2375	containerName := fmt.Sprintf("gordon-%s-%s", sanitizeName(ownerDomain), serviceName)
2376	existingContainer := s.findContainerByName(ctx, containerName)
2377	if existingContainer != nil {
2378		return existingContainer, nil
2379	}
2380
2381	containerNameLegacy := fmt.Sprintf("gordon-%s-%s", sanitizeNameLegacy(ownerDomain), serviceName)
2382	existingContainer = s.findContainerByName(ctx, containerNameLegacy)
2383	if existingContainer == nil {
2384		return nil, nil
2385	}
2386
2387	log.Info().Str("container_name", containerNameLegacy).Msg("found attachment with legacy naming, will be replaced")
2388	if err := s.runtime.StopContainer(ctx, existingContainer.ID); err != nil {
2389		return nil, log.WrapErr(err, "failed to stop legacy attachment container")
2390	}
2391	if err := s.runtime.RemoveContainer(ctx, existingContainer.ID, true); err != nil {
2392		return nil, log.WrapErr(err, "failed to remove legacy attachment container")
2393	}
2394
2395	return nil, nil
2396}
2397
2398// Utility functions
2399
2400func normalizeImageRef(image string) string {
2401	// Extract repository from image reference, handling port numbers correctly.
2402	// Examples:
2403	//   nginx:latest -> docker.io/library/nginx
2404	//   user/repo:tag -> docker.io/user/repo
2405	//   localhost:5000/image:tag -> localhost:5000/image
2406	//   registry.example.com/image:tag -> registry.example.com/image
2407	//
2408	// Note: This function assumes valid Docker image references. Edge cases like
2409	// "registry.com:5000" (registry with port but no image name) are not valid
2410	// image references per Docker's naming conventions, which require at least
2411	// one path component after the registry (e.g., "registry.com:5000/image").
2412
2413	// Find the tag separator: the last colon that isn't part of a port number.
2414	// A colon is part of a port if there's no slash after it until the next colon.
2415	repo := image
2416	lastColon := strings.LastIndex(image, ":")
2417	if lastColon != -1 {
2418		afterColon := image[lastColon+1:]
2419		// If there's no slash after the colon, it's the tag separator
2420		if !strings.Contains(afterColon, "/") {
2421			repo = image[:lastColon]
2422		}
2423	}
2424
2425	if !strings.Contains(repo, "/") {
2426		return "docker.io/library/" + repo
2427	}
2428
2429	if strings.Count(repo, "/") == 1 && !strings.Contains(strings.Split(repo, "/")[0], ".") && !strings.Contains(strings.Split(repo, "/")[0], ":") {
2430		return "docker.io/" + repo
2431	}
2432
2433	return repo
2434}
2435
2436func generateVolumeName(prefix, domainName, volumePath string) string {
2437	return fmt.Sprintf("%s-%s-%s",
2438		prefix,
2439		strings.ReplaceAll(domainName, ".", "-"),
2440		strings.ReplaceAll(strings.Trim(volumePath, "/"), "/", "-"))
2441}
2442
2443func mergeEnvironmentVariables(dockerfileEnv, userEnv []string) []string {
2444	envMap := make(map[string]string)
2445
2446	for _, env := range dockerfileEnv {
2447		if k, v, ok := strings.Cut(env, "="); ok {
2448			envMap[k] = v
2449		}
2450	}
2451
2452	for _, env := range userEnv {
2453		if k, v, ok := strings.Cut(env, "="); ok {
2454			envMap[k] = v
2455		}
2456	}
2457
2458	keys := make([]string, 0, len(envMap))
2459	for k := range envMap {
2460		keys = append(keys, k)
2461	}
2462	slices.Sort(keys)
2463
2464	result := make([]string, 0, len(envMap))
2465	for _, k := range keys {
2466		result = append(result, k+"="+envMap[k])
2467	}
2468
2469	return result
2470}
2471
2472// extractServiceName gets service name from image reference.
2473// "my-postgres:latest" → "postgres", "redis:7" → "redis"
2474func extractServiceName(image string) string {
2475	// Remove tag
2476	parts := strings.Split(image, ":")
2477	name := parts[0]
2478
2479	// Remove registry prefix if present
2480	if strings.Contains(name, "/") {
2481		nameParts := strings.Split(name, "/")
2482		name = nameParts[len(nameParts)-1]
2483	}
2484
2485	// Remove common prefixes like "my-"
2486	name = strings.TrimPrefix(name, "my-")
2487
2488	return name
2489}
2490
2491// sanitizeName is a convenience wrapper around domain.SanitizeDomainForContainer.
2492func sanitizeName(d string) string {
2493	return domain.SanitizeDomainForContainer(d)
2494}
2495
2496// sanitizeNameLegacy is a convenience wrapper around domain.SanitizeDomainForContainerLegacy.
2497func sanitizeNameLegacy(d string) string {
2498	return domain.SanitizeDomainForContainerLegacy(d)
2499}
2500
2501func isContainerNotFoundError(err error) bool {
2502	if err == nil {
2503		return false
2504	}
2505	if errors.Is(err, domain.ErrContainerNotFound) {
2506		return true
2507	}
2508	msg := strings.ToLower(err.Error())
2509	return strings.Contains(msg, "no such container") ||
2510		strings.Contains(msg, "no container with name or id") ||
2511		strings.Contains(msg, "container not found")
2512}
2513
2514func isConnectionRefusedError(err error) bool {
2515	if err == nil {
2516		return false
2517	}
2518	return strings.Contains(strings.ToLower(err.Error()), "connection refused")
2519}
2520
2521// findContainerByName finds a container by its name.
2522func (s *Service) findContainerByName(ctx context.Context, name string) *domain.Container {
2523	containers, err := s.runtime.ListContainers(ctx, true)
2524	if err != nil {
2525		return nil
2526	}
2527
2528	for _, c := range containers {
2529		if c.Name == name {
2530			return c
2531		}
2532	}
2533	return nil
2534}
2535
2536func (s *Service) waitForReady(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig) error {
2537	if err := s.pollContainerRunning(ctx, containerID); err != nil {
2538		return err
2539	}
2540
2541	s.mu.RLock()
2542	cfg := s.config
2543	s.mu.RUnlock()
2544
2545	readinessMode := cfg.ReadinessMode
2546	if readinessMode == "" {
2547		readinessMode = "delay"
2548	}
2549
2550	switch readinessMode {
2551	case "delay":
2552		return s.waitForReadyByDelay(ctx, containerID)
2553	case "docker-health":
2554		_, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(ctx, containerID)
2555		if err != nil {
2556			return err
2557		}
2558		if !hasHealthcheck {
2559			return errors.New("no healthcheck detected")
2560		}
2561		return s.waitForHealthy(ctx, containerID, cfg.HealthTimeout)
2562	default: // auto
2563		return s.readinessCascade(ctx, containerID, containerConfig, cfg)
2564	}
2565}
2566
2567// readinessCascade auto-detects the strongest available readiness signal:
2568//  1. Docker healthcheck (if present) → wait for healthy status
2569//  2. HTTP probe (if gordon.health label set) → GET until 2xx/3xx
2570//  3. Default HTTP probe (GET / on exposed port) → wait for 2xx/3xx
2571//  4. TCP probe (if port info available) → connect until accepted
2572//  5. Delay fallback (last resort) → waitForReadyByDelay
2573func (s *Service) readinessCascade(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) error {
2574	log := zerowrap.FromCtx(ctx)
2575
2576	// 1. Docker healthcheck
2577	_, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(ctx, containerID)
2578	if err != nil {
2579		return err
2580	}
2581	if hasHealthcheck {
2582		log.Info().Msg("readiness cascade: using Docker healthcheck")
2583		return s.waitForHealthy(ctx, containerID, cfg.HealthTimeout)
2584	}
2585
2586	// 2. HTTP probe via gordon.health label
2587	if probed, probeErr := s.tryHTTPProbe(ctx, containerID, containerConfig, cfg); probed {
2588		return probeErr
2589	}
2590
2591	// 3. Default HTTP probe on root path (if port info available)
2592	if probed, probeErr := s.tryDefaultHTTPProbe(ctx, containerID, containerConfig, cfg); probed {
2593		return probeErr
2594	}
2595
2596	// 4. TCP probe (if port info available)
2597	if probed, probeErr := s.tryTCPProbe(ctx, containerID, containerConfig, cfg); probed {
2598		return probeErr
2599	}
2600
2601	// 5. Delay fallback
2602	log.Info().Msg("readiness cascade: using delay fallback")
2603	return s.waitForReadyByDelay(ctx, containerID)
2604}
2605
2606// waitForAttachmentReady runs a reduced readiness cascade for attachments:
2607// Docker healthcheck → TCP probe → delay fallback.
2608// HTTP probes are skipped — attachments are typically databases/caches.
2609func (s *Service) waitForAttachmentReady(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig) error {
2610	if err := s.pollContainerRunning(ctx, containerID); err != nil {
2611		return err
2612	}
2613
2614	log := zerowrap.FromCtx(ctx)
2615
2616	s.mu.RLock()
2617	cfg := s.config
2618	s.mu.RUnlock()
2619
2620	// 1. Docker healthcheck (if present)
2621	_, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(ctx, containerID)
2622	if err != nil {
2623		return err
2624	}
2625	if hasHealthcheck {
2626		log.Info().Msg("attachment readiness: using Docker healthcheck")
2627		timeout := cfg.HealthTimeout
2628		if timeout == 0 {
2629			timeout = 90 * time.Second
2630		}
2631		return s.waitForHealthy(ctx, containerID, timeout)
2632	}
2633
2634	// 2. TCP probe (if port info available)
2635	if containerConfig != nil && len(containerConfig.Ports) > 0 {
2636		ip, port, probeErr := s.resolveProbeEndpoint(ctx, containerID, containerConfig)
2637		if probeErr == nil && ip != "" && port > 0 {
2638			addr := fmt.Sprintf("%s:%d", ip, port)
2639			timeout := cfg.AttachmentReadinessTimeout
2640			if timeout == 0 {
2641				timeout = 30 * time.Second
2642			}
2643			log.Info().Str("addr", addr).Dur("timeout", timeout).Msg("attachment readiness: using TCP probe")
2644			return tcpProbe(ctx, addr, timeout)
2645		}
2646		log.Debug().Err(probeErr).Msg("attachment readiness: TCP probe skipped, could not resolve endpoint")
2647	}
2648
2649	// 3. Delay fallback
2650	log.Info().Msg("attachment readiness: using delay fallback")
2651	return s.waitForReadyByDelay(ctx, containerID)
2652}
2653
2654// tryHTTPProbe attempts an HTTP probe if the gordon.health label is set.
2655// Returns (true, err) if the probe was attempted, (false, nil) if skipped.
2656func (s *Service) tryHTTPProbe(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) (bool, error) {
2657	log := zerowrap.FromCtx(ctx)
2658	if containerConfig == nil {
2659		return false, nil
2660	}
2661	healthPath, ok := containerConfig.Labels[domain.LabelHealth]
2662	if !ok || healthPath == "" {
2663		return false, nil
2664	}
2665	ip, port, probeErr := s.resolveProbeEndpoint(ctx, containerID, containerConfig)
2666	if probeErr != nil || ip == "" || port <= 0 {
2667		log.Debug().Err(probeErr).Msg("readiness cascade: HTTP probe skipped, could not resolve container endpoint")
2668		return false, nil
2669	}
2670	url := fmt.Sprintf("http://%s:%d%s", ip, port, healthPath)
2671	timeout := cfg.HTTPProbeTimeout
2672	if timeout == 0 {
2673		timeout = 60 * time.Second
2674	}
2675	log.Info().Str("url", url).Dur("timeout", timeout).Msg("readiness cascade: using HTTP probe")
2676	return true, httpProbe(ctx, url, timeout)
2677}
2678
2679// tryDefaultHTTPProbe attempts an HTTP GET on "/" using the container's
2680// exposed port. This is the default readiness check for web containers —
2681// it verifies the app is actually serving HTTP, not just accepting TCP.
2682// Returns (true, err) if the probe was attempted, (false, nil) if skipped.
2683func (s *Service) tryDefaultHTTPProbe(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) (bool, error) {
2684	log := zerowrap.FromCtx(ctx)
2685	if containerConfig == nil || len(containerConfig.Ports) == 0 {
2686		return false, nil
2687	}
2688	ip, port, probeErr := s.resolveProbeEndpoint(ctx, containerID, containerConfig)
2689	if probeErr != nil || ip == "" || port <= 0 {
2690		log.Debug().Err(probeErr).Msg("readiness cascade: default HTTP probe skipped, could not resolve container endpoint")
2691		return false, nil
2692	}
2693	url := fmt.Sprintf("http://%s:%d/", ip, port)
2694	timeout := cfg.HTTPProbeTimeout
2695	if timeout == 0 {
2696		timeout = 60 * time.Second
2697	}
2698	log.Info().Str("url", url).Dur("timeout", timeout).Msg("readiness cascade: using default HTTP alive probe")
2699	return true, httpAliveProbe(ctx, url, timeout)
2700}
2701
2702// tryTCPProbe attempts a TCP probe if port info is available.
2703// Returns (true, err) if the probe was attempted, (false, nil) if skipped.
2704func (s *Service) tryTCPProbe(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) (bool, error) {
2705	log := zerowrap.FromCtx(ctx)
2706	if containerConfig == nil || len(containerConfig.Ports) == 0 {
2707		return false, nil
2708	}
2709	ip, port, probeErr := s.resolveProbeEndpoint(ctx, containerID, containerConfig)
2710	if probeErr != nil || ip == "" || port <= 0 {
2711		log.Debug().Err(probeErr).Msg("readiness cascade: TCP probe skipped, could not resolve container endpoint")
2712		return false, nil
2713	}
2714	addr := fmt.Sprintf("%s:%d", ip, port)
2715	timeout := cfg.TCPProbeTimeout
2716	if timeout == 0 {
2717		timeout = 30 * time.Second
2718	}
2719	log.Info().Str("addr", addr).Dur("timeout", timeout).Msg("readiness cascade: using TCP probe")
2720	return true, tcpProbe(ctx, addr, timeout)
2721}
2722
2723// resolveContainerEndpoint returns a host-reachable address for probing.
2724// In rootless podman/Docker setups, container internal IPs are not routable
2725// from the host. We use the host port binding (127.0.0.1:<mapped_port>)
2726// which is always reachable. Falls back to the container's internal IP
2727// only if no host port mapping exists (e.g. host-network mode).
2728func (s *Service) resolveProbeEndpoint(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig) (string, int, error) {
2729	log := zerowrap.FromCtx(ctx)
2730
2731	internalPort := 0
2732	if containerConfig != nil {
2733		if labels := containerConfig.Labels; labels != nil {
2734			for _, key := range []string{domain.LabelProxyPort, domain.LabelPort} {
2735				if portStr, ok := labels[key]; ok && portStr != "" {
2736					port, err := strconv.Atoi(portStr)
2737					if err == nil && port > 0 && port <= 65535 {
2738						internalPort = port
2739						break
2740					}
2741					log.Warn().Str("label", key).Str("port_value", portStr).Msg("invalid probe port label value")
2742				}
2743			}
2744		}
2745	}
2746
2747	if internalPort == 0 {
2748		_, resolvedPort, err := s.runtime.GetContainerNetworkInfo(ctx, containerID)
2749		if err != nil {
2750			return "", 0, err
2751		}
2752		internalPort = resolvedPort
2753	}
2754
2755	hostPort, hostErr := s.runtime.GetContainerPort(ctx, containerID, internalPort)
2756	if hostErr == nil && hostPort > 0 {
2757		log.Debug().
2758			Int("internal_port", internalPort).
2759			Int("host_port", hostPort).
2760			Msg("resolved container endpoint via host port binding")
2761		return "127.0.0.1", hostPort, nil
2762	}
2763
2764	ip, _, fallbackErr := s.runtime.GetContainerNetworkInfo(ctx, containerID)
2765	if fallbackErr != nil {
2766		return "", 0, fallbackErr
2767	}
2768	log.Debug().
2769		Str("ip", ip).
2770		Int("port", internalPort).
2771		Msg("resolved container endpoint via internal IP (no host port binding)")
2772	return ip, internalPort, nil
2773}
2774
2775// waitForReadyByDelay waits using the legacy running+delay strategy.
2776func (s *Service) waitForReadyByDelay(ctx context.Context, containerID string) error {
2777	log := zerowrap.FromCtx(ctx)
2778
2779	s.mu.RLock()
2780	cfg := s.config
2781	s.mu.RUnlock()
2782
2783	delay := cfg.ReadinessDelay
2784	if delay == 0 {
2785		delay = 5 * time.Second
2786	}
2787
2788	log.Debug().Dur("delay", delay).Msg("waiting for container readiness")
2789	select {
2790	case <-time.After(delay):
2791	case <-ctx.Done():
2792		return ctx.Err()
2793	}
2794
2795	// Verify still running after delay
2796	running, err := s.runtime.IsContainerRunning(ctx, containerID)
2797	if err != nil {
2798		return err
2799	}
2800	if running {
2801		return nil
2802	}
2803
2804	return s.waitForRecovery(ctx, containerID)
2805}
2806
2807func (s *Service) waitForHealthy(ctx context.Context, containerID string, timeout time.Duration) error {
2808	if timeout == 0 {
2809		timeout = 90 * time.Second
2810	}
2811	waitCtx, cancel := context.WithTimeout(ctx, timeout)
2812	defer cancel()
2813
2814	var lastStatus string
2815	for {
2816		status, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(waitCtx, containerID)
2817		if err != nil {
2818			return err
2819		}
2820		if !hasHealthcheck {
2821			return errors.New("no healthcheck detected")
2822		}
2823		// Treat empty health status as transitional startup. Some runtimes can
2824		// temporarily report an empty status before first probe results.
2825		if status == "" {
2826			status = "starting"
2827		}
2828		if status == "healthy" {
2829			return nil
2830		}
2831		lastStatus = status
2832
2833		select {
2834		case <-time.After(time.Second):
2835		case <-waitCtx.Done():
2836			return fmt.Errorf("container healthcheck timeout after %s (last status: %s)", timeout, lastStatus)
2837		}
2838	}
2839}
2840
2841// pollContainerRunning polls until the container is running (max 30 seconds).
2842func (s *Service) pollContainerRunning(ctx context.Context, containerID string) error {
2843	for i := 0; i < 30; i++ {
2844		running, err := s.runtime.IsContainerRunning(ctx, containerID)
2845		if err != nil {
2846			return err
2847		}
2848		if running {
2849			return nil
2850		}
2851		if i == 29 {
2852			return fmt.Errorf("container did not start within 30 seconds")
2853		}
2854		select {
2855		case <-time.After(time.Second):
2856		case <-ctx.Done():
2857			return ctx.Err()
2858		}
2859	}
2860	return fmt.Errorf("container did not start within 30 seconds")
2861}
2862
2863// waitForRecovery waits for a container to recover within the recovery window.
2864func (s *Service) waitForRecovery(ctx context.Context, containerID string) error {
2865	log := zerowrap.FromCtx(ctx)
2866	log.Warn().
2867		Dur("recovery_window", readinessRecoveryWindow).
2868		Msg("container not running after readiness delay, waiting for recovery")
2869
2870	deadline := time.Now().Add(readinessRecoveryWindow)
2871	for time.Now().Before(deadline) {
2872		select {
2873		case <-time.After(time.Second):
2874		case <-ctx.Done():
2875			return ctx.Err()
2876		}
2877
2878		running, err := s.runtime.IsContainerRunning(ctx, containerID)
2879		if err != nil {
2880			return err
2881		}
2882		if running {
2883			log.Info().Msg("container recovered during readiness recovery window")
2884			return nil
2885		}
2886	}
2887
2888	return fmt.Errorf("container not running after readiness delay and recovery window")
2889}
2890
2891// publishContainerDeployed publishes a container.deployed event.
2892func (s *Service) publishContainerDeployed(ctx context.Context, domainName, containerID string) {
2893	payload := &domain.ContainerEventPayload{
2894		ContainerID: containerID,
2895		Domain:      domainName,
2896		Action:      "deployed",
2897	}
2898
2899	if err := s.eventBus.Publish(domain.EventContainerDeployed, payload); err != nil {
2900		log := zerowrap.FromCtx(ctx)
2901		log.Warn().Err(err).Msg("failed to publish container deployed event")
2902	}
2903}
2904
2905// rewriteToRegistryDomain rewrites an image reference to use the configured registry domain.
2906// e.g., "myapp:latest" -> "registry.example.com/myapp:latest"
2907func rewriteToRegistryDomain(imageRef, registryDomain string) string {
2908	if registryDomain == "" {
2909		return imageRef
2910	}
2911
2912	prefix := registryDomain + "/"
2913	if strings.HasPrefix(imageRef, prefix) {
2914		return imageRef
2915	}
2916
2917	return prefix + imageRef
2918}
2919
2920// rewriteToLocalRegistry rewrites an image reference to use the local registry address.
2921// e.g., "registry.example.com/myapp:latest" -> "localhost:5000/myapp:latest"
2922func rewriteToLocalRegistry(imageRef, registryDomain string, registryPort int) string {
2923	if imageRef == "" {
2924		return imageRef
2925	}
2926
2927	localRegistry := fmt.Sprintf("localhost:%d", registryPort)
2928	localPrefix := localRegistry + "/"
2929	imageRef = strings.TrimPrefix(imageRef, localPrefix)
2930
2931	registryDomain = strings.TrimSuffix(registryDomain, "/")
2932	if registryDomain != "" {
2933		prefix := registryDomain + "/"
2934		imageRef = strings.TrimPrefix(imageRef, prefix)
2935	}
2936
2937	return localPrefix + imageRef
2938}