service.go RawHistoryFolder

1// Package container implements the container management use case.
2package container
3
4import (
5	"context"
6	"errors"
7	"fmt"
8	"maps"
9	"slices"
10	"strconv"
11	"strings"
12	"sync"
13	"time"
14
15	"github.com/bnema/zerowrap"
16	"go.opentelemetry.io/otel"
17	"go.opentelemetry.io/otel/attribute"
18	"go.opentelemetry.io/otel/codes"
19	"go.opentelemetry.io/otel/metric"
20	"go.opentelemetry.io/otel/trace"
21
22	"github.com/bnema/gordon/internal/adapters/out/telemetry"
23	"github.com/bnema/gordon/internal/boundaries/out"
24	"github.com/bnema/gordon/internal/domain"
25)
26
27// Config holds configuration needed by the container service.
28type Config struct {
29	RegistryAuthEnabled      bool
30	RegistryDomain           string
31	RegistryPort             int
32	ServiceTokenUsername     string
33	ServiceToken             string
34	InternalRegistryUsername string
35	InternalRegistryPassword string
36	PullPolicy               string
37	VolumeAutoCreate         bool
38	VolumePrefix             string
39	VolumePreserve           bool
40	NetworkIsolation         bool
41	NetworkPrefix            string
42	NetworkGroups            map[string][]string
43	Attachments              map[string][]string
44	ReadinessDelay           time.Duration // Delay after container starts before considering it ready
45	ReadinessMode            string        // Readiness strategy: auto, docker-health, delay
46	HealthTimeout            time.Duration // Max wait for health-based readiness
47	DrainDelay               time.Duration // Grace period after cache invalidation before stopping old container
48	DrainDelayConfigured     bool          // True when deploy.drain_delay was explicitly configured
49	DrainMode                string        // Drain strategy: auto, inflight, delay
50	DrainTimeout             time.Duration // Max wait for in-flight requests to drain
51	StabilizationDelay       time.Duration // Post-switch monitoring window (default 2s)
52	TCPProbeTimeout          time.Duration // TCP probe timeout (default 30s)
53	HTTPProbeTimeout         time.Duration // HTTP probe timeout (default 60s)
54	DefaultMemoryLimit       int64         // Default memory limit in bytes for containers (0 = no limit)
55	DefaultNanoCPUs          int64         // Default CPU quota in nanoseconds for containers (0 = no limit)
56	DefaultPidsLimit         int64         // Default max PIDs for containers (0 = no limit)
57}
58
59var tracer = otel.Tracer("gordon.container")
60
61const (
62	PullPolicyAlways       = "always"
63	PullPolicyIfNotPresent = "if-not-present"
64	PullPolicyIfTagChanged = "if-tag-changed"
65
66	// readinessRecoveryWindow is an additional grace window used when a container
67	// is briefly not running at the end of readiness delay. This avoids false
68	// negatives during short startup flaps.
69	readinessRecoveryWindow = 30 * time.Second
70	internalPullMaxAttempts = 3
71)
72
73// Service implements the ContainerService interface.
74type Service struct {
75	runtime          out.ContainerRuntime
76	envLoader        out.EnvLoader
77	eventBus         out.EventPublisher
78	logWriter        out.ContainerLogWriter
79	cacheInvalidator out.ProxyCacheInvalidator
80	drainWaiter      out.ProxyDrainWaiter
81	config           Config
82	configProvider   AttachmentConfigProvider // live config reads for attachments/networks (may be nil)
83	metrics          *telemetry.Metrics
84	containers       map[string]*domain.Container
85	attachments      map[string][]string // ownerDomain → []containerIDs
86	managedCount     int64               // tracks UpDownCounter value for delta computation
87	mu               sync.RWMutex
88	deployMu         sync.Map       // per-domain deploy locks (domain → *domainDeployLock)
89	cleanupWg        sync.WaitGroup // tracks background old-container cleanup goroutines
90	monitor          *Monitor
91}
92
93// domainDeployLock is a context-aware mutex using a buffered channel.
94// It allows Lock() to be interrupted by context cancellation.
95type domainDeployLock struct {
96	ch chan struct{}
97}
98
99// newDomainDeployLock creates a new lock that is initially unlocked.
100func newDomainDeployLock() *domainDeployLock {
101	l := &domainDeployLock{ch: make(chan struct{}, 1)}
102	l.ch <- struct{}{}
103	return l
104}
105
106// Lock acquires the lock or returns an error if the context is cancelled.
107func (l *domainDeployLock) Lock(ctx context.Context) error {
108	// Fast path: honor already-canceled contexts before blocking.
109	if err := ctx.Err(); err != nil {
110		return err
111	}
112	select {
113	case <-ctx.Done():
114		return ctx.Err()
115	case <-l.ch:
116		// Re-check context after acquisition: if canceled while we were
117		// in the select, release the token so the lock isn't consumed.
118		if err := ctx.Err(); err != nil {
119			l.ch <- struct{}{}
120			return err
121		}
122		return nil
123	}
124}
125
126// Unlock releases the lock.
127func (l *domainDeployLock) Unlock() {
128	select {
129	case l.ch <- struct{}{}:
130	default:
131		// This should never happen if Lock/Unlock are used correctly.
132		// Panic to detect double-unlocks or other misuse early.
133		panic("domainDeployLock: double unlock detected")
134	}
135}
136
137// NewService creates a new container service.
138func NewService(
139	runtime out.ContainerRuntime,
140	envLoader out.EnvLoader,
141	eventBus out.EventPublisher,
142	logWriter out.ContainerLogWriter,
143	config Config,
144	configProvider AttachmentConfigProvider,
145) *Service {
146	return &Service{
147		runtime:        runtime,
148		envLoader:      envLoader,
149		eventBus:       eventBus,
150		logWriter:      logWriter,
151		config:         config,
152		configProvider: configProvider,
153		containers:     make(map[string]*domain.Container),
154		attachments:    make(map[string][]string),
155	}
156}
157
158// SetMetrics sets the telemetry metrics for the container service.
159func (s *Service) SetMetrics(m *telemetry.Metrics) {
160	s.metrics = m
161}
162
163// SetProxyCacheInvalidator sets the proxy cache invalidator for synchronous
164// cache invalidation during zero-downtime deployments. This must be called
165// after construction because the proxy service is created after the container
166// service during application initialization.
167func (s *Service) SetProxyCacheInvalidator(inv out.ProxyCacheInvalidator) {
168	s.mu.Lock()
169	s.cacheInvalidator = inv
170	s.mu.Unlock()
171}
172
173// SetProxyDrainWaiter sets the proxy in-flight drain waiter used during
174// zero-downtime replacement before stopping the old container.
175func (s *Service) SetProxyDrainWaiter(waiter out.ProxyDrainWaiter) {
176	s.mu.Lock()
177	s.drainWaiter = waiter
178	s.mu.Unlock()
179}
180
181// domainDeployMu returns a per-domain lock for serializing deploy operations.
182func (s *Service) domainDeployMu(domain string) *domainDeployLock {
183	v, _ := s.deployMu.LoadOrStore(domain, newDomainDeployLock())
184	// Type assertion is safe: LoadOrStore always stores *domainDeployLock values,
185	// and any concurrent LoadOrStore calls will also store *domainDeployLock.
186	return v.(*domainDeployLock)
187}
188
189// acquireDomainDeployLock acquires the per-domain deploy lock with context cancellation support.
190// Returns an unlock function that must be called to release the lock.
191func (s *Service) acquireDomainDeployLock(ctx context.Context, domain string) (func(), error) {
192	lock := s.domainDeployMu(domain)
193	if err := lock.Lock(ctx); err != nil {
194		return nil, err
195	}
196	return func() { lock.Unlock() }, nil
197}
198
199// buildContainerConfig constructs the container configuration for deployment.
200func (s *Service) deploymentContainerName(containerDomain string, existing *domain.Container) string {
201	canonicalName := fmt.Sprintf("gordon-%s", containerDomain)
202	if existing == nil {
203		return canonicalName
204	}
205
206	// Alternate temporary names if the tracked container was left with a temp suffix.
207	// This prevents name collisions after interrupted zero-downtime deploys.
208	newName := canonicalName + "-new"
209	nextName := canonicalName + "-next"
210	switch existing.Name {
211	case newName:
212		return nextName
213	case nextName:
214		return newName
215	default:
216		return newName
217	}
218}
219
220// containerConfigInput groups the parameters for building a container configuration.
221type containerConfigInput struct {
222	Domain       string
223	Image        string
224	ImageRef     string
225	ExposedPorts []int
226	EnvVars      []string
227	EnvHash      string
228	Volumes      map[string]string
229	NetworkName  string
230	ImageLabels  map[string]string
231	Existing     *domain.Container
232}
233
234func (s *Service) buildContainerConfig(in containerConfigInput) *domain.ContainerConfig {
235	s.mu.RLock()
236	cfg := s.config
237	s.mu.RUnlock()
238
239	containerName := s.deploymentContainerName(in.Domain, in.Existing)
240	labels := map[string]string{
241		domain.LabelDomain:  in.Domain,
242		domain.LabelEnvHash: in.EnvHash,
243		domain.LabelImage:   in.Image,
244		domain.LabelManaged: "true",
245		domain.LabelRoute:   in.Domain,
246	}
247
248	// Propagate proxy/port labels from image so readiness probes can find them
249	for _, key := range []string{domain.LabelProxyPort, domain.LabelPort, domain.LabelHealth} {
250		if v, ok := in.ImageLabels[key]; ok && v != "" {
251			labels[key] = v
252		}
253	}
254
255	// Ensure the label-specified proxy port is included in exposed ports
256	// so Docker creates a host port binding for it.
257	ports := in.ExposedPorts
258	if portStr, ok := labels[domain.LabelProxyPort]; ok {
259		if port, err := strconv.Atoi(portStr); err == nil && port > 0 && port <= 65535 {
260			if !slices.Contains(ports, port) {
261				ports = append(ports, port)
262			}
263		}
264	}
265
266	return &domain.ContainerConfig{
267		Image:       in.ImageRef,
268		Name:        containerName,
269		Ports:       ports,
270		Env:         in.EnvVars,
271		Volumes:     in.Volumes,
272		NetworkMode: in.NetworkName,
273		Hostname:    in.Domain,
274		Labels:      labels,
275		AutoRemove:  false,
276		MemoryLimit: cfg.DefaultMemoryLimit,
277		NanoCPUs:    cfg.DefaultNanoCPUs,
278		PidsLimit:   cfg.DefaultPidsLimit,
279	}
280}
281
282// Deploy creates and starts a container for the given route.
283// Implements zero-downtime deployment: new container starts before old one stops.
284func (s *Service) Deploy(ctx context.Context, route domain.Route) (*domain.Container, error) {
285	ctx, span := tracer.Start(ctx, "container.deploy",
286		trace.WithAttributes(
287			attribute.String("domain", route.Domain),
288			attribute.String("image", route.Image),
289		))
290	defer span.End()
291
292	deployStart := time.Now()
293
294	// Serialize deploys for the same domain to prevent race conditions
295	// (e.g. multiple image.pushed events + explicit deploy call from CLI).
296	unlock, err := s.acquireDomainDeployLock(ctx, route.Domain)
297	if err != nil {
298		return nil, err
299	}
300	defer unlock()
301
302	// Enrich context with use case fields for all downstream logs
303	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
304		zerowrap.FieldLayer:   "usecase",
305		zerowrap.FieldUseCase: "Deploy",
306		"domain":              route.Domain,
307	})
308	log := zerowrap.FromCtx(ctx)
309
310	// Record deploy metrics and trace status
311	defer s.recordDeployMetrics(ctx, span, route, deployStart, &err)
312
313	existing, hasExisting := s.resolveExistingContainer(ctx, route.Domain)
314
315	resources, err := s.prepareDeployResources(ctx, route, existing)
316	if err != nil {
317		return nil, err
318	}
319
320	// Skip redundant deploy: if the existing container is already running
321	// the exact same image (by Docker image ID), return it immediately.
322	// This prevents the double-deploy caused by the event-based deploy
323	// (triggered by image.pushed) racing with the explicit CLI deploy call.
324	if hasExisting {
325		existingForSkip := existing
326		if existingForSkip.ImageID == "" && existingForSkip.Image != "" && normalizeImageRef(existingForSkip.Image) == normalizeImageRef(route.Image) {
327			existingForSkip = s.containerForRedundantCheck(ctx, existingForSkip)
328		}
329		if existingForSkip.ImageID != "" {
330			if skip, container := s.skipRedundantDeploy(ctx, existingForSkip, resources.actualImageRef, resources.envHash); skip {
331				return container, nil
332			}
333		}
334	}
335
336	newContainer, err := s.createStartedContainer(ctx, route, existing, resources)
337	if err != nil {
338		return nil, err
339	}
340
341	invalidated := s.activateDeployedContainer(ctx, route.Domain, newContainer)
342
343	// Post-switch stabilization: verify new container stays running
344	if hasExisting {
345		stable, stabilizeErr := s.stabilizeNewContainer(ctx, route.Domain, newContainer, existing)
346		if stabilizeErr != nil {
347			// Both old and new containers are dead; assign to named return so
348			// the deferred recordDeployMetrics and span see the failure.
349			err = stabilizeErr
350			return nil, err
351		}
352		if !stable {
353			// Rollback performed — old container is restored
354			return existing, nil
355		}
356	}
357
358	// Finalize old container in the background — the new container is already
359	// serving traffic, so there's no reason to block the deploy response while
360	// waiting for the old container to stop (which can take 20s if the app
361	// doesn't handle SIGTERM).
362	s.cleanupWg.Add(1)
363	go func() {
364		defer s.cleanupWg.Done()
365		s.finalizePreviousContainer(context.WithoutCancel(ctx), route.Domain, existing, hasExisting, invalidated, newContainer.ID)
366	}()
367
368	// Start container log collection (non-blocking, errors don't fail deployment)
369	s.startLogCollection(ctx, newContainer.ID, route.Domain)
370
371	log.Info().
372		Str("image", route.Image).
373		Str(zerowrap.FieldEntityID, newContainer.ID).
374		Ints("ports", newContainer.Ports).
375		Str("network", resources.networkName).
376		Bool("zero_downtime", hasExisting).
377		Msg("container deployed successfully")
378
379	return newContainer, nil
380}
381
382// recordDeployMetrics records span error status and deploy metrics at the end of a Deploy call.
383// It is called via defer and receives a pointer to the named return error so it can observe
384// the final error value after all deferred functions have run.
385func (s *Service) recordDeployMetrics(ctx context.Context, span trace.Span, route domain.Route, start time.Time, errPtr *error) {
386	err := *errPtr
387	if err != nil {
388		span.RecordError(err)
389		span.SetStatus(codes.Error, err.Error())
390	}
391	if s.metrics == nil {
392		return
393	}
394	attrs := metric.WithAttributes(
395		attribute.String("domain", route.Domain),
396		attribute.String("image", route.Image),
397	)
398	s.metrics.DeployTotal.Add(ctx, 1, attrs)
399	s.metrics.DeployDuration.Record(ctx, time.Since(start).Seconds(), attrs)
400	if err != nil {
401		s.metrics.DeployErrors.Add(ctx, 1, attrs)
402	}
403}
404
405// skipRedundantDeploy checks whether the existing container is already running
406// the same image (by Docker image ID) as the one we are about to deploy.
407// When a push triggers both an event-based deploy and an explicit CLI deploy,
408// the second one arrives after the first has already completed; this avoids
409// a full redundant create-start-readiness cycle.
410func (s *Service) skipRedundantDeploy(ctx context.Context, existing *domain.Container, actualImageRef, envHash string) (bool, *domain.Container) {
411	log := zerowrap.FromCtx(ctx)
412
413	newImageID, err := s.runtime.GetImageID(ctx, actualImageRef)
414	if err != nil {
415		log.Debug().Err(err).Msg("cannot resolve image ID for redundancy check, proceeding with deploy")
416		return false, nil
417	}
418
419	if existing.ImageID != newImageID {
420		return false, nil
421	}
422
423	// Verify the container is actually still running before skipping.
424	running, err := s.runtime.IsContainerRunning(ctx, existing.ID)
425	if err != nil || !running {
426		return false, nil
427	}
428
429	if existing.Labels != nil {
430		existingEnvHash, hasEnvHash := existing.Labels[domain.LabelEnvHash]
431		if !hasEnvHash || existingEnvHash != envHash {
432			log.Info().
433				Str("container_id", existing.ID).
434				Bool("has_env_hash", hasEnvHash).
435				Msg("env changed, proceeding with deploy despite same image")
436			return false, nil
437		}
438	} else {
439		return false, nil
440	}
441
442	log.Info().
443		Str("container_id", existing.ID).
444		Str("image_id", newImageID).
445		Msg("skipping redundant deploy: container already running this image")
446
447	return true, existing
448}
449
450func (s *Service) containerForRedundantCheck(ctx context.Context, existing *domain.Container) *domain.Container {
451	if existing == nil || existing.ImageID != "" || existing.ID == "" {
452		return existing
453	}
454
455	log := zerowrap.FromCtx(ctx)
456	inspected, err := s.runtime.InspectContainer(ctx, existing.ID)
457	if err != nil {
458		log.Debug().Err(err).Str("container_id", existing.ID).Msg("cannot inspect existing container for redundancy check")
459		return existing
460	}
461	if inspected == nil || inspected.ImageID == "" {
462		return existing
463	}
464
465	existingCopy := *existing
466	existingCopy.ImageID = inspected.ImageID
467	return &existingCopy
468}
469
470type deployResources struct {
471	networkName    string
472	actualImageRef string
473	exposedPorts   []int
474	imageLabels    map[string]string
475	envVars        []string
476	envHash        string
477	volumes        map[string]string
478}
479
480// resolveExistingContainer returns the currently running container for a domain.
481// It first checks the in-memory map (fast path). If not found, it queries the
482// runtime for a running container with matching name and managed label. This
483// handles cases where Gordon restarted and in-memory state is stale.
484//
485// The slow path checks canonical, -new, and -next names because a previous
486// deploy may have been interrupted (e.g. eventbus timeout, Gordon restart)
487// leaving the active container under a temp name. Canonical is preferred when
488// multiple matches exist.
489func (s *Service) resolveExistingContainer(ctx context.Context, domainName string) (*domain.Container, bool) {
490	// Fast path: check in-memory state
491	s.mu.RLock()
492	container, ok := s.containers[domainName]
493	s.mu.RUnlock()
494	if ok {
495		return container, true
496	}
497
498	// Slow path: query runtime for running containers
499	log := zerowrap.FromCtx(ctx)
500	canonicalName := fmt.Sprintf("gordon-%s", domainName)
501	candidateNames := map[string]bool{
502		canonicalName:           true,
503		canonicalName + "-new":  true,
504		canonicalName + "-next": true,
505	}
506
507	running, err := s.runtime.ListContainers(ctx, false)
508	if err != nil {
509		log.Warn().Err(err).Msg("failed to list running containers for existing container resolution")
510		return nil, false
511	}
512
513	// Prefer canonical name; fall back to any running managed temp container.
514	var best *domain.Container
515	for _, c := range running {
516		if !candidateNames[c.Name] || c.Labels[domain.LabelManaged] != "true" {
517			continue
518		}
519		if best == nil || c.Name == canonicalName {
520			best = c
521		}
522	}
523
524	if best != nil {
525		log.Info().
526			Str("container_id", best.ID).
527			Str("container_name", best.Name).
528			Msg("resolved existing container from runtime (in-memory state was stale)")
529
530		// Update in-memory state so subsequent lookups are fast
531		s.mu.Lock()
532		if _, alreadyTracked := s.containers[domainName]; !alreadyTracked {
533			s.managedCount++
534		}
535		s.containers[domainName] = best
536		s.mu.Unlock()
537
538		return best, true
539	}
540
541	return nil, false
542}
543
544func (s *Service) prepareDeployResources(ctx context.Context, route domain.Route, existing *domain.Container) (*deployResources, error) {
545	log := zerowrap.FromCtx(ctx)
546
547	existingID := ""
548	if existing != nil {
549		existingID = existing.ID
550	}
551	if err := s.cleanupOrphanedContainers(ctx, route.Domain, existingID); err != nil {
552		log.WrapErr(err, "failed to cleanup orphaned containers")
553	}
554
555	imageRef := s.buildImageRef(route.Image)
556	actualImageRef, err := s.ensureImage(ctx, imageRef)
557	if err != nil {
558		return nil, err
559	}
560
561	networkName := s.getNetworkForApp(route.Domain)
562	if err := s.createNetworkIfNeeded(ctx, networkName); err != nil {
563		return nil, log.WrapErr(err, "failed to create network")
564	}
565	if err := s.deployAttachments(ctx, route.Domain, networkName); err != nil {
566		return nil, log.WrapErr(err, "failed to deploy attachments")
567	}
568
569	exposedPorts, err := s.runtime.GetImageExposedPorts(ctx, actualImageRef)
570	if err != nil {
571		log.WrapErr(err, "failed to get exposed ports, using defaults")
572		exposedPorts = []int{80, 8080, 3000}
573	}
574
575	imageLabels, err := s.runtime.GetImageLabels(ctx, actualImageRef)
576	if err != nil {
577		log.Warn().Err(err).Msg("failed to get image labels, skipping label propagation")
578		imageLabels = nil
579	}
580
581	envVars, err := s.loadEnvironment(ctx, route.Env, route.Domain, actualImageRef)
582	if err != nil {
583		return nil, err
584	}
585	envHash := hashEnvironment(envVars)
586
587	volumes, err := s.setupVolumes(ctx, route.Domain, actualImageRef)
588	if err != nil {
589		return nil, err
590	}
591
592	return &deployResources{
593		networkName:    networkName,
594		actualImageRef: actualImageRef,
595		exposedPorts:   exposedPorts,
596		imageLabels:    imageLabels,
597		envVars:        envVars,
598		envHash:        envHash,
599		volumes:        volumes,
600	}, nil
601}
602
603func (s *Service) createStartedContainer(ctx context.Context, route domain.Route, existing *domain.Container, resources *deployResources) (*domain.Container, error) {
604	ctx, span := tracer.Start(ctx, "container.create_and_start")
605	defer span.End()
606
607	log := zerowrap.FromCtx(ctx)
608
609	containerConfig := s.buildContainerConfig(containerConfigInput{
610		Domain:       route.Domain,
611		Image:        route.Image,
612		ImageRef:     resources.actualImageRef,
613		ExposedPorts: resources.exposedPorts,
614		EnvVars:      resources.envVars,
615		EnvHash:      resources.envHash,
616		Volumes:      resources.volumes,
617		NetworkName:  resources.networkName,
618		ImageLabels:  resources.imageLabels,
619		Existing:     existing,
620	})
621
622	newContainer, err := s.runtime.CreateContainer(ctx, containerConfig)
623	if err != nil {
624		return nil, log.WrapErr(err, "failed to create container")
625	}
626	if err := s.runtime.StartContainer(ctx, newContainer.ID); err != nil {
627		s.runtime.RemoveContainer(ctx, newContainer.ID, true)
628		return nil, log.WrapErr(err, "failed to start container")
629	}
630	if !domain.IsSkipReadiness(ctx) {
631		if err := s.waitForReady(ctx, newContainer.ID, containerConfig); err != nil {
632			s.cleanupFailedContainer(ctx, newContainer.ID)
633			return nil, log.WrapErr(err, "container failed readiness check")
634		}
635	}
636
637	inspected, err := s.runtime.InspectContainer(ctx, newContainer.ID)
638	if err != nil {
639		s.cleanupFailedContainer(ctx, newContainer.ID)
640		return nil, log.WrapErr(err, "failed to inspect started container")
641	}
642
643	return inspected, nil
644}
645
646func (s *Service) activateDeployedContainer(ctx context.Context, domainName string, container *domain.Container) bool {
647	s.mu.Lock()
648	_, wasTracked := s.containers[domainName]
649	s.containers[domainName] = container
650	if !wasTracked {
651		s.managedCount++
652	}
653	s.mu.Unlock()
654
655	// Track managed container count (only increment for new domains, not replacements)
656	if !wasTracked && s.metrics != nil {
657		s.metrics.ManagedContainers.Add(ctx, 1)
658	}
659
660	s.publishContainerDeployed(ctx, domainName, container.ID)
661
662	s.mu.RLock()
663	inv := s.cacheInvalidator
664	s.mu.RUnlock()
665	if inv != nil {
666		inv.InvalidateTarget(ctx, domainName)
667		return true
668	}
669
670	return false
671}
672
673// stabilizeNewContainer monitors the new container briefly after traffic switch.
674// If it crashes during this window, rolls back to old container.
675// Returns true if stabilization succeeded, false if rollback was performed.
676// Returns an error only when both new and old containers are dead.
677func (s *Service) stabilizeNewContainer(ctx context.Context, domainName string, newContainer, oldContainer *domain.Container) (bool, error) {
678	log := zerowrap.FromCtx(ctx)
679
680	if oldContainer == nil {
681		return true, nil
682	}
683
684	s.mu.RLock()
685	delay := s.config.StabilizationDelay
686	s.mu.RUnlock()
687	if delay == 0 {
688		delay = 2 * time.Second
689	}
690
691	// Brief stabilization: verify new container is still running after delay
692	select {
693	case <-time.After(delay):
694	case <-ctx.Done():
695		return true, nil
696	}
697
698	running, err := s.runtime.IsContainerRunning(ctx, newContainer.ID)
699	if err != nil || !running {
700		log.Error().
701			Str("new_container", newContainer.ID).
702			Str("old_container", oldContainer.ID).
703			Msg("new container crashed during stabilization, rolling back to old")
704
705		// Verify old container is still running before restoring it.
706		// If both old and new are dead (e.g., OOM), returning a dead
707		// container as "existing" would leave the domain in a broken state.
708		oldRunning, oldErr := s.runtime.IsContainerRunning(ctx, oldContainer.ID)
709		if oldErr != nil || !oldRunning {
710			log.Error().
711				Str("old_container", oldContainer.ID).
712				Msg("old container is also not running, cannot rollback")
713
714			// Cleanup failed new container
715			if stopErr := s.runtime.StopContainer(ctx, newContainer.ID); stopErr != nil {
716				log.WrapErrWithFields(stopErr, "failed to stop failed new container during rollback", map[string]any{zerowrap.FieldEntityID: newContainer.ID})
717			}
718			if removeErr := s.runtime.RemoveContainer(ctx, newContainer.ID, true); removeErr != nil {
719				log.WrapErrWithFields(removeErr, "failed to remove failed new container during rollback", map[string]any{zerowrap.FieldEntityID: newContainer.ID})
720			}
721
722			return false, fmt.Errorf("stabilization failed: new container crashed and old container %s is also not running", oldContainer.ID)
723		}
724
725		// Rollback: restore old container as tracked
726		s.mu.Lock()
727		s.containers[domainName] = oldContainer
728		s.mu.Unlock()
729
730		// Re-invalidate proxy cache to point back to old
731		s.mu.RLock()
732		inv := s.cacheInvalidator
733		s.mu.RUnlock()
734		if inv != nil {
735			inv.InvalidateTarget(ctx, domainName)
736		}
737
738		// Cleanup failed new container
739		if stopErr := s.runtime.StopContainer(ctx, newContainer.ID); stopErr != nil {
740			log.WrapErrWithFields(stopErr, "failed to stop failed new container during rollback", map[string]any{zerowrap.FieldEntityID: newContainer.ID})
741		}
742		if removeErr := s.runtime.RemoveContainer(ctx, newContainer.ID, true); removeErr != nil {
743			log.WrapErrWithFields(removeErr, "failed to remove failed new container during rollback", map[string]any{zerowrap.FieldEntityID: newContainer.ID})
744		}
745
746		return false, nil
747	}
748
749	return true, nil
750}
751
752func (s *Service) finalizePreviousContainer(ctx context.Context, domainName string, existing *domain.Container, hasExisting, invalidated bool, newContainerID string) {
753	if !hasExisting {
754		return
755	}
756
757	if invalidated {
758		s.waitForDrain(ctx, existing.ID)
759	}
760
761	s.cleanupOldContainer(ctx, existing, newContainerID, domainName)
762}
763
764func (s *Service) waitForDrain(ctx context.Context, oldContainerID string) {
765	log := zerowrap.FromCtx(ctx)
766
767	s.mu.RLock()
768	cfg := s.config
769	waiter := s.drainWaiter
770	s.mu.RUnlock()
771
772	drainMode := cfg.DrainMode
773	if drainMode == "" {
774		drainMode = "delay"
775	}
776
777	shouldUseInFlight := (drainMode == "inflight" || drainMode == "auto") && waiter != nil
778	if shouldUseInFlight {
779		timeout := cfg.DrainTimeout
780		if timeout == 0 {
781			timeout = 30 * time.Second
782		}
783		drained := waiter.WaitForNoInFlight(ctx, oldContainerID, timeout)
784		if !drained {
785			log.Warn().
786				Str("old_container_id", oldContainerID).
787				Dur("drain_timeout", timeout).
788				Msg("drain wait timed out; old container may still have in-flight traffic")
789		}
790		return
791	}
792
793	drainDelay := 2 * time.Second
794	if cfg.DrainDelayConfigured {
795		drainDelay = cfg.DrainDelay
796	}
797	if drainDelay <= 0 {
798		return
799	}
800	select {
801	case <-time.After(drainDelay):
802	case <-ctx.Done():
803	}
804}
805
806// restartContainerWithRecovery restarts a container, reconciling stale state if needed.
807// It returns the (possibly refreshed) container and updated attachment IDs.
808func (s *Service) restartContainerWithRecovery(ctx context.Context, domainName string, container *domain.Container, attachmentIDs []string) (*domain.Container, []string, error) {
809	log := zerowrap.FromCtx(ctx)
810
811	if err := s.runtime.RestartContainer(ctx, container.ID); err != nil {
812		if !isContainerNotFoundError(err) {
813			return nil, nil, log.WrapErr(err, "failed to restart container")
814		}
815
816		// In-memory container ID can become stale after external runtime changes.
817		// Re-sync and retry once with the latest tracked container.
818		log.Warn().
819			Err(err).
820			Str(zerowrap.FieldEntityID, container.ID).
821			Msg("tracked container missing during restart, attempting state reconciliation")
822
823		if syncErr := s.SyncContainers(ctx); syncErr != nil {
824			log.Warn().Err(syncErr).Msg("failed to sync container state during restart recovery")
825		}
826
827		s.mu.RLock()
828		refreshed, refreshedExists := s.containers[domainName]
829		attachmentIDs = append([]string{}, s.attachments[domainName]...)
830		s.mu.RUnlock()
831
832		if !refreshedExists || refreshed == nil {
833			return nil, nil, domain.ErrContainerNotFound
834		}
835		if err := s.runtime.RestartContainer(ctx, refreshed.ID); err != nil {
836			return nil, nil, log.WrapErr(err, "failed to restart container after state reconciliation")
837		}
838		return refreshed, attachmentIDs, nil
839	}
840
841	return container, attachmentIDs, nil
842}
843
844// Restart restarts a running container for the given domain.
845func (s *Service) Restart(ctx context.Context, domainName string, withAttachments bool) error {
846	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
847		zerowrap.FieldLayer:   "usecase",
848		zerowrap.FieldUseCase: "Restart",
849		"domain":              domainName,
850	})
851	log := zerowrap.FromCtx(ctx)
852
853	s.mu.RLock()
854	container, exists := s.containers[domainName]
855	attachments := s.attachments[domainName]
856	var attachmentIDs []string
857	if len(attachments) > 0 {
858		attachmentIDs = make([]string, len(attachments))
859		copy(attachmentIDs, attachments)
860	}
861	s.mu.RUnlock()
862
863	if !exists || container == nil {
864		return domain.ErrContainerNotFound
865	}
866
867	// When attachments are requested, sync runtime state first to get accurate
868	// attachment tracking, then check if configured attachments are deployed.
869	// If fewer attachment containers are tracked than configured, fail early with
870	// guidance rather than restarting the main container into a broken state
871	// (e.g., missing database hostname).
872	if withAttachments {
873		if syncErr := s.SyncContainers(ctx); syncErr != nil {
874			log.Warn().Err(syncErr).Msg("failed to sync container state before attachment check")
875		}
876
877		// Re-read attachment IDs after sync
878		s.mu.RLock()
879		if freshAttachments := s.attachments[domainName]; len(freshAttachments) > 0 {
880			attachmentIDs = make([]string, len(freshAttachments))
881			copy(attachmentIDs, freshAttachments)
882		} else {
883			attachmentIDs = nil
884		}
885		s.mu.RUnlock()
886
887		configuredAttachments := s.resolveAttachmentsForDomain(domainName)
888		if len(configuredAttachments) > 0 && len(attachmentIDs) < len(configuredAttachments) {
889			missing := len(configuredAttachments) - len(attachmentIDs)
890			return fmt.Errorf("%w: domain %q has %d configured attachment(s) but only %d deployed (%d missing)",
891				domain.ErrAttachmentNotDeployed, domainName, len(configuredAttachments), len(attachmentIDs), missing)
892		}
893	}
894
895	// Restart the main container (with stale-state recovery)
896	container, attachmentIDs, err := s.restartContainerWithRecovery(ctx, domainName, container, attachmentIDs)
897	if err != nil {
898		return err
899	}
900	log.Info().Str(zerowrap.FieldEntityID, container.ID).Msg("container restarted")
901
902	// Record restart metric
903	if s.metrics != nil {
904		s.metrics.ContainerRestarts.Add(ctx, 1, metric.WithAttributes(
905			attribute.String("domain", domainName),
906			attribute.String("source", "api"),
907		))
908	}
909
910	// Restart attachments if requested
911	if withAttachments && len(attachmentIDs) > 0 {
912		for _, attachID := range attachmentIDs {
913			if err := s.runtime.RestartContainer(ctx, attachID); err != nil {
914				log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to restart attachment")
915				continue // Don't fail the whole operation for one attachment
916			}
917			log.Info().Str("attachment_id", attachID).Msg("attachment restarted")
918		}
919	}
920
921	return nil
922}
923
924// Stop stops a running container.
925func (s *Service) Stop(ctx context.Context, containerID string) error {
926	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
927		zerowrap.FieldLayer:    "usecase",
928		zerowrap.FieldUseCase:  "Stop",
929		zerowrap.FieldEntityID: containerID,
930	})
931	log := zerowrap.FromCtx(ctx)
932
933	// Stop log collection before stopping container
934	if s.logWriter != nil {
935		if err := s.logWriter.StopLogging(containerID); err != nil {
936			log.Warn().Err(err).Msg("failed to stop container log collection")
937		}
938	}
939
940	if err := s.runtime.StopContainer(ctx, containerID); err != nil {
941		return log.WrapErr(err, "failed to stop container")
942	}
943
944	log.Info().Msg("container stopped")
945	return nil
946}
947
948// Remove removes a container.
949func (s *Service) Remove(ctx context.Context, containerID string, force bool) error {
950	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
951		zerowrap.FieldLayer:    "usecase",
952		zerowrap.FieldUseCase:  "Remove",
953		zerowrap.FieldEntityID: containerID,
954	})
955	log := zerowrap.FromCtx(ctx)
956
957	// Stop log collection before removing container
958	if s.logWriter != nil {
959		if err := s.logWriter.StopLogging(containerID); err != nil {
960			log.Warn().Err(err).Msg("failed to stop container log collection")
961		}
962	}
963
964	// Find domain and attachment IDs for cleanup
965	containerDomain, attachmentIDs, cfg := s.findContainerContext(containerID)
966
967	if err := s.runtime.RemoveContainer(ctx, containerID, force); err != nil {
968		return log.WrapErr(err, "failed to remove container")
969	}
970
971	// Cleanup volumes
972	if containerDomain != "" && !cfg.VolumePreserve {
973		if err := s.cleanupVolumesForDomain(ctx, containerDomain); err != nil {
974			log.WrapErrWithFields(err, "failed to cleanup volumes", map[string]any{"domain": containerDomain})
975		}
976	}
977
978	// Clean up attachment containers
979	s.removeAttachments(ctx, attachmentIDs)
980
981	// Remove from tracking
982	s.mu.Lock()
983	var removedDomain string
984	for d, c := range s.containers {
985		if c.ID == containerID {
986			delete(s.containers, d)
987			removedDomain = d
988			s.managedCount--
989			log.Info().Str("domain", d).Msg("container removed")
990			break
991		}
992	}
993	if removedDomain != "" {
994		delete(s.attachments, removedDomain)
995	}
996	s.mu.Unlock()
997
998	// Decrement managed container count
999	if removedDomain != "" && s.metrics != nil {
1000		s.metrics.ManagedContainers.Add(ctx, -1)
1001	}
1002
1003	// Note: We intentionally do NOT clean up deployMu entries to avoid a race condition:
1004	// If Remove deletes the mutex entry while a concurrent Deploy is acquiring the lock,
1005	// the Deploy would create a fresh mutex, breaking serialization and allowing concurrent
1006	// deploys to the same domain. The memory footprint is acceptable (one small struct per
1007	// domain ever deployed), and this is the safer choice for correctness.
1008
1009	// Cleanup network
1010	if removedDomain != "" && cfg.NetworkIsolation {
1011		networkName := s.getNetworkForApp(removedDomain)
1012		if err := s.cleanupNetworkIfEmpty(ctx, networkName); err != nil {
1013			log.WrapErrWithFields(err, "failed to cleanup network", map[string]any{"network": networkName})
1014		}
1015	}
1016
1017	return nil
1018}
1019
1020// findContainerContext looks up the domain, attachment IDs, and config snapshot for a container.
1021func (s *Service) findContainerContext(containerID string) (string, []string, Config) {
1022	s.mu.RLock()
1023	defer s.mu.RUnlock()
1024	cfg := s.config
1025	var containerDomain string
1026	for d, c := range s.containers {
1027		if c.ID == containerID {
1028			containerDomain = d
1029			break
1030		}
1031	}
1032	var attachmentIDs []string
1033	if containerDomain != "" {
1034		attachmentIDs = append(attachmentIDs, s.attachments[containerDomain]...)
1035	}
1036	return containerDomain, attachmentIDs, cfg
1037}
1038
1039// removeAttachments stops and removes attachment containers best-effort.
1040func (s *Service) removeAttachments(ctx context.Context, attachmentIDs []string) {
1041	log := zerowrap.FromCtx(ctx)
1042	for _, attachID := range attachmentIDs {
1043		if s.logWriter != nil {
1044			if err := s.logWriter.StopLogging(attachID); err != nil {
1045				log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to stop attachment log collection")
1046			}
1047		}
1048		if err := s.runtime.StopContainer(ctx, attachID); err != nil {
1049			log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to stop attachment container")
1050		}
1051		if err := s.runtime.RemoveContainer(ctx, attachID, true); err != nil {
1052			log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to remove attachment container")
1053		}
1054	}
1055}
1056
1057// Get retrieves a container by domain.
1058func (s *Service) Get(_ context.Context, domainName string) (*domain.Container, bool) {
1059	s.mu.RLock()
1060	defer s.mu.RUnlock()
1061	container, exists := s.containers[domainName]
1062	return container, exists
1063}
1064
1065// List returns all managed containers.
1066func (s *Service) List(_ context.Context) map[string]*domain.Container {
1067	s.mu.RLock()
1068	defer s.mu.RUnlock()
1069
1070	result := make(map[string]*domain.Container, len(s.containers))
1071	maps.Copy(result, s.containers)
1072	return result
1073}
1074
1075// ListRoutesWithDetails returns routes with network and attachment info.
1076// Note: Container map is copied under lock, then external calls are made without lock.
1077// If containers are removed between copy and runtime calls, errors are handled gracefully
1078// (network becomes empty string). This trade-off avoids holding locks during I/O.
1079func (s *Service) ListRoutesWithDetails(ctx context.Context) []domain.RouteInfo {
1080	s.mu.RLock()
1081	containers := make(map[string]*domain.Container, len(s.containers))
1082	maps.Copy(containers, s.containers)
1083	s.mu.RUnlock()
1084
1085	// Fetch all attachments once to avoid N+1 queries
1086	attachmentsByDomain := s.getAllAttachments(ctx)
1087
1088	results := make([]domain.RouteInfo, 0, len(containers))
1089	for domainName, container := range containers {
1090		network := ""
1091		image := ""
1092		containerID := ""
1093		status := ""
1094		if container != nil {
1095			containerID = container.ID
1096			status = container.Status
1097			image = container.Image
1098			if container.Labels != nil {
1099				if labelImage, ok := container.Labels[domain.LabelImage]; ok && labelImage != "" {
1100					image = labelImage
1101				}
1102			}
1103			// Strip registry domain prefix from image for cleaner display
1104			image = s.stripRegistryPrefix(image)
1105			if networkName, err := s.runtime.GetContainerNetwork(ctx, container.ID); err == nil {
1106				network = networkName
1107			}
1108		}
1109
1110		results = append(results, domain.RouteInfo{
1111			Domain:          domainName,
1112			Image:           image,
1113			ContainerID:     containerID,
1114			ContainerStatus: status,
1115			Network:         network,
1116			Attachments:     attachmentsByDomain[domainName],
1117		})
1118	}
1119
1120	return results
1121}
1122
1123// stripRegistryPrefix removes the configured registry domain prefix from an image reference.
1124// For example, "reg.example.com/myapp:latest" becomes "myapp:latest" if registry domain is "reg.example.com".
1125func (s *Service) stripRegistryPrefix(image string) string {
1126	s.mu.RLock()
1127	cfg := s.config
1128	s.mu.RUnlock()
1129
1130	if cfg.RegistryDomain == "" {
1131		return image
1132	}
1133	prefix := strings.TrimSuffix(cfg.RegistryDomain, "/") + "/"
1134	if strings.HasPrefix(image, prefix) {
1135		return strings.TrimPrefix(image, prefix)
1136	}
1137	return image
1138}
1139
1140// ListAttachments returns attachments for a domain.
1141func (s *Service) ListAttachments(ctx context.Context, domainName string) []domain.Attachment {
1142	return s.getAttachmentsForDomain(ctx, domainName)
1143}
1144
1145// ListNetworks returns Gordon-managed networks.
1146func (s *Service) ListNetworks(ctx context.Context) ([]*domain.NetworkInfo, error) {
1147	s.mu.RLock()
1148	cfg := s.config
1149	s.mu.RUnlock()
1150
1151	networks, err := s.runtime.ListNetworks(ctx)
1152	if err != nil {
1153		return nil, err
1154	}
1155
1156	var filtered []*domain.NetworkInfo
1157	for _, network := range networks {
1158		if strings.HasPrefix(network.Name, cfg.NetworkPrefix+"-") && network.Labels[domain.LabelManaged] == "true" {
1159			filtered = append(filtered, network)
1160		}
1161	}
1162
1163	return filtered, nil
1164}
1165
1166// HealthCheck performs health checks on all containers.
1167func (s *Service) HealthCheck(ctx context.Context) map[string]bool {
1168	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1169		zerowrap.FieldLayer:   "usecase",
1170		zerowrap.FieldUseCase: "HealthCheck",
1171	})
1172	log := zerowrap.FromCtx(ctx)
1173
1174	// Copy container IDs under lock, then release before making Docker API calls
1175	s.mu.RLock()
1176	snapshot := make(map[string]string, len(s.containers))
1177	for d, c := range s.containers {
1178		snapshot[d] = c.ID
1179	}
1180	s.mu.RUnlock()
1181
1182	health := make(map[string]bool, len(snapshot))
1183	for d, id := range snapshot {
1184		running, err := s.runtime.IsContainerRunning(ctx, id)
1185		if err != nil {
1186			log.WrapErrWithFields(err, "health check failed", map[string]any{"domain": d, zerowrap.FieldEntityID: id})
1187			health[d] = false
1188		} else {
1189			health[d] = running
1190		}
1191	}
1192	return health
1193}
1194
1195// SyncContainers synchronizes containers with runtime state.
1196func (s *Service) SyncContainers(ctx context.Context) error {
1197	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1198		zerowrap.FieldLayer:   "usecase",
1199		zerowrap.FieldUseCase: "SyncContainers",
1200	})
1201	log := zerowrap.FromCtx(ctx)
1202
1203	// List containers without holding lock to avoid blocking during Docker API call
1204	allContainers, err := s.runtime.ListContainers(ctx, false)
1205	if err != nil {
1206		return log.WrapErr(err, "failed to list containers")
1207	}
1208
1209	managed := make(map[string]*domain.Container)
1210	attachments := make(map[string][]string)
1211	for _, c := range allContainers {
1212		if c.Labels == nil || c.Labels[domain.LabelManaged] != "true" {
1213			continue
1214		}
1215		if c.Labels[domain.LabelAttachment] == "true" {
1216			owner := c.Labels[domain.LabelAttachedTo]
1217			if owner != "" {
1218				attachments[owner] = append(attachments[owner], c.ID)
1219			}
1220			continue
1221		}
1222		if d, ok := c.Labels[domain.LabelDomain]; ok {
1223			managed[d] = c
1224		}
1225	}
1226
1227	s.mu.Lock()
1228	s.containers = managed
1229	s.attachments = attachments
1230	newCount := int64(len(managed))
1231	delta := newCount - s.managedCount
1232	s.managedCount = newCount
1233	s.mu.Unlock()
1234
1235	// Report initial/delta managed container count to OTel UpDownCounter.
1236	if delta != 0 && s.metrics != nil {
1237		s.metrics.ManagedContainers.Add(ctx, delta)
1238	}
1239
1240	log.Info().Int(zerowrap.FieldCount, len(managed)).Msg("container state synchronized")
1241	return nil
1242}
1243
1244// AutoStart starts containers for the provided routes that aren't running.
1245// It skips readiness checks to avoid blocking boot; the background monitor
1246// handles crash recovery.
1247func (s *Service) AutoStart(ctx context.Context, routes []domain.Route) error {
1248	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1249		zerowrap.FieldLayer:   "usecase",
1250		zerowrap.FieldUseCase: "AutoStart",
1251	})
1252	log := zerowrap.FromCtx(ctx)
1253
1254	log.Info().Int("route_count", len(routes)).Msg("auto-starting containers for configured routes")
1255
1256	// Filter out routes that already have a running container.
1257	var pending []domain.Route
1258	skipped := 0
1259	for _, route := range routes {
1260		if _, exists := s.Get(ctx, route.Domain); exists {
1261			log.Debug().Str("domain", route.Domain).Msg("container already running, skipping")
1262			skipped++
1263		} else {
1264			pending = append(pending, route)
1265		}
1266	}
1267
1268	if len(pending) == 0 {
1269		log.Info().Int("skipped", skipped).Msg("auto-start completed, all containers already running")
1270		return nil
1271	}
1272
1273	// Deploy all pending routes concurrently with readiness checks skipped.
1274	deployCtx := domain.WithSkipReadiness(ctx)
1275
1276	type result struct {
1277		route domain.Route
1278		err   error
1279	}
1280	results := make(chan result, len(pending))
1281	sem := make(chan struct{}, 4) // limit concurrency
1282
1283	for _, route := range pending {
1284		sem <- struct{}{}
1285		go func(r domain.Route) {
1286			defer func() { <-sem }()
1287
1288			log.Info().Str("domain", r.Domain).Str("image", r.Image).Msg("auto-starting container for route")
1289
1290			if _, err := s.Deploy(deployCtx, r); err != nil {
1291				results <- result{route: r, err: err}
1292				return
1293			}
1294			results <- result{route: r}
1295		}(route)
1296	}
1297
1298	var started, errCount int
1299	for i := 0; i < len(pending); i++ {
1300		res := <-results
1301		if res.err != nil {
1302			log.Warn().Err(res.err).Str("domain", res.route.Domain).Msg("failed to auto-start container")
1303			errCount++
1304		} else {
1305			started++
1306		}
1307	}
1308
1309	log.Info().
1310		Int("started", started).
1311		Int("skipped", skipped).
1312		Int("errors", errCount).
1313		Msg("auto-start completed")
1314
1315	if errCount > 0 {
1316		return fmt.Errorf("auto-start completed with %d errors", errCount)
1317	}
1318	return nil
1319}
1320
1321// Shutdown gracefully shuts down all managed containers.
1322func (s *Service) Shutdown(ctx context.Context) error {
1323	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1324		zerowrap.FieldLayer:   "usecase",
1325		zerowrap.FieldUseCase: "Shutdown",
1326	})
1327	log := zerowrap.FromCtx(ctx)
1328	log.Info().Msg("shutting down container manager...")
1329
1330	// Wait for any in-flight background cleanup goroutines to finish
1331	s.cleanupWg.Wait()
1332
1333	// Containers are left running across Gordon restarts.
1334	// SyncContainers + AutoStart will pick them back up on next boot.
1335
1336	// Close log writer to stop all log collection
1337	if s.logWriter != nil {
1338		if err := s.logWriter.Close(); err != nil {
1339			log.Warn().Err(err).Msg("failed to close container log writer")
1340		}
1341	}
1342
1343	log.Info().Msg("container manager shutdown complete")
1344	return nil
1345}
1346
1347// WaitForCleanup blocks until all background cleanup goroutines complete.
1348// Intended for use in tests to avoid mock assertion races.
1349func (s *Service) WaitForCleanup() {
1350	s.cleanupWg.Wait()
1351}
1352
1353// StartMonitor begins background monitoring of tracked containers.
1354// The monitor restarts crashed containers and detects crash loops.
1355// Safe to call multiple times; subsequent calls are no-ops.
1356func (s *Service) StartMonitor(ctx context.Context) {
1357	s.mu.Lock()
1358	if s.monitor != nil {
1359		s.mu.Unlock()
1360		return
1361	}
1362	s.monitor = newMonitor(s)
1363	m := s.monitor
1364	m.Start(ctx)
1365	s.mu.Unlock()
1366}
1367
1368// StopMonitor stops the background container monitor.
1369// Safe to call multiple times; subsequent calls are no-ops.
1370func (s *Service) StopMonitor() {
1371	s.mu.Lock()
1372	m := s.monitor
1373	s.monitor = nil
1374	s.mu.Unlock()
1375	if m != nil {
1376		m.Stop()
1377	}
1378}
1379
1380// UpdateConfig updates the service configuration.
1381func (s *Service) UpdateConfig(config Config) {
1382	s.mu.Lock()
1383	s.config = config
1384	s.mu.Unlock()
1385}
1386
1387// UpdateAttachments updates only the attachment configuration in the service.
1388// This is called after a config reload to propagate attachment changes without restart.
1389// The incoming map is deep-copied so external callers cannot mutate service state.
1390func (s *Service) UpdateAttachments(attachments map[string][]string) {
1391	var copied map[string][]string
1392	if attachments != nil {
1393		copied = make(map[string][]string, len(attachments))
1394		for k, v := range attachments {
1395			sl := make([]string, len(v))
1396			copy(sl, v)
1397			copied[k] = sl
1398		}
1399	}
1400	s.mu.Lock()
1401	s.config.Attachments = copied
1402	s.mu.Unlock()
1403}
1404
1405// Helper methods
1406
1407// cleanupFailedContainer stops and removes a container that failed to start properly.
1408func (s *Service) cleanupFailedContainer(ctx context.Context, containerID string) {
1409	log := zerowrap.FromCtx(ctx)
1410	if err := s.runtime.StopContainer(ctx, containerID); err != nil {
1411		log.Warn().Err(err).Str(zerowrap.FieldEntityID, containerID).Msg("failed to stop container after failure")
1412	}
1413	if err := s.runtime.RemoveContainer(ctx, containerID, true); err != nil {
1414		log.Warn().Err(err).Str(zerowrap.FieldEntityID, containerID).Msg("failed to remove container after failure")
1415	}
1416}
1417
1418// cleanupOldContainer stops and removes an old container after zero-downtime switch.
1419// It also renames the new container to the canonical name.
1420func (s *Service) cleanupOldContainer(ctx context.Context, old *domain.Container, newContainerID, domainName string) {
1421	log := zerowrap.FromCtx(ctx)
1422	log.Info().Str(zerowrap.FieldEntityID, old.ID).Msg("stopping old container after zero-downtime switch")
1423
1424	if s.logWriter != nil {
1425		if err := s.logWriter.StopLogging(old.ID); err != nil {
1426			log.Warn().Err(err).Str(zerowrap.FieldEntityID, old.ID).Msg("failed to stop logging for old container")
1427		}
1428	}
1429
1430	if err := s.runtime.StopContainer(ctx, old.ID); err != nil {
1431		log.Warn().Err(err).Str(zerowrap.FieldEntityID, old.ID).Msg("failed to stop old container")
1432	}
1433
1434	if err := s.runtime.RemoveContainer(ctx, old.ID, true); err != nil {
1435		log.Warn().Err(err).Str(zerowrap.FieldEntityID, old.ID).Msg("failed to remove old container")
1436	}
1437
1438	// Rename new container to canonical name
1439	canonicalName := fmt.Sprintf("gordon-%s", domainName)
1440	if err := s.runtime.RenameContainer(ctx, newContainerID, canonicalName); err != nil {
1441		log.Warn().Err(err).Str("canonical_name", canonicalName).Msg("failed to rename container to canonical name")
1442	}
1443}
1444
1445// startLogCollection starts log collection for a container in the background.
1446// Errors are logged but don't fail the calling operation.
1447func (s *Service) startLogCollection(ctx context.Context, containerID, domainName string) {
1448	if s.logWriter == nil {
1449		return
1450	}
1451
1452	log := zerowrap.FromCtx(ctx)
1453
1454	// Use context.WithoutCancel so the log stream outlives the HTTP request.
1455	// Without this, the Docker log stream closes when the deploy response completes.
1456	bgCtx := context.WithoutCancel(ctx)
1457
1458	logStream, err := s.runtime.GetContainerLogs(bgCtx, containerID, true)
1459	if err != nil {
1460		log.Warn().Err(err).Msg("failed to get container logs for collection")
1461		return
1462	}
1463
1464	if err := s.logWriter.StartLogging(bgCtx, containerID, domainName, logStream); err != nil {
1465		log.Warn().Err(err).Msg("failed to start container log collection")
1466		logStream.Close()
1467	}
1468}
1469
1470func (s *Service) buildImageRef(image string) string {
1471	s.mu.RLock()
1472	cfg := s.config
1473	s.mu.RUnlock()
1474
1475	if !cfg.RegistryAuthEnabled || cfg.RegistryDomain == "" {
1476		return image
1477	}
1478
1479	// Normalize registry domain by trimming any trailing slash
1480	reg := strings.TrimSuffix(cfg.RegistryDomain, "/")
1481
1482	// Check if image already has the registry domain prefix
1483	prefix := reg + "/"
1484	if strings.HasPrefix(image, prefix) {
1485		return image
1486	}
1487
1488	// Check if image already has an explicit registry (host:port/ or host.domain/).
1489	// We need to detect patterns like:
1490	// - "docker.io/library/nginx:latest" (has dot and slash)
1491	// - "localhost:5001/myapp:latest" (has host:port/ pattern)
1492	// - "myregistry:5000/app:v1" (has host:port/ pattern)
1493	// - "[fd00::1]:5000/app:v1" (IPv6 with port)
1494	if hasExplicitRegistry(image) {
1495		return image
1496	}
1497
1498	return fmt.Sprintf("%s/%s", reg, image)
1499}
1500
1501// hasExplicitRegistry checks if an image reference already includes an explicit registry.
1502// Returns true for patterns like "docker.io/image", "localhost:5000/image", "localhost/image",
1503// "registry:8080/image", "[fd00::1]/image", "[fd00::1]:5000/image".
1504func hasExplicitRegistry(image string) bool {
1505	// Find the first slash which separates registry from image name
1506	slashIdx := strings.Index(image, "/")
1507	if slashIdx == -1 {
1508		return false // No slash means no registry prefix (e.g., "myapp:latest")
1509	}
1510
1511	// Extract the part before the first slash (potential registry)
1512	registryPart := image[:slashIdx]
1513
1514	// Check for bracketed IPv6 address (e.g., "[fd00::1]" or "[fd00::1]:5000")
1515	if strings.HasPrefix(registryPart, "[") {
1516		// Look for closing bracket
1517		if closeBracket := strings.Index(registryPart, "]"); closeBracket != -1 {
1518			// Valid bracketed IPv6: either ends at ] or has ]:port
1519			return true
1520		}
1521	}
1522
1523	// Check for localhost (with or without port)
1524	if registryPart == "localhost" {
1525		return true
1526	}
1527
1528	// Check for host:port pattern (e.g., "localhost:5000", "registry:8080")
1529	if colonIdx := strings.LastIndex(registryPart, ":"); colonIdx != -1 {
1530		port := registryPart[colonIdx+1:]
1531		// If everything after colon is digits, it's a port
1532		if len(port) > 0 && isNumeric(port) {
1533			return true
1534		}
1535	}
1536
1537	// Check for domain pattern (has a dot, e.g., "docker.io", "gcr.io")
1538	if strings.Contains(registryPart, ".") {
1539		return true
1540	}
1541
1542	return false
1543}
1544
1545// isNumeric checks if a string contains only digits.
1546func isNumeric(s string) bool {
1547	for _, c := range s {
1548		if c < '0' || c > '9' {
1549			return false
1550		}
1551	}
1552	return true
1553}
1554
1555func normalizePullPolicy(policy string) string {
1556	switch strings.ToLower(strings.TrimSpace(policy)) {
1557	case PullPolicyAlways:
1558		return PullPolicyAlways
1559	case PullPolicyIfTagChanged:
1560		return PullPolicyIfTagChanged
1561	case PullPolicyIfNotPresent:
1562		return PullPolicyIfNotPresent
1563	default:
1564		return PullPolicyIfNotPresent
1565	}
1566}
1567
1568func isDigestRef(imageRef string) bool {
1569	return strings.Contains(imageRef, "@sha256:")
1570}
1571
1572func (s *Service) pullRefForDeploy(ctx context.Context, imageRef string) (string, bool) {
1573	if !domain.IsInternalDeploy(ctx) {
1574		return imageRef, false
1575	}
1576	s.mu.RLock()
1577	cfg := s.config
1578	s.mu.RUnlock()
1579	return rewriteToLocalRegistry(imageRef, cfg.RegistryDomain, cfg.RegistryPort), true
1580}
1581
1582// ensureImage ensures the image is available locally, pulling if needed.
1583// Returns the image reference to use for container operations:
1584//   - For digest references (@sha256:...), returns the pullRef since Docker can't tag digests
1585//   - For tagged images, returns the original imageRef after tagging the pulled image
1586func (s *Service) ensureImage(ctx context.Context, imageRef string) (string, error) {
1587	ctx, span := tracer.Start(ctx, "container.ensure_image",
1588		trace.WithAttributes(attribute.String("image", imageRef)))
1589	defer span.End()
1590
1591	ctx = zerowrap.CtxWithField(ctx, "image", imageRef)
1592	log := zerowrap.FromCtx(ctx)
1593
1594	// Determine if this is an internal deploy and what reference to use for pulls.
1595	pullRef, isInternal := s.pullRefForDeploy(ctx, imageRef)
1596	if pullRef != imageRef {
1597		log.Info().
1598			Str("original_ref", imageRef).
1599			Str("pull_ref", pullRef).
1600			Msg("internal deploy: using localhost registry for pull")
1601	}
1602
1603	found, err := s.ensureLocalImage(ctx, imageRef, pullRef)
1604	if err != nil {
1605		return "", err
1606	}
1607	if found {
1608		return imageRef, nil
1609	}
1610
1611	// Pull image
1612	log.Info().Msg("pulling image from registry")
1613
1614	if err := s.pullImage(ctx, pullRef, isInternal); err != nil {
1615		return "", err
1616	}
1617
1618	// For digest references, we can't create a tag (Docker doesn't allow it).
1619	// In this case, use the pullRef directly since the image is available under that reference.
1620	if strings.Contains(imageRef, "@sha256:") {
1621		log.Info().
1622			Str("pull_ref", pullRef).
1623			Msg("digest reference: using pull reference for container operations")
1624		return pullRef, nil
1625	}
1626
1627	if err := s.tagImageIfNeeded(ctx, pullRef, imageRef); err != nil {
1628		return "", err
1629	}
1630
1631	// Clean up the temporary pull reference tag to avoid duplicate entries
1632	if pullRef != imageRef {
1633		if err := s.runtime.UntagImage(ctx, pullRef); err != nil {
1634			// Log but don't fail - the canonical tag is already applied
1635			log.Debug().Err(err).Str("pull_ref", pullRef).Msg("failed to remove temporary pull tag")
1636		}
1637	}
1638
1639	log.Info().Msg("image pulled successfully")
1640	return imageRef, nil
1641}
1642
1643func (s *Service) ensureLocalImage(ctx context.Context, imageRef, pullRef string) (bool, error) {
1644	log := zerowrap.FromCtx(ctx)
1645
1646	// For internal deploys (image push events), always pull fresh image
1647	// because the same tag may reference new content (e.g., latest tag updated)
1648	if domain.IsInternalDeploy(ctx) {
1649		log.Info().Msg("internal deploy detected, forcing image pull to ensure latest content")
1650		return false, nil
1651	}
1652
1653	s.mu.RLock()
1654	cfg := s.config
1655	s.mu.RUnlock()
1656
1657	pullPolicy := normalizePullPolicy(cfg.PullPolicy)
1658	switch pullPolicy {
1659	case PullPolicyAlways:
1660		log.Info().Str("pull_policy", pullPolicy).Msg("pull policy forces image pull")
1661		return false, nil
1662	case PullPolicyIfTagChanged:
1663		if !isDigestRef(imageRef) {
1664			log.Info().Str("pull_policy", pullPolicy).Msg("tag reference detected, pulling to check for updates")
1665			return false, nil
1666		}
1667	}
1668
1669	localImages, err := s.runtime.ListImages(ctx)
1670	if err != nil {
1671		log.WrapErr(err, "failed to list local images, will attempt pull")
1672		return false, nil
1673	}
1674
1675	normalizedRef := normalizeImageRef(imageRef)
1676	normalizedPullRef := normalizeImageRef(pullRef)
1677	for _, img := range localImages {
1678		normalizedImage := normalizeImageRef(img)
1679		if normalizedImage == normalizedRef {
1680			log.Info().Msg("image found locally, skipping pull")
1681			return true, nil
1682		}
1683		if normalizedImage == normalizedPullRef {
1684			if err := s.tagImageIfNeeded(ctx, pullRef, imageRef); err != nil {
1685				return false, err
1686			}
1687			// Clean up the temporary pull reference tag
1688			if pullRef != imageRef {
1689				if err := s.runtime.UntagImage(ctx, pullRef); err != nil {
1690					log.Debug().Err(err).Str("pull_ref", pullRef).Msg("failed to remove temporary pull tag")
1691				}
1692			}
1693			log.Info().Msg("image found locally, skipping pull")
1694			return true, nil
1695		}
1696	}
1697
1698	return false, nil
1699}
1700
1701func (s *Service) pullImage(ctx context.Context, pullRef string, isInternal bool) error {
1702	s.mu.RLock()
1703	cfg := s.config
1704	s.mu.RUnlock()
1705
1706	log := zerowrap.FromCtx(ctx)
1707
1708	switch {
1709	case isInternal && cfg.RegistryAuthEnabled:
1710		if cfg.InternalRegistryUsername == "" || cfg.InternalRegistryPassword == "" {
1711			return log.WrapErr(fmt.Errorf("internal registry auth not configured"), "failed to pull image for internal deploy")
1712		}
1713		if err := s.pullImageWithRetry(ctx, func(ctx context.Context) error {
1714			return s.runtime.PullImageWithAuth(ctx, pullRef, cfg.InternalRegistryUsername, cfg.InternalRegistryPassword)
1715		}, isInternal); err != nil {
1716			return log.WrapErr(err, "failed to pull image with internal auth")
1717		}
1718	case isInternal:
1719		if err := s.pullImageWithRetry(ctx, func(ctx context.Context) error {
1720			return s.runtime.PullImage(ctx, pullRef)
1721		}, isInternal); err != nil {
1722			return log.WrapErr(err, "failed to pull image")
1723		}
1724	case cfg.RegistryAuthEnabled:
1725		if cfg.ServiceTokenUsername == "" || cfg.ServiceToken == "" {
1726			return log.WrapErr(fmt.Errorf("registry service token not configured"), "failed to pull image for registry auth")
1727		}
1728		if err := s.runtime.PullImageWithAuth(ctx, pullRef, cfg.ServiceTokenUsername, cfg.ServiceToken); err != nil {
1729			return log.WrapErr(err, "failed to pull image with auth")
1730		}
1731	default:
1732		if err := s.runtime.PullImage(ctx, pullRef); err != nil {
1733			return log.WrapErr(err, "failed to pull image")
1734		}
1735	}
1736
1737	return nil
1738}
1739
1740func (s *Service) pullImageWithRetry(ctx context.Context, pullFn func(context.Context) error, isInternal bool) error {
1741	log := zerowrap.FromCtx(ctx)
1742
1743	for attempt := 1; attempt <= internalPullMaxAttempts; attempt++ {
1744		err := pullFn(ctx)
1745		if err == nil {
1746			return nil
1747		}
1748		if !isInternal || !isConnectionRefusedError(err) || attempt == internalPullMaxAttempts {
1749			return fmt.Errorf("internal image pull failed after %d attempts: %w", attempt, err)
1750		}
1751
1752		backoff := time.Duration(attempt) * time.Second
1753		log.Warn().
1754			Err(err).
1755			Int("attempt", attempt).
1756			Dur("backoff", backoff).
1757			Msg("internal image pull failed with connection refused, retrying")
1758
1759		select {
1760		case <-time.After(backoff):
1761		case <-ctx.Done():
1762			return ctx.Err()
1763		}
1764	}
1765	return nil
1766}
1767
1768func (s *Service) tagImageIfNeeded(ctx context.Context, sourceRef, targetRef string) error {
1769	if sourceRef == targetRef {
1770		return nil
1771	}
1772
1773	// Cannot create a tag with a digest reference - Docker/Podman doesn't allow it.
1774	// When using digest references (image@sha256:...), skip tagging as the image
1775	// is already available by its digest.
1776	if strings.Contains(targetRef, "@sha256:") {
1777		return nil
1778	}
1779
1780	log := zerowrap.FromCtx(ctx)
1781	if err := s.runtime.TagImage(ctx, sourceRef, targetRef); err != nil {
1782		return log.WrapErr(err, "failed to tag image from pull reference")
1783	}
1784	return nil
1785}
1786
1787func (s *Service) loadEnvironment(ctx context.Context, preResolved []string, domainName, imageRef string) ([]string, error) {
1788	log := zerowrap.FromCtx(ctx)
1789
1790	var userEnvVars []string
1791	if len(preResolved) > 0 {
1792		userEnvVars = preResolved
1793	} else {
1794		var err error
1795		userEnvVars, err = s.envLoader.LoadEnv(ctx, domainName)
1796		if err != nil {
1797			return nil, log.WrapErr(err, "failed to load environment variables")
1798		}
1799	}
1800
1801	dockerfileEnvVars, err := s.runtime.InspectImageEnv(ctx, imageRef)
1802	if err != nil {
1803		log.WrapErr(err, "failed to inspect image environment")
1804		dockerfileEnvVars = []string{}
1805	}
1806
1807	return mergeEnvironmentVariables(dockerfileEnvVars, userEnvVars), nil
1808}
1809
1810func (s *Service) setupVolumes(ctx context.Context, domainName, imageRef string) (map[string]string, error) {
1811	s.mu.RLock()
1812	cfg := s.config
1813	s.mu.RUnlock()
1814
1815	log := zerowrap.FromCtx(ctx)
1816	volumes := make(map[string]string)
1817
1818	if !cfg.VolumeAutoCreate {
1819		return volumes, nil
1820	}
1821
1822	volumePaths, err := s.runtime.InspectImageVolumes(ctx, imageRef)
1823	if err != nil {
1824		log.WrapErr(err, "failed to inspect image volumes")
1825		return volumes, nil
1826	}
1827
1828	for _, path := range volumePaths {
1829		name := generateVolumeName(cfg.VolumePrefix, domainName, path)
1830
1831		exists, err := s.runtime.VolumeExists(ctx, name)
1832		if err != nil {
1833			log.WrapErrWithFields(err, "failed to check volume", map[string]any{"volume": name})
1834			continue
1835		}
1836
1837		if !exists {
1838			if err := s.runtime.CreateVolume(ctx, name); err != nil {
1839				log.WrapErrWithFields(err, "failed to create volume", map[string]any{"volume": name})
1840				continue
1841			}
1842			log.Info().Str("volume", name).Str(zerowrap.FieldPath, path).Msg("created volume")
1843		}
1844
1845		volumes[path] = name
1846	}
1847
1848	return volumes, nil
1849}
1850
1851func (s *Service) getNetworkForApp(domainName string) string {
1852	var networkIsolation bool
1853	var networkGroups map[string][]string
1854
1855	if s.configProvider != nil {
1856		snap := s.configProvider.GetAttachmentConfig()
1857		networkGroups = snap.NetworkGroups
1858		s.mu.RLock()
1859		networkIsolation = s.config.NetworkIsolation
1860		s.mu.RUnlock()
1861	} else {
1862		s.mu.RLock()
1863		networkIsolation = s.config.NetworkIsolation
1864		networkGroups = s.config.NetworkGroups
1865		s.mu.RUnlock()
1866	}
1867
1868	if !networkIsolation {
1869		return "bridge"
1870	}
1871
1872	for groupName, domains := range networkGroups {
1873		if slices.Contains(domains, domainName) {
1874			return s.generateNetworkName(groupName)
1875		}
1876	}
1877
1878	return s.generateNetworkName(domainName)
1879}
1880
1881func (s *Service) generateNetworkName(identifier string) string {
1882	s.mu.RLock()
1883	cfg := s.config
1884	s.mu.RUnlock()
1885
1886	return fmt.Sprintf("%s-%s", cfg.NetworkPrefix, strings.ReplaceAll(identifier, ".", "-"))
1887}
1888
1889func (s *Service) createNetworkIfNeeded(ctx context.Context, networkName string) error {
1890	if networkName == "bridge" || networkName == "default" {
1891		return nil
1892	}
1893
1894	ctx = zerowrap.CtxWithField(ctx, "network", networkName)
1895	log := zerowrap.FromCtx(ctx)
1896
1897	exists, err := s.runtime.NetworkExists(ctx, networkName)
1898	if err != nil {
1899		return log.WrapErr(err, "failed to check network existence")
1900	}
1901
1902	if !exists {
1903		if err := s.runtime.CreateNetwork(ctx, networkName, map[string]string{"driver": "bridge"}); err != nil {
1904			return log.WrapErr(err, "failed to create network")
1905		}
1906		log.Info().Msg("created network for app isolation")
1907	}
1908
1909	return nil
1910}
1911
1912func (s *Service) cleanupOrphanedContainers(ctx context.Context, domainName string, skipContainerID string) error {
1913	log := zerowrap.FromCtx(ctx)
1914	expectedName := fmt.Sprintf("gordon-%s", domainName)
1915	expectedNewName := expectedName + "-new"
1916	expectedNextName := expectedName + "-next"
1917
1918	allContainers, err := s.runtime.ListContainers(ctx, true)
1919	if err != nil {
1920		return err
1921	}
1922
1923	for _, c := range allContainers {
1924		if (c.Name == expectedName || c.Name == expectedNewName || c.Name == expectedNextName) && c.ID != skipContainerID {
1925			// Skip any running container regardless of name — a running
1926			// temp container may be actively serving traffic while the
1927			// new container stabilizes, or may be from a concurrent deploy.
1928			if c.Status == "running" {
1929				log.Debug().
1930					Str(zerowrap.FieldEntityID, c.ID).
1931					Str("container_name", c.Name).
1932					Str(zerowrap.FieldStatus, c.Status).
1933					Msg("skipping running container during orphan cleanup")
1934				continue
1935			}
1936
1937			log.Info().Str(zerowrap.FieldEntityID, c.ID).Str("container_name", c.Name).Str(zerowrap.FieldStatus, c.Status).Msg("found orphaned container, removing")
1938
1939			if err := s.runtime.StopContainer(ctx, c.ID); err != nil {
1940				log.WrapErrWithFields(err, "failed to stop orphaned container", map[string]any{zerowrap.FieldEntityID: c.ID})
1941			}
1942
1943			if err := s.runtime.RemoveContainer(ctx, c.ID, true); err != nil {
1944				log.WrapErrWithFields(err, "failed to remove orphaned container", map[string]any{zerowrap.FieldEntityID: c.ID})
1945			}
1946		}
1947	}
1948
1949	return nil
1950}
1951
1952func (s *Service) cleanupVolumesForDomain(_ context.Context, _ string) error {
1953	return nil
1954}
1955
1956func (s *Service) cleanupNetworkIfEmpty(ctx context.Context, networkName string) error {
1957	if networkName == "bridge" || networkName == "default" {
1958		return nil
1959	}
1960
1961	ctx = zerowrap.CtxWithField(ctx, "network", networkName)
1962	log := zerowrap.FromCtx(ctx)
1963
1964	networks, err := s.runtime.ListNetworks(ctx)
1965	if err != nil {
1966		return log.WrapErr(err, "failed to list networks")
1967	}
1968
1969	for _, n := range networks {
1970		if n.Name == networkName && len(n.Containers) == 0 {
1971			if err := s.runtime.RemoveNetwork(ctx, networkName); err != nil {
1972				return err
1973			}
1974			log.Info().Msg("cleaned up empty network")
1975			break
1976		}
1977	}
1978
1979	return nil
1980}
1981
1982func (s *Service) deployAttachments(ctx context.Context, domainName, networkName string) error {
1983	// Collect attachments from both domain-specific and network group configs
1984	attachments := s.resolveAttachmentsForDomain(domainName)
1985	if len(attachments) == 0 {
1986		return nil
1987	}
1988
1989	log := zerowrap.FromCtx(ctx)
1990	var deployed []string // track successfully deployed attachment IDs for rollback
1991	for _, svc := range attachments {
1992		if err := s.deployAttachedService(ctx, domainName, svc, networkName); err != nil {
1993			log.WrapErrWithFields(err, "failed to deploy attachment", map[string]any{zerowrap.FieldService: svc, "domain": domainName})
1994
1995			// Rollback: clean up already-deployed attachments
1996			for _, id := range deployed {
1997				if stopErr := s.runtime.StopContainer(ctx, id); stopErr != nil {
1998					log.Warn().Err(stopErr).Str("attachment_id", id).Msg("rollback: failed to stop attachment")
1999				}
2000				if rmErr := s.runtime.RemoveContainer(ctx, id, true); rmErr != nil {
2001					log.Warn().Err(rmErr).Str("attachment_id", id).Msg("rollback: failed to remove attachment")
2002				}
2003			}
2004			// Deregister rolled-back attachments
2005			if len(deployed) > 0 {
2006				s.mu.Lock()
2007				remaining := s.attachments[domainName]
2008				filtered := remaining[:0]
2009				rollbackSet := make(map[string]bool, len(deployed))
2010				for _, id := range deployed {
2011					rollbackSet[id] = true
2012				}
2013				for _, id := range remaining {
2014					if !rollbackSet[id] {
2015						filtered = append(filtered, id)
2016					}
2017				}
2018				s.attachments[domainName] = filtered
2019				s.mu.Unlock()
2020			}
2021
2022			return fmt.Errorf("failed to deploy attachment %q (rolled back %d already-deployed)", svc, len(deployed))
2023		}
2024		// Collect IDs of attachments tracked under this domain after successful deploy
2025		s.mu.RLock()
2026		ids := s.attachments[domainName]
2027		if len(ids) > 0 {
2028			latest := ids[len(ids)-1]
2029			deployed = append(deployed, latest)
2030		}
2031		s.mu.RUnlock()
2032	}
2033	return nil
2034}
2035
2036// resolveAttachmentsForDomain returns attachments for a domain by checking:
2037// 1. Direct domain attachments (attachments[domain])
2038// 2. Network group attachments (attachments[group] where domain is in network_groups[group])
2039func (s *Service) resolveAttachmentsForDomain(domainName string) []string {
2040	var attachments map[string][]string
2041	var networkGroups map[string][]string
2042
2043	if s.configProvider != nil {
2044		snap := s.configProvider.GetAttachmentConfig()
2045		attachments = snap.Attachments
2046		networkGroups = snap.NetworkGroups
2047	} else {
2048		s.mu.RLock()
2049		attachments = s.config.Attachments
2050		networkGroups = s.config.NetworkGroups
2051		s.mu.RUnlock()
2052	}
2053
2054	seen := make(map[string]bool)
2055	var result []string
2056
2057	// First, add domain-specific attachments
2058	if domainAttachments, ok := attachments[domainName]; ok {
2059		for _, img := range domainAttachments {
2060			if !seen[img] {
2061				seen[img] = true
2062				result = append(result, img)
2063			}
2064		}
2065	}
2066
2067	// Then, find which network group this domain belongs to and add group attachments
2068	for groupName, domains := range networkGroups {
2069		if slices.Contains(domains, domainName) {
2070			if groupAttachments, ok := attachments[groupName]; ok {
2071				for _, img := range groupAttachments {
2072					if !seen[img] {
2073						seen[img] = true
2074						result = append(result, img)
2075					}
2076				}
2077			}
2078			break // Domain can only be in one network group
2079		}
2080	}
2081
2082	return result
2083}
2084
2085func (s *Service) getAttachmentsForDomain(ctx context.Context, domainName string) []domain.Attachment {
2086	containers, err := s.runtime.ListContainers(ctx, true)
2087	if err != nil {
2088		return nil
2089	}
2090
2091	return s.filterAttachments(ctx, containers, domainName)
2092}
2093
2094// getAllAttachments fetches all containers once and returns attachments grouped by domain.
2095// This avoids N+1 queries when listing multiple routes.
2096func (s *Service) getAllAttachments(ctx context.Context) map[string][]domain.Attachment {
2097	containers, err := s.runtime.ListContainers(ctx, true)
2098	if err != nil {
2099		return nil
2100	}
2101
2102	result := make(map[string][]domain.Attachment)
2103	for _, container := range containers {
2104		if container.Labels == nil {
2105			continue
2106		}
2107		if container.Labels[domain.LabelAttachment] != "true" {
2108			continue
2109		}
2110		ownerDomain := container.Labels[domain.LabelAttachedTo]
2111		if ownerDomain == "" {
2112			continue
2113		}
2114		image := container.Image
2115		serviceName := container.Name
2116		if labelImage, ok := container.Labels[domain.LabelImage]; ok && labelImage != "" {
2117			image = labelImage
2118			serviceName = extractServiceName(labelImage)
2119		}
2120		// Get the container's network for display in the routes table
2121		network := ""
2122		if networkName, err := s.runtime.GetContainerNetwork(ctx, container.ID); err == nil {
2123			network = networkName
2124		}
2125		attachment := domain.Attachment{
2126			Name:        serviceName,
2127			Image:       s.stripRegistryPrefix(image),
2128			ContainerID: container.ID,
2129			Status:      container.Status,
2130			Network:     network,
2131			Ports:       append([]int(nil), container.Ports...),
2132		}
2133		result[ownerDomain] = append(result[ownerDomain], attachment)
2134	}
2135
2136	return result
2137}
2138
2139// filterAttachments extracts attachments for a specific domain from a container list.
2140func (s *Service) filterAttachments(ctx context.Context, containers []*domain.Container, domainName string) []domain.Attachment {
2141	attachments := make([]domain.Attachment, 0)
2142	for _, container := range containers {
2143		if container.Labels == nil {
2144			continue
2145		}
2146		if container.Labels[domain.LabelAttachment] != "true" {
2147			continue
2148		}
2149		if container.Labels[domain.LabelAttachedTo] != domainName {
2150			continue
2151		}
2152		image := container.Image
2153		serviceName := container.Name
2154		if labelImage, ok := container.Labels[domain.LabelImage]; ok && labelImage != "" {
2155			image = labelImage
2156			serviceName = extractServiceName(labelImage)
2157		}
2158		// Get the container's network for display in the routes table
2159		network := ""
2160		if networkName, err := s.runtime.GetContainerNetwork(ctx, container.ID); err == nil {
2161			network = networkName
2162		}
2163		attachment := domain.Attachment{
2164			Name:        serviceName,
2165			Image:       s.stripRegistryPrefix(image),
2166			ContainerID: container.ID,
2167			Status:      container.Status,
2168			Network:     network,
2169			Ports:       append([]int(nil), container.Ports...),
2170		}
2171		attachments = append(attachments, attachment)
2172	}
2173
2174	return attachments
2175}
2176
2177func (s *Service) deployAttachedService(ctx context.Context, ownerDomain, serviceImage, networkName string) error {
2178	log := zerowrap.FromCtx(ctx)
2179
2180	// Parse service name from image (e.g., "my-postgres:latest" → "postgres")
2181	serviceName := extractServiceName(serviceImage)
2182	containerName := fmt.Sprintf("gordon-%s-%s", sanitizeName(ownerDomain), serviceName)
2183
2184	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
2185		"attachment":     serviceName,
2186		"container_name": containerName,
2187		"owner_domain":   ownerDomain,
2188	})
2189
2190	// Check if already running (idempotent)
2191	// Try new name first, then fall back to legacy name for backwards compatibility
2192	existingContainer, err := s.resolveExistingAttachment(ctx, ownerDomain, serviceName)
2193	if err != nil {
2194		return err
2195	}
2196
2197	if existingContainer != nil {
2198		shouldSkip, err := s.handleRunningAttachment(ctx, existingContainer, containerName, serviceImage)
2199		if err != nil {
2200			return err
2201		}
2202		if shouldSkip {
2203			return nil
2204		}
2205		if existingContainer.Status == string(domain.ContainerStatusRunning) {
2206			existingContainer = nil
2207		}
2208	}
2209
2210	if err := s.removeStoppedAttachment(ctx, existingContainer, containerName); err != nil {
2211		return err
2212	}
2213
2214	log.Info().Str(zerowrap.FieldService, serviceImage).Msg("deploying attached service")
2215
2216	// Ensure image (canonical ref is returned; internal pulls may use the local registry).
2217	imageRef := s.buildImageRef(serviceImage)
2218	actualImageRef, err := s.ensureImage(ctx, imageRef)
2219	if err != nil {
2220		return err
2221	}
2222
2223	// Get image metadata
2224	exposedPorts, err := s.runtime.GetImageExposedPorts(ctx, actualImageRef)
2225	if err != nil {
2226		log.WrapErr(err, "failed to get exposed ports for attachment, using defaults")
2227		exposedPorts = []int{}
2228	}
2229
2230	// Setup volumes (attachments need persistent data)
2231	volumes, err := s.setupVolumes(ctx, containerName, actualImageRef)
2232	if err != nil {
2233		log.WrapErr(err, "failed to setup volumes for attachment")
2234		volumes = make(map[string]string)
2235	}
2236
2237	// Load environment (attachment-specific env file)
2238	envVars, err := s.loadEnvironment(ctx, nil, containerName, actualImageRef)
2239	if err != nil {
2240		log.WrapErr(err, "failed to load environment for attachment")
2241		envVars = []string{}
2242	}
2243	envHash := hashEnvironment(envVars)
2244
2245	s.mu.RLock()
2246	cfg := s.config
2247	s.mu.RUnlock()
2248
2249	// Create container with attachment labels (use actual ref, track original in labels)
2250	config := &domain.ContainerConfig{
2251		Image:       actualImageRef,
2252		Name:        containerName,
2253		Hostname:    serviceName, // Internal DNS: postgres, redis, etc.
2254		Ports:       exposedPorts,
2255		Env:         envVars,
2256		Volumes:     volumes,
2257		NetworkMode: networkName, // Same network as main app
2258		Labels: map[string]string{
2259			domain.LabelManaged:    "true",
2260			domain.LabelAttachment: "true",
2261			domain.LabelAttachedTo: ownerDomain,
2262			domain.LabelEnvHash:    envHash,
2263			domain.LabelImage:      serviceImage,
2264		},
2265		MemoryLimit: cfg.DefaultMemoryLimit,
2266		NanoCPUs:    cfg.DefaultNanoCPUs,
2267		PidsLimit:   cfg.DefaultPidsLimit,
2268	}
2269
2270	container, err := s.runtime.CreateContainer(ctx, config)
2271	if err != nil {
2272		return log.WrapErr(err, "failed to create attachment container")
2273	}
2274
2275	// Start container
2276	if err := s.runtime.StartContainer(ctx, container.ID); err != nil {
2277		s.runtime.RemoveContainer(ctx, container.ID, true)
2278		return log.WrapErr(err, "failed to start attachment container")
2279	}
2280
2281	// Track attachment
2282	s.mu.Lock()
2283	s.attachments[ownerDomain] = append(s.attachments[ownerDomain], container.ID)
2284	s.mu.Unlock()
2285
2286	// Start log collection for attachment
2287	s.startLogCollection(ctx, container.ID, containerName)
2288
2289	log.Info().Str(zerowrap.FieldEntityID, container.ID).Msg("attachment deployed successfully")
2290	return nil
2291}
2292
2293func (s *Service) attachmentEnvDrifted(ctx context.Context, existing *domain.Container, containerName, serviceImage string) (bool, error) {
2294	currentEnv, err := s.loadEnvironment(ctx, nil, containerName, s.buildImageRef(serviceImage))
2295	if err != nil {
2296		return false, err
2297	}
2298
2299	currentHash := hashEnvironment(currentEnv)
2300	existingHash, ok := existing.Labels[domain.LabelEnvHash]
2301	if ok && existingHash == currentHash {
2302		return false, nil
2303	}
2304
2305	return true, nil
2306}
2307
2308func (s *Service) handleRunningAttachment(ctx context.Context, existing *domain.Container, containerName, serviceImage string) (bool, error) {
2309	if existing.Status != string(domain.ContainerStatusRunning) {
2310		return false, nil
2311	}
2312
2313	log := zerowrap.FromCtx(ctx)
2314
2315	// Check image drift
2316	existingImage := existing.Labels[domain.LabelImage]
2317	imageDrifted := existingImage != "" && existingImage != serviceImage
2318
2319	// Check env drift
2320	envDrifted, err := s.attachmentEnvDrifted(ctx, existing, containerName, serviceImage)
2321	if err != nil {
2322		return false, log.WrapErr(err, "failed to check attachment env drift")
2323	}
2324
2325	if !envDrifted && !imageDrifted {
2326		log.Debug().Str("container_name", containerName).Msg("attachment already running with current env and image, skipping")
2327		return true, nil
2328	}
2329
2330	if imageDrifted {
2331		log.Info().Str("container_name", containerName).Str("existing_image", existingImage).Str("desired_image", serviceImage).Msg("attachment image changed, recreating")
2332	}
2333	if envDrifted {
2334		log.Info().Str("container_name", containerName).Msg("attachment env changed, recreating")
2335	}
2336
2337	if err := s.runtime.StopContainer(ctx, existing.ID); err != nil {
2338		return false, log.WrapErr(err, "failed to stop attachment for drift update")
2339	}
2340	if err := s.runtime.RemoveContainer(ctx, existing.ID, true); err != nil {
2341		return false, log.WrapErr(err, "failed to remove attachment for drift update")
2342	}
2343
2344	return false, nil
2345}
2346
2347func (s *Service) removeStoppedAttachment(ctx context.Context, existing *domain.Container, containerName string) error {
2348	if existing == nil {
2349		return nil
2350	}
2351
2352	log := zerowrap.FromCtx(ctx)
2353	log.Info().Str("container_name", containerName).Msg("removing stopped attachment container")
2354	if err := s.runtime.RemoveContainer(ctx, existing.ID, true); err != nil {
2355		return log.WrapErr(err, "failed to remove existing attachment container")
2356	}
2357
2358	return nil
2359}
2360
2361func (s *Service) resolveExistingAttachment(ctx context.Context, ownerDomain, serviceName string) (*domain.Container, error) {
2362	log := zerowrap.FromCtx(ctx)
2363	containerName := fmt.Sprintf("gordon-%s-%s", sanitizeName(ownerDomain), serviceName)
2364	existingContainer := s.findContainerByName(ctx, containerName)
2365	if existingContainer != nil {
2366		return existingContainer, nil
2367	}
2368
2369	containerNameLegacy := fmt.Sprintf("gordon-%s-%s", sanitizeNameLegacy(ownerDomain), serviceName)
2370	existingContainer = s.findContainerByName(ctx, containerNameLegacy)
2371	if existingContainer == nil {
2372		return nil, nil
2373	}
2374
2375	log.Info().Str("container_name", containerNameLegacy).Msg("found attachment with legacy naming, will be replaced")
2376	if err := s.runtime.StopContainer(ctx, existingContainer.ID); err != nil {
2377		return nil, log.WrapErr(err, "failed to stop legacy attachment container")
2378	}
2379	if err := s.runtime.RemoveContainer(ctx, existingContainer.ID, true); err != nil {
2380		return nil, log.WrapErr(err, "failed to remove legacy attachment container")
2381	}
2382
2383	return nil, nil
2384}
2385
2386// Utility functions
2387
2388func normalizeImageRef(image string) string {
2389	// Extract repository from image reference, handling port numbers correctly.
2390	// Examples:
2391	//   nginx:latest -> docker.io/library/nginx
2392	//   user/repo:tag -> docker.io/user/repo
2393	//   localhost:5000/image:tag -> localhost:5000/image
2394	//   registry.example.com/image:tag -> registry.example.com/image
2395	//
2396	// Note: This function assumes valid Docker image references. Edge cases like
2397	// "registry.com:5000" (registry with port but no image name) are not valid
2398	// image references per Docker's naming conventions, which require at least
2399	// one path component after the registry (e.g., "registry.com:5000/image").
2400
2401	// Find the tag separator: the last colon that isn't part of a port number.
2402	// A colon is part of a port if there's no slash after it until the next colon.
2403	repo := image
2404	lastColon := strings.LastIndex(image, ":")
2405	if lastColon != -1 {
2406		afterColon := image[lastColon+1:]
2407		// If there's no slash after the colon, it's the tag separator
2408		if !strings.Contains(afterColon, "/") {
2409			repo = image[:lastColon]
2410		}
2411	}
2412
2413	if !strings.Contains(repo, "/") {
2414		return "docker.io/library/" + repo
2415	}
2416
2417	if strings.Count(repo, "/") == 1 && !strings.Contains(strings.Split(repo, "/")[0], ".") && !strings.Contains(strings.Split(repo, "/")[0], ":") {
2418		return "docker.io/" + repo
2419	}
2420
2421	return repo
2422}
2423
2424func generateVolumeName(prefix, domainName, volumePath string) string {
2425	return fmt.Sprintf("%s-%s-%s",
2426		prefix,
2427		strings.ReplaceAll(domainName, ".", "-"),
2428		strings.ReplaceAll(strings.Trim(volumePath, "/"), "/", "-"))
2429}
2430
2431func mergeEnvironmentVariables(dockerfileEnv, userEnv []string) []string {
2432	envMap := make(map[string]string)
2433
2434	for _, env := range dockerfileEnv {
2435		if k, v, ok := strings.Cut(env, "="); ok {
2436			envMap[k] = v
2437		}
2438	}
2439
2440	for _, env := range userEnv {
2441		if k, v, ok := strings.Cut(env, "="); ok {
2442			envMap[k] = v
2443		}
2444	}
2445
2446	keys := make([]string, 0, len(envMap))
2447	for k := range envMap {
2448		keys = append(keys, k)
2449	}
2450	slices.Sort(keys)
2451
2452	result := make([]string, 0, len(envMap))
2453	for _, k := range keys {
2454		result = append(result, k+"="+envMap[k])
2455	}
2456
2457	return result
2458}
2459
2460// extractServiceName gets service name from image reference.
2461// "my-postgres:latest" → "postgres", "redis:7" → "redis"
2462func extractServiceName(image string) string {
2463	// Remove tag
2464	parts := strings.Split(image, ":")
2465	name := parts[0]
2466
2467	// Remove registry prefix if present
2468	if strings.Contains(name, "/") {
2469		nameParts := strings.Split(name, "/")
2470		name = nameParts[len(nameParts)-1]
2471	}
2472
2473	// Remove common prefixes like "my-"
2474	name = strings.TrimPrefix(name, "my-")
2475
2476	return name
2477}
2478
2479// sanitizeName is a convenience wrapper around domain.SanitizeDomainForContainer.
2480func sanitizeName(d string) string {
2481	return domain.SanitizeDomainForContainer(d)
2482}
2483
2484// sanitizeNameLegacy is a convenience wrapper around domain.SanitizeDomainForContainerLegacy.
2485func sanitizeNameLegacy(d string) string {
2486	return domain.SanitizeDomainForContainerLegacy(d)
2487}
2488
2489func isContainerNotFoundError(err error) bool {
2490	if err == nil {
2491		return false
2492	}
2493	if errors.Is(err, domain.ErrContainerNotFound) {
2494		return true
2495	}
2496	msg := strings.ToLower(err.Error())
2497	return strings.Contains(msg, "no such container") ||
2498		strings.Contains(msg, "no container with name or id") ||
2499		strings.Contains(msg, "container not found")
2500}
2501
2502func isConnectionRefusedError(err error) bool {
2503	if err == nil {
2504		return false
2505	}
2506	return strings.Contains(strings.ToLower(err.Error()), "connection refused")
2507}
2508
2509// findContainerByName finds a container by its name.
2510func (s *Service) findContainerByName(ctx context.Context, name string) *domain.Container {
2511	containers, err := s.runtime.ListContainers(ctx, true)
2512	if err != nil {
2513		return nil
2514	}
2515
2516	for _, c := range containers {
2517		if c.Name == name {
2518			return c
2519		}
2520	}
2521	return nil
2522}
2523
2524func (s *Service) waitForReady(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig) error {
2525	if err := s.pollContainerRunning(ctx, containerID); err != nil {
2526		return err
2527	}
2528
2529	s.mu.RLock()
2530	cfg := s.config
2531	s.mu.RUnlock()
2532
2533	readinessMode := cfg.ReadinessMode
2534	if readinessMode == "" {
2535		readinessMode = "delay"
2536	}
2537
2538	switch readinessMode {
2539	case "delay":
2540		return s.waitForReadyByDelay(ctx, containerID)
2541	case "docker-health":
2542		_, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(ctx, containerID)
2543		if err != nil {
2544			return err
2545		}
2546		if !hasHealthcheck {
2547			return errors.New("no healthcheck detected")
2548		}
2549		return s.waitForHealthy(ctx, containerID, cfg.HealthTimeout)
2550	default: // auto
2551		return s.readinessCascade(ctx, containerID, containerConfig, cfg)
2552	}
2553}
2554
2555// readinessCascade auto-detects the strongest available readiness signal:
2556//  1. Docker healthcheck (if present) → wait for healthy status
2557//  2. HTTP probe (if gordon.health label set) → GET until 2xx/3xx
2558//  3. Default HTTP probe (GET / on exposed port) → wait for 2xx/3xx
2559//  4. TCP probe (if port info available) → connect until accepted
2560//  5. Delay fallback (last resort) → waitForReadyByDelay
2561func (s *Service) readinessCascade(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) error {
2562	log := zerowrap.FromCtx(ctx)
2563
2564	// 1. Docker healthcheck
2565	_, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(ctx, containerID)
2566	if err != nil {
2567		return err
2568	}
2569	if hasHealthcheck {
2570		log.Info().Msg("readiness cascade: using Docker healthcheck")
2571		return s.waitForHealthy(ctx, containerID, cfg.HealthTimeout)
2572	}
2573
2574	// 2. HTTP probe via gordon.health label
2575	if probed, probeErr := s.tryHTTPProbe(ctx, containerID, containerConfig, cfg); probed {
2576		return probeErr
2577	}
2578
2579	// 3. Default HTTP probe on root path (if port info available)
2580	if probed, probeErr := s.tryDefaultHTTPProbe(ctx, containerID, containerConfig, cfg); probed {
2581		return probeErr
2582	}
2583
2584	// 4. TCP probe (if port info available)
2585	if probed, probeErr := s.tryTCPProbe(ctx, containerID, containerConfig, cfg); probed {
2586		return probeErr
2587	}
2588
2589	// 5. Delay fallback
2590	log.Info().Msg("readiness cascade: using delay fallback")
2591	return s.waitForReadyByDelay(ctx, containerID)
2592}
2593
2594// tryHTTPProbe attempts an HTTP probe if the gordon.health label is set.
2595// Returns (true, err) if the probe was attempted, (false, nil) if skipped.
2596func (s *Service) tryHTTPProbe(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) (bool, error) {
2597	log := zerowrap.FromCtx(ctx)
2598	if containerConfig == nil {
2599		return false, nil
2600	}
2601	healthPath, ok := containerConfig.Labels[domain.LabelHealth]
2602	if !ok || healthPath == "" {
2603		return false, nil
2604	}
2605	ip, port, probeErr := s.resolveProbeEndpoint(ctx, containerID, containerConfig)
2606	if probeErr != nil || ip == "" || port <= 0 {
2607		log.Debug().Err(probeErr).Msg("readiness cascade: HTTP probe skipped, could not resolve container endpoint")
2608		return false, nil
2609	}
2610	url := fmt.Sprintf("http://%s:%d%s", ip, port, healthPath)
2611	timeout := cfg.HTTPProbeTimeout
2612	if timeout == 0 {
2613		timeout = 60 * time.Second
2614	}
2615	log.Info().Str("url", url).Dur("timeout", timeout).Msg("readiness cascade: using HTTP probe")
2616	return true, httpProbe(ctx, url, timeout)
2617}
2618
2619// tryDefaultHTTPProbe attempts an HTTP GET on "/" using the container's
2620// exposed port. This is the default readiness check for web containers —
2621// it verifies the app is actually serving HTTP, not just accepting TCP.
2622// Returns (true, err) if the probe was attempted, (false, nil) if skipped.
2623func (s *Service) tryDefaultHTTPProbe(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) (bool, error) {
2624	log := zerowrap.FromCtx(ctx)
2625	if containerConfig == nil || len(containerConfig.Ports) == 0 {
2626		return false, nil
2627	}
2628	ip, port, probeErr := s.resolveProbeEndpoint(ctx, containerID, containerConfig)
2629	if probeErr != nil || ip == "" || port <= 0 {
2630		log.Debug().Err(probeErr).Msg("readiness cascade: default HTTP probe skipped, could not resolve container endpoint")
2631		return false, nil
2632	}
2633	url := fmt.Sprintf("http://%s:%d/", ip, port)
2634	timeout := cfg.HTTPProbeTimeout
2635	if timeout == 0 {
2636		timeout = 60 * time.Second
2637	}
2638	log.Info().Str("url", url).Dur("timeout", timeout).Msg("readiness cascade: using default HTTP alive probe")
2639	return true, httpAliveProbe(ctx, url, timeout)
2640}
2641
2642// tryTCPProbe attempts a TCP probe if port info is available.
2643// Returns (true, err) if the probe was attempted, (false, nil) if skipped.
2644func (s *Service) tryTCPProbe(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) (bool, error) {
2645	log := zerowrap.FromCtx(ctx)
2646	if containerConfig == nil || len(containerConfig.Ports) == 0 {
2647		return false, nil
2648	}
2649	ip, port, probeErr := s.resolveProbeEndpoint(ctx, containerID, containerConfig)
2650	if probeErr != nil || ip == "" || port <= 0 {
2651		log.Debug().Err(probeErr).Msg("readiness cascade: TCP probe skipped, could not resolve container endpoint")
2652		return false, nil
2653	}
2654	addr := fmt.Sprintf("%s:%d", ip, port)
2655	timeout := cfg.TCPProbeTimeout
2656	if timeout == 0 {
2657		timeout = 30 * time.Second
2658	}
2659	log.Info().Str("addr", addr).Dur("timeout", timeout).Msg("readiness cascade: using TCP probe")
2660	return true, tcpProbe(ctx, addr, timeout)
2661}
2662
2663// resolveContainerEndpoint returns a host-reachable address for probing.
2664// In rootless podman/Docker setups, container internal IPs are not routable
2665// from the host. We use the host port binding (127.0.0.1:<mapped_port>)
2666// which is always reachable. Falls back to the container's internal IP
2667// only if no host port mapping exists (e.g. host-network mode).
2668func (s *Service) resolveProbeEndpoint(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig) (string, int, error) {
2669	log := zerowrap.FromCtx(ctx)
2670
2671	internalPort := 0
2672	if containerConfig != nil {
2673		if labels := containerConfig.Labels; labels != nil {
2674			for _, key := range []string{domain.LabelProxyPort, domain.LabelPort} {
2675				if portStr, ok := labels[key]; ok && portStr != "" {
2676					port, err := strconv.Atoi(portStr)
2677					if err == nil && port > 0 && port <= 65535 {
2678						internalPort = port
2679						break
2680					}
2681					log.Warn().Str("label", key).Str("port_value", portStr).Msg("invalid probe port label value")
2682				}
2683			}
2684		}
2685	}
2686
2687	if internalPort == 0 {
2688		_, resolvedPort, err := s.runtime.GetContainerNetworkInfo(ctx, containerID)
2689		if err != nil {
2690			return "", 0, err
2691		}
2692		internalPort = resolvedPort
2693	}
2694
2695	hostPort, hostErr := s.runtime.GetContainerPort(ctx, containerID, internalPort)
2696	if hostErr == nil && hostPort > 0 {
2697		log.Debug().
2698			Int("internal_port", internalPort).
2699			Int("host_port", hostPort).
2700			Msg("resolved container endpoint via host port binding")
2701		return "127.0.0.1", hostPort, nil
2702	}
2703
2704	ip, _, fallbackErr := s.runtime.GetContainerNetworkInfo(ctx, containerID)
2705	if fallbackErr != nil {
2706		return "", 0, fallbackErr
2707	}
2708	log.Debug().
2709		Str("ip", ip).
2710		Int("port", internalPort).
2711		Msg("resolved container endpoint via internal IP (no host port binding)")
2712	return ip, internalPort, nil
2713}
2714
2715// waitForReadyByDelay waits using the legacy running+delay strategy.
2716func (s *Service) waitForReadyByDelay(ctx context.Context, containerID string) error {
2717	log := zerowrap.FromCtx(ctx)
2718
2719	s.mu.RLock()
2720	cfg := s.config
2721	s.mu.RUnlock()
2722
2723	delay := cfg.ReadinessDelay
2724	if delay == 0 {
2725		delay = 5 * time.Second
2726	}
2727
2728	log.Debug().Dur("delay", delay).Msg("waiting for container readiness")
2729	select {
2730	case <-time.After(delay):
2731	case <-ctx.Done():
2732		return ctx.Err()
2733	}
2734
2735	// Verify still running after delay
2736	running, err := s.runtime.IsContainerRunning(ctx, containerID)
2737	if err != nil {
2738		return err
2739	}
2740	if running {
2741		return nil
2742	}
2743
2744	return s.waitForRecovery(ctx, containerID)
2745}
2746
2747func (s *Service) waitForHealthy(ctx context.Context, containerID string, timeout time.Duration) error {
2748	if timeout == 0 {
2749		timeout = 90 * time.Second
2750	}
2751	waitCtx, cancel := context.WithTimeout(ctx, timeout)
2752	defer cancel()
2753
2754	var lastStatus string
2755	for {
2756		status, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(waitCtx, containerID)
2757		if err != nil {
2758			return err
2759		}
2760		if !hasHealthcheck {
2761			return errors.New("no healthcheck detected")
2762		}
2763		// Treat empty health status as transitional startup. Some runtimes can
2764		// temporarily report an empty status before first probe results.
2765		if status == "" {
2766			status = "starting"
2767		}
2768		if status == "healthy" {
2769			return nil
2770		}
2771		lastStatus = status
2772
2773		select {
2774		case <-time.After(time.Second):
2775		case <-waitCtx.Done():
2776			return fmt.Errorf("container healthcheck timeout after %s (last status: %s)", timeout, lastStatus)
2777		}
2778	}
2779}
2780
2781// pollContainerRunning polls until the container is running (max 30 seconds).
2782func (s *Service) pollContainerRunning(ctx context.Context, containerID string) error {
2783	for i := 0; i < 30; i++ {
2784		running, err := s.runtime.IsContainerRunning(ctx, containerID)
2785		if err != nil {
2786			return err
2787		}
2788		if running {
2789			return nil
2790		}
2791		if i == 29 {
2792			return fmt.Errorf("container did not start within 30 seconds")
2793		}
2794		select {
2795		case <-time.After(time.Second):
2796		case <-ctx.Done():
2797			return ctx.Err()
2798		}
2799	}
2800	return fmt.Errorf("container did not start within 30 seconds")
2801}
2802
2803// waitForRecovery waits for a container to recover within the recovery window.
2804func (s *Service) waitForRecovery(ctx context.Context, containerID string) error {
2805	log := zerowrap.FromCtx(ctx)
2806	log.Warn().
2807		Dur("recovery_window", readinessRecoveryWindow).
2808		Msg("container not running after readiness delay, waiting for recovery")
2809
2810	deadline := time.Now().Add(readinessRecoveryWindow)
2811	for time.Now().Before(deadline) {
2812		select {
2813		case <-time.After(time.Second):
2814		case <-ctx.Done():
2815			return ctx.Err()
2816		}
2817
2818		running, err := s.runtime.IsContainerRunning(ctx, containerID)
2819		if err != nil {
2820			return err
2821		}
2822		if running {
2823			log.Info().Msg("container recovered during readiness recovery window")
2824			return nil
2825		}
2826	}
2827
2828	return fmt.Errorf("container not running after readiness delay and recovery window")
2829}
2830
2831// publishContainerDeployed publishes a container.deployed event.
2832func (s *Service) publishContainerDeployed(ctx context.Context, domainName, containerID string) {
2833	payload := &domain.ContainerEventPayload{
2834		ContainerID: containerID,
2835		Domain:      domainName,
2836		Action:      "deployed",
2837	}
2838
2839	if err := s.eventBus.Publish(domain.EventContainerDeployed, payload); err != nil {
2840		log := zerowrap.FromCtx(ctx)
2841		log.Warn().Err(err).Msg("failed to publish container deployed event")
2842	}
2843}
2844
2845// rewriteToRegistryDomain rewrites an image reference to use the configured registry domain.
2846// e.g., "myapp:latest" -> "registry.example.com/myapp:latest"
2847func rewriteToRegistryDomain(imageRef, registryDomain string) string {
2848	if registryDomain == "" {
2849		return imageRef
2850	}
2851
2852	prefix := registryDomain + "/"
2853	if strings.HasPrefix(imageRef, prefix) {
2854		return imageRef
2855	}
2856
2857	return prefix + imageRef
2858}
2859
2860// rewriteToLocalRegistry rewrites an image reference to use the local registry address.
2861// e.g., "registry.example.com/myapp:latest" -> "localhost:5000/myapp:latest"
2862func rewriteToLocalRegistry(imageRef, registryDomain string, registryPort int) string {
2863	if imageRef == "" {
2864		return imageRef
2865	}
2866
2867	localRegistry := fmt.Sprintf("localhost:%d", registryPort)
2868	localPrefix := localRegistry + "/"
2869	imageRef = strings.TrimPrefix(imageRef, localPrefix)
2870
2871	registryDomain = strings.TrimSuffix(registryDomain, "/")
2872	if registryDomain != "" {
2873		prefix := registryDomain + "/"
2874		imageRef = strings.TrimPrefix(imageRef, prefix)
2875	}
2876
2877	return localPrefix + imageRef
2878}