service.go RawHistoryFolder

1// Package container implements the container management use case.
2package container
3
4import (
5	"bufio"
6	"context"
7	"encoding/binary"
8	"errors"
9	"fmt"
10	"io"
11	"maps"
12	"slices"
13	"strconv"
14	"strings"
15	"sync"
16	"time"
17
18	"github.com/bnema/zerowrap"
19	"go.opentelemetry.io/otel"
20	"go.opentelemetry.io/otel/attribute"
21	"go.opentelemetry.io/otel/codes"
22	"go.opentelemetry.io/otel/metric"
23	"go.opentelemetry.io/otel/trace"
24
25	"github.com/bnema/gordon/internal/adapters/out/telemetry"
26	"github.com/bnema/gordon/internal/boundaries/out"
27	"github.com/bnema/gordon/internal/domain"
28)
29
30// Config holds configuration needed by the container service.
31type Config struct {
32	RegistryAuthEnabled        bool
33	RegistryDomain             string
34	RegistryPort               int
35	ServiceTokenUsername       string
36	ServiceToken               string
37	InternalRegistryUsername   string
38	InternalRegistryPassword   string
39	PullPolicy                 string
40	VolumeAutoCreate           bool
41	VolumePrefix               string
42	VolumePreserve             bool
43	NetworkIsolation           bool
44	NetworkPrefix              string
45	NetworkGroups              map[string][]string
46	Attachments                map[string][]string
47	ReadinessDelay             time.Duration // Delay after container starts before considering it ready
48	ReadinessMode              string        // Readiness strategy: auto, docker-health, delay
49	HealthTimeout              time.Duration // Max wait for health-based readiness
50	DrainDelay                 time.Duration // Grace period after cache invalidation before stopping old container
51	DrainDelayConfigured       bool          // True when deploy.drain_delay was explicitly configured
52	DrainMode                  string        // Drain strategy: auto, inflight, delay
53	DrainTimeout               time.Duration // Max wait for in-flight requests to drain
54	StabilizationDelay         time.Duration // Post-switch monitoring window (default 2s)
55	TCPProbeTimeout            time.Duration // TCP probe timeout (default 30s)
56	HTTPProbeTimeout           time.Duration // HTTP probe timeout (default 60s)
57	AttachmentReadinessTimeout time.Duration // Max wait for attachment readiness (default 30s)
58	DefaultMemoryLimit         int64         // Default memory limit in bytes for containers (0 = no limit)
59	DefaultNanoCPUs            int64         // Default CPU quota in nanoseconds for containers (0 = no limit)
60	DefaultPidsLimit           int64         // Default max PIDs for containers (0 = no limit)
61}
62
63var tracer = otel.Tracer("gordon.container")
64
65const (
66	PullPolicyAlways       = "always"
67	PullPolicyIfNotPresent = "if-not-present"
68	PullPolicyIfTagChanged = "if-tag-changed"
69
70	// readinessRecoveryWindow is an additional grace window used when a container
71	// is briefly not running at the end of readiness delay. This avoids false
72	// negatives during short startup flaps.
73	readinessRecoveryWindow = 30 * time.Second
74	internalPullMaxAttempts = 3
75	deployFailureLogLimit   = 20
76)
77
78// Service implements the ContainerService interface.
79type Service struct {
80	runtime          out.ContainerRuntime
81	envLoader        out.EnvLoader
82	eventBus         out.EventPublisher
83	logWriter        out.ContainerLogWriter
84	cacheInvalidator out.ProxyCacheInvalidator
85	drainWaiter      out.ProxyDrainWaiter
86	config           Config
87	configProvider   AttachmentConfigProvider // live config reads for attachments/networks (may be nil)
88	metrics          *telemetry.Metrics
89	containers       map[string]*domain.Container
90	attachments      map[string][]string // ownerDomain → []containerIDs
91	managedCount     int64               // tracks UpDownCounter value for delta computation
92	mu               sync.RWMutex
93	deployMu         sync.Map       // per-domain deploy locks (domain → *domainDeployLock)
94	cleanupWg        sync.WaitGroup // tracks background old-container cleanup goroutines
95	monitor          *Monitor
96}
97
98// domainDeployLock is a context-aware mutex using a buffered channel.
99// It allows Lock() to be interrupted by context cancellation.
100type domainDeployLock struct {
101	ch chan struct{}
102}
103
104// newDomainDeployLock creates a new lock that is initially unlocked.
105func newDomainDeployLock() *domainDeployLock {
106	l := &domainDeployLock{ch: make(chan struct{}, 1)}
107	l.ch <- struct{}{}
108	return l
109}
110
111// Lock acquires the lock or returns an error if the context is cancelled.
112func (l *domainDeployLock) Lock(ctx context.Context) error {
113	// Fast path: honor already-canceled contexts before blocking.
114	if err := ctx.Err(); err != nil {
115		return err
116	}
117	select {
118	case <-ctx.Done():
119		return ctx.Err()
120	case <-l.ch:
121		// Re-check context after acquisition: if canceled while we were
122		// in the select, release the token so the lock isn't consumed.
123		if err := ctx.Err(); err != nil {
124			l.ch <- struct{}{}
125			return err
126		}
127		return nil
128	}
129}
130
131// Unlock releases the lock.
132func (l *domainDeployLock) Unlock() {
133	select {
134	case l.ch <- struct{}{}:
135	default:
136		// This should never happen if Lock/Unlock are used correctly.
137		// Panic to detect double-unlocks or other misuse early.
138		panic("domainDeployLock: double unlock detected")
139	}
140}
141
142// NewService creates a new container service.
143func NewService(
144	runtime out.ContainerRuntime,
145	envLoader out.EnvLoader,
146	eventBus out.EventPublisher,
147	logWriter out.ContainerLogWriter,
148	config Config,
149	configProvider AttachmentConfigProvider,
150) *Service {
151	return &Service{
152		runtime:        runtime,
153		envLoader:      envLoader,
154		eventBus:       eventBus,
155		logWriter:      logWriter,
156		config:         config,
157		configProvider: configProvider,
158		containers:     make(map[string]*domain.Container),
159		attachments:    make(map[string][]string),
160	}
161}
162
163// SetMetrics sets the telemetry metrics for the container service.
164func (s *Service) SetMetrics(m *telemetry.Metrics) {
165	s.metrics = m
166}
167
168// SetProxyCacheInvalidator sets the proxy cache invalidator for synchronous
169// cache invalidation during zero-downtime deployments. This must be called
170// after construction because the proxy service is created after the container
171// service during application initialization.
172func (s *Service) SetProxyCacheInvalidator(inv out.ProxyCacheInvalidator) {
173	s.mu.Lock()
174	s.cacheInvalidator = inv
175	s.mu.Unlock()
176}
177
178// SetProxyDrainWaiter sets the proxy in-flight drain waiter used during
179// zero-downtime replacement before stopping the old container.
180func (s *Service) SetProxyDrainWaiter(waiter out.ProxyDrainWaiter) {
181	s.mu.Lock()
182	s.drainWaiter = waiter
183	s.mu.Unlock()
184}
185
186// domainDeployMu returns a per-domain lock for serializing deploy operations.
187func (s *Service) domainDeployMu(domain string) *domainDeployLock {
188	v, _ := s.deployMu.LoadOrStore(domain, newDomainDeployLock())
189	// Type assertion is safe: LoadOrStore always stores *domainDeployLock values,
190	// and any concurrent LoadOrStore calls will also store *domainDeployLock.
191	return v.(*domainDeployLock)
192}
193
194// acquireDomainDeployLock acquires the per-domain deploy lock with context cancellation support.
195// Returns an unlock function that must be called to release the lock.
196func (s *Service) acquireDomainDeployLock(ctx context.Context, domain string) (func(), error) {
197	lock := s.domainDeployMu(domain)
198	if err := lock.Lock(ctx); err != nil {
199		return nil, err
200	}
201	return func() { lock.Unlock() }, nil
202}
203
204// buildContainerConfig constructs the container configuration for deployment.
205func (s *Service) deploymentContainerName(containerDomain string, existing *domain.Container) string {
206	canonicalName := fmt.Sprintf("gordon-%s", containerDomain)
207	if existing == nil {
208		return canonicalName
209	}
210
211	// Alternate temporary names if the tracked container was left with a temp suffix.
212	// This prevents name collisions after interrupted zero-downtime deploys.
213	newName := canonicalName + "-new"
214	nextName := canonicalName + "-next"
215	switch existing.Name {
216	case newName:
217		return nextName
218	case nextName:
219		return newName
220	default:
221		return newName
222	}
223}
224
225// containerConfigInput groups the parameters for building a container configuration.
226type containerConfigInput struct {
227	Domain       string
228	Image        string
229	ImageRef     string
230	ExposedPorts []int
231	EnvVars      []string
232	EnvHash      string
233	Volumes      map[string]string
234	NetworkName  string
235	ImageLabels  map[string]string
236	Existing     *domain.Container
237}
238
239func (s *Service) buildContainerConfig(in containerConfigInput) *domain.ContainerConfig {
240	s.mu.RLock()
241	cfg := s.config
242	s.mu.RUnlock()
243
244	containerName := s.deploymentContainerName(in.Domain, in.Existing)
245	labels := map[string]string{
246		domain.LabelDomain:  in.Domain,
247		domain.LabelEnvHash: in.EnvHash,
248		domain.LabelImage:   in.Image,
249		domain.LabelManaged: "true",
250		domain.LabelRoute:   in.Domain,
251	}
252
253	// Propagate proxy/port labels from image so readiness probes can find them
254	for _, key := range []string{domain.LabelProxyPort, domain.LabelPort, domain.LabelHealth} {
255		if v, ok := in.ImageLabels[key]; ok && v != "" {
256			labels[key] = v
257		}
258	}
259
260	// Ensure the label-specified proxy port is included in exposed ports
261	// so Docker creates a host port binding for it.
262	ports := in.ExposedPorts
263	if portStr, ok := labels[domain.LabelProxyPort]; ok {
264		if port, err := strconv.Atoi(portStr); err == nil && port > 0 && port <= 65535 {
265			if !slices.Contains(ports, port) {
266				ports = append(ports, port)
267			}
268		}
269	}
270
271	return &domain.ContainerConfig{
272		Image:       in.ImageRef,
273		Name:        containerName,
274		Ports:       ports,
275		Env:         in.EnvVars,
276		Volumes:     in.Volumes,
277		NetworkMode: in.NetworkName,
278		Hostname:    in.Domain,
279		Labels:      labels,
280		AutoRemove:  false,
281		MemoryLimit: cfg.DefaultMemoryLimit,
282		NanoCPUs:    cfg.DefaultNanoCPUs,
283		PidsLimit:   cfg.DefaultPidsLimit,
284	}
285}
286
287// Deploy creates and starts a container for the given route.
288// Implements zero-downtime deployment: new container starts before old one stops.
289func (s *Service) Deploy(ctx context.Context, route domain.Route) (*domain.Container, error) {
290	ctx, span := tracer.Start(ctx, "container.deploy",
291		trace.WithAttributes(
292			attribute.String("domain", route.Domain),
293			attribute.String("image", route.Image),
294		))
295	defer span.End()
296
297	deployStart := time.Now()
298
299	// Serialize deploys for the same domain to prevent race conditions
300	// (e.g. multiple image.pushed events + explicit deploy call from CLI).
301	unlock, err := s.acquireDomainDeployLock(ctx, route.Domain)
302	if err != nil {
303		return nil, err
304	}
305	defer unlock()
306
307	// Enrich context with use case fields for all downstream logs
308	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
309		zerowrap.FieldLayer:   "usecase",
310		zerowrap.FieldUseCase: "Deploy",
311		"domain":              route.Domain,
312	})
313	log := zerowrap.FromCtx(ctx)
314
315	// Record deploy metrics and trace status
316	defer s.recordDeployMetrics(ctx, span, route, deployStart, &err)
317
318	existing, hasExisting := s.resolveExistingContainer(ctx, route.Domain)
319
320	resources, err := s.prepareDeployResources(ctx, route, existing)
321	if err != nil {
322		if deployErr := s.wrapPullDeployFailure(route, existing, err); deployErr != nil {
323			err = deployErr
324			return nil, err
325		}
326		return nil, err
327	}
328
329	// Skip redundant deploy: if the existing container is already running
330	// the exact same image (by Docker image ID), return it immediately.
331	// This prevents the double-deploy caused by the event-based deploy
332	// (triggered by image.pushed) racing with the explicit CLI deploy call.
333	if hasExisting {
334		existingForSkip := existing
335		if existingForSkip.ImageID == "" && existingForSkip.Image != "" && normalizeImageRef(existingForSkip.Image) == normalizeImageRef(route.Image) {
336			existingForSkip = s.containerForRedundantCheck(ctx, existingForSkip)
337		}
338		if existingForSkip.ImageID != "" {
339			if skip, container := s.skipRedundantDeploy(ctx, existingForSkip, resources.actualImageRef, resources.envHash); skip {
340				return container, nil
341			}
342		}
343	}
344
345	newContainer, err := s.createStartedContainer(ctx, route, existing, resources)
346	if err != nil {
347		return nil, err
348	}
349
350	invalidated := s.activateDeployedContainer(ctx, route.Domain, newContainer)
351
352	// Post-switch stabilization: verify new container stays running
353	if hasExisting {
354		stable, stabilizeErr := s.stabilizeNewContainer(ctx, route.Domain, newContainer, existing)
355		if stabilizeErr != nil {
356			// Both old and new containers are dead; assign to named return so
357			// the deferred recordDeployMetrics and span see the failure.
358			err = stabilizeErr
359			return nil, err
360		}
361		if !stable {
362			// Rollback performed — old container is restored
363			return existing, nil
364		}
365	}
366
367	// Finalize old container in the background — the new container is already
368	// serving traffic, so there's no reason to block the deploy response while
369	// waiting for the old container to stop (which can take 20s if the app
370	// doesn't handle SIGTERM).
371	s.cleanupWg.Add(1)
372	go func() {
373		defer s.cleanupWg.Done()
374		s.finalizePreviousContainer(context.WithoutCancel(ctx), route.Domain, existing, hasExisting, invalidated, newContainer.ID)
375	}()
376
377	// Start container log collection (non-blocking, errors don't fail deployment)
378	s.startLogCollection(ctx, newContainer.ID, route.Domain)
379
380	log.Info().
381		Str("image", route.Image).
382		Str(zerowrap.FieldEntityID, newContainer.ID).
383		Ints("ports", newContainer.Ports).
384		Str("network", resources.networkName).
385		Bool("zero_downtime", hasExisting).
386		Msg("container deployed successfully")
387
388	return newContainer, nil
389}
390
391// recordDeployMetrics records span error status and deploy metrics at the end of a Deploy call.
392// It is called via defer and receives a pointer to the named return error so it can observe
393// the final error value after all deferred functions have run.
394func (s *Service) recordDeployMetrics(ctx context.Context, span trace.Span, route domain.Route, start time.Time, errPtr *error) {
395	err := *errPtr
396	if err != nil {
397		span.RecordError(err)
398		span.SetStatus(codes.Error, err.Error())
399	}
400	if s.metrics == nil {
401		return
402	}
403	attrs := metric.WithAttributes(
404		attribute.String("domain", route.Domain),
405		attribute.String("image", route.Image),
406	)
407	s.metrics.DeployTotal.Add(ctx, 1, attrs)
408	s.metrics.DeployDuration.Record(ctx, time.Since(start).Seconds(), attrs)
409	if err != nil {
410		s.metrics.DeployErrors.Add(ctx, 1, attrs)
411	}
412}
413
414// skipRedundantDeploy checks whether the existing container is already running
415// the same image (by Docker image ID) as the one we are about to deploy.
416// When a push triggers both an event-based deploy and an explicit CLI deploy,
417// the second one arrives after the first has already completed; this avoids
418// a full redundant create-start-readiness cycle.
419func (s *Service) skipRedundantDeploy(ctx context.Context, existing *domain.Container, actualImageRef, envHash string) (bool, *domain.Container) {
420	log := zerowrap.FromCtx(ctx)
421
422	newImageID, err := s.runtime.GetImageID(ctx, actualImageRef)
423	if err != nil {
424		log.Debug().Err(err).Msg("cannot resolve image ID for redundancy check, proceeding with deploy")
425		return false, nil
426	}
427
428	if existing.ImageID != newImageID {
429		return false, nil
430	}
431
432	// Verify the container is actually still running before skipping.
433	running, err := s.runtime.IsContainerRunning(ctx, existing.ID)
434	if err != nil || !running {
435		return false, nil
436	}
437
438	if existing.Labels != nil {
439		existingEnvHash, hasEnvHash := existing.Labels[domain.LabelEnvHash]
440		if !hasEnvHash || existingEnvHash != envHash {
441			log.Info().
442				Str("container_id", existing.ID).
443				Bool("has_env_hash", hasEnvHash).
444				Msg("env changed, proceeding with deploy despite same image")
445			return false, nil
446		}
447	} else {
448		return false, nil
449	}
450
451	log.Info().
452		Str("container_id", existing.ID).
453		Str("image_id", newImageID).
454		Msg("skipping redundant deploy: container already running this image")
455
456	return true, existing
457}
458
459func (s *Service) containerForRedundantCheck(ctx context.Context, existing *domain.Container) *domain.Container {
460	if existing == nil || existing.ImageID != "" || existing.ID == "" {
461		return existing
462	}
463
464	log := zerowrap.FromCtx(ctx)
465	inspected, err := s.runtime.InspectContainer(ctx, existing.ID)
466	if err != nil {
467		log.Debug().Err(err).Str("container_id", existing.ID).Msg("cannot inspect existing container for redundancy check")
468		return existing
469	}
470	if inspected == nil || inspected.ImageID == "" {
471		return existing
472	}
473
474	existingCopy := *existing
475	existingCopy.ImageID = inspected.ImageID
476	return &existingCopy
477}
478
479type deployResources struct {
480	networkName    string
481	actualImageRef string
482	exposedPorts   []int
483	imageLabels    map[string]string
484	envVars        []string
485	envHash        string
486	volumes        map[string]string
487}
488
489// resolveExistingContainer returns the currently running container for a domain.
490// It first checks the in-memory map (fast path). If not found, it queries the
491// runtime for a running container with matching name and managed label. This
492// handles cases where Gordon restarted and in-memory state is stale.
493//
494// The slow path checks canonical, -new, and -next names because a previous
495// deploy may have been interrupted (e.g. eventbus timeout, Gordon restart)
496// leaving the active container under a temp name. Canonical is preferred when
497// multiple matches exist.
498func (s *Service) resolveExistingContainer(ctx context.Context, domainName string) (*domain.Container, bool) {
499	// Fast path: check in-memory state
500	s.mu.RLock()
501	container, ok := s.containers[domainName]
502	s.mu.RUnlock()
503	if ok {
504		return container, true
505	}
506
507	// Slow path: query runtime for running containers
508	log := zerowrap.FromCtx(ctx)
509	canonicalName := fmt.Sprintf("gordon-%s", domainName)
510	candidateNames := map[string]bool{
511		canonicalName:           true,
512		canonicalName + "-new":  true,
513		canonicalName + "-next": true,
514	}
515
516	running, err := s.runtime.ListContainers(ctx, false)
517	if err != nil {
518		log.Warn().Err(err).Msg("failed to list running containers for existing container resolution")
519		return nil, false
520	}
521
522	// Prefer canonical name; fall back to any running managed temp container.
523	var best *domain.Container
524	for _, c := range running {
525		if !candidateNames[c.Name] || c.Labels[domain.LabelManaged] != "true" {
526			continue
527		}
528		if best == nil || c.Name == canonicalName {
529			best = c
530		}
531	}
532
533	if best != nil {
534		log.Info().
535			Str("container_id", best.ID).
536			Str("container_name", best.Name).
537			Msg("resolved existing container from runtime (in-memory state was stale)")
538
539		// Update in-memory state so subsequent lookups are fast
540		s.mu.Lock()
541		if _, alreadyTracked := s.containers[domainName]; !alreadyTracked {
542			s.managedCount++
543		}
544		s.containers[domainName] = best
545		s.mu.Unlock()
546
547		return best, true
548	}
549
550	return nil, false
551}
552
553func (s *Service) prepareDeployResources(ctx context.Context, route domain.Route, existing *domain.Container) (*deployResources, error) {
554	log := zerowrap.FromCtx(ctx)
555
556	existingID := ""
557	if existing != nil {
558		existingID = existing.ID
559	}
560	if err := s.cleanupOrphanedContainers(ctx, route.Domain, existingID); err != nil {
561		log.WrapErr(err, "failed to cleanup orphaned containers")
562	}
563
564	imageRef := s.buildImageRef(route.Image)
565	actualImageRef, err := s.ensureImage(ctx, imageRef)
566	if err != nil {
567		return nil, err
568	}
569
570	networkName := s.getNetworkForApp(route.Domain)
571	if err := s.createNetworkIfNeeded(ctx, networkName); err != nil {
572		return nil, log.WrapErr(err, "failed to create network")
573	}
574	if err := s.deployAttachments(ctx, route.Domain, networkName); err != nil {
575		return nil, log.WrapErr(err, "failed to deploy attachments")
576	}
577
578	exposedPorts, err := s.runtime.GetImageExposedPorts(ctx, actualImageRef)
579	if err != nil {
580		log.WrapErr(err, "failed to get exposed ports, using defaults")
581		exposedPorts = []int{80, 8080, 3000}
582	}
583
584	imageLabels, err := s.runtime.GetImageLabels(ctx, actualImageRef)
585	if err != nil {
586		log.Warn().Err(err).Msg("failed to get image labels, skipping label propagation")
587		imageLabels = nil
588	}
589
590	envVars, err := s.loadEnvironment(ctx, route.Env, route.Domain, actualImageRef)
591	if err != nil {
592		return nil, err
593	}
594	envHash := hashEnvironment(envVars)
595
596	volumes, err := s.setupVolumes(ctx, route.Domain, actualImageRef)
597	if err != nil {
598		return nil, err
599	}
600
601	return &deployResources{
602		networkName:    networkName,
603		actualImageRef: actualImageRef,
604		exposedPorts:   exposedPorts,
605		imageLabels:    imageLabels,
606		envVars:        envVars,
607		envHash:        envHash,
608		volumes:        volumes,
609	}, nil
610}
611
612func (s *Service) createStartedContainer(ctx context.Context, route domain.Route, existing *domain.Container, resources *deployResources) (*domain.Container, error) {
613	ctx, span := tracer.Start(ctx, "container.create_and_start")
614	defer span.End()
615
616	log := zerowrap.FromCtx(ctx)
617
618	containerConfig := s.buildContainerConfig(containerConfigInput{
619		Domain:       route.Domain,
620		Image:        route.Image,
621		ImageRef:     resources.actualImageRef,
622		ExposedPorts: resources.exposedPorts,
623		EnvVars:      resources.envVars,
624		EnvHash:      resources.envHash,
625		Volumes:      resources.volumes,
626		NetworkName:  resources.networkName,
627		ImageLabels:  resources.imageLabels,
628		Existing:     existing,
629	})
630
631	newContainer, err := s.runtime.CreateContainer(ctx, containerConfig)
632	if err != nil {
633		return nil, log.WrapErr(err, "failed to create container")
634	}
635	if err := s.runtime.StartContainer(ctx, newContainer.ID); err != nil {
636		s.runtime.RemoveContainer(ctx, newContainer.ID, true)
637		return nil, log.WrapErr(err, "failed to start container")
638	}
639	if !domain.IsSkipReadiness(ctx) {
640		if err := s.waitForReady(ctx, newContainer.ID, containerConfig); err != nil {
641			deployErr := s.newDeployFailure(newContainer, "container failed readiness check", err, s.captureRecentContainerLogs(ctx, newContainer.ID))
642			s.cleanupFailedContainer(ctx, newContainer.ID)
643			return nil, deployErr
644		}
645	}
646
647	inspected, err := s.runtime.InspectContainer(ctx, newContainer.ID)
648	if err != nil {
649		s.cleanupFailedContainer(ctx, newContainer.ID)
650		return nil, log.WrapErr(err, "failed to inspect started container")
651	}
652
653	return inspected, nil
654}
655
656func (s *Service) newDeployFailure(container *domain.Container, cause string, err error, logs []string) error {
657	deployErr := &domain.DeployFailureError{
658		Summary: "failed to deploy",
659		Cause:   cause,
660		Hint:    inferDeployFailureHint(err),
661		Logs:    logs,
662		Err:     err,
663	}
664	if container != nil {
665		deployErr.ContainerName = container.Name
666		deployErr.ContainerID = container.ID
667	}
668
669	return deployErr
670}
671
672func (s *Service) wrapPullDeployFailure(route domain.Route, existing *domain.Container, err error) error {
673	if !isPullFailure(err) {
674		return nil
675	}
676
677	return s.newDeployFailure(&domain.Container{Name: s.deploymentContainerName(route.Domain, existing)}, "failed to pull image", err, nil)
678}
679
680func (s *Service) captureRecentContainerLogs(ctx context.Context, containerID string) []string {
681	if ctx.Err() != nil {
682		return nil
683	}
684
685	logStream, err := s.runtime.GetContainerLogs(ctx, containerID, false)
686	if err != nil {
687		return nil
688	}
689	defer func() {
690		_ = logStream.Close()
691	}()
692
693	logs, err := parseDockerLogLines(logStream, deployFailureLogLimit)
694	if err != nil {
695		return nil
696	}
697
698	return logs
699}
700
701func parseDockerLogLines(r io.Reader, maxLines int) ([]string, error) {
702	reader := bufio.NewReader(r)
703	collector := newLogLineCollector(maxLines)
704
705	header, err := reader.Peek(8)
706	if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, bufio.ErrBufferFull) {
707		return nil, err
708	}
709
710	if len(header) >= 8 && looksLikeDockerLogStream(header) {
711		return parseDockerFramedLogLines(reader, collector)
712	}
713
714	return parsePlainLogLines(reader, collector)
715}
716
717func parseDockerFramedLogLines(reader *bufio.Reader, collector *logLineCollector) ([]string, error) {
718	for {
719		frameHeader := make([]byte, 8)
720		if _, err := io.ReadFull(reader, frameHeader); err != nil {
721			if isLogStreamEOF(err) {
722				collector.flush()
723				return collector.lines, nil
724			}
725			return nil, err
726		}
727
728		size := int(binary.BigEndian.Uint32(frameHeader[4:8]))
729		if size <= 0 {
730			continue
731		}
732
733		payload, err := readLogPayload(reader, size)
734		if err != nil {
735			collector.addChunk(string(payload))
736			collector.flush()
737			if isLogStreamEOF(err) {
738				return collector.lines, nil
739			}
740			return nil, err
741		}
742
743		collector.addChunk(string(payload))
744	}
745}
746
747func parsePlainLogLines(reader *bufio.Reader, collector *logLineCollector) ([]string, error) {
748
749	scanner := bufio.NewScanner(reader)
750	for scanner.Scan() {
751		collector.addLine(scanner.Text())
752	}
753	if err := scanner.Err(); err != nil {
754		return nil, err
755	}
756
757	collector.flush()
758	return collector.lines, nil
759}
760
761func readLogPayload(reader *bufio.Reader, size int) ([]byte, error) {
762	payload := make([]byte, size)
763	_, err := io.ReadFull(reader, payload)
764	return payload, err
765}
766
767func isLogStreamEOF(err error) bool {
768	return errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF)
769}
770
771func looksLikeDockerLogStream(data []byte) bool {
772	return len(data) >= 8 && data[1] == 0 && data[2] == 0 && data[3] == 0
773}
774
775type logLineCollector struct {
776	lines    []string
777	pending  string
778	maxLines int
779}
780
781func newLogLineCollector(maxLines int) *logLineCollector {
782	return &logLineCollector{maxLines: maxLines}
783}
784
785func (c *logLineCollector) addChunk(chunk string) {
786	for _, part := range strings.SplitAfter(chunk, "\n") {
787		c.pending += part
788		if strings.HasSuffix(part, "\n") {
789			c.flush()
790		}
791	}
792}
793
794func (c *logLineCollector) addLine(line string) {
795	line = strings.TrimRight(line, "\r")
796	if line == "" {
797		return
798	}
799	c.lines = append(c.lines, line)
800	if c.maxLines > 0 && len(c.lines) > c.maxLines {
801		c.lines = c.lines[len(c.lines)-c.maxLines:]
802	}
803}
804
805func (c *logLineCollector) flush() {
806	if c.pending == "" {
807		return
808	}
809	c.addLine(strings.TrimRight(c.pending, "\r\n"))
810	c.pending = ""
811}
812
813func inferDeployFailureHint(err error) string {
814	if err == nil {
815		return ""
816	}
817
818	msg := strings.ToLower(err.Error())
819	switch {
820	case strings.Contains(msg, "healthcheck timeout"):
821		return "check startup logs and confirm the health endpoint becomes ready within the configured timeout"
822	case strings.Contains(msg, "unhealthy"):
823		return "inspect recent app logs and verify the healthcheck command reflects actual readiness"
824	case strings.Contains(msg, "no healthcheck detected"):
825		return "add a Docker healthcheck or switch readiness mode"
826	case isPullFailure(err), strings.Contains(msg, "access denied"), strings.Contains(msg, "unauthorized"):
827		return "verify registry auth and confirm the image exists at the requested tag"
828	default:
829		return ""
830	}
831}
832
833func isPullFailure(err error) bool {
834	if err == nil {
835		return false
836	}
837
838	if errors.Is(err, domain.ErrImagePullFailed) {
839		return true
840	}
841
842	msg := strings.ToLower(err.Error())
843	return strings.Contains(msg, "failed to pull image") ||
844		strings.Contains(msg, "pull access denied")
845}
846
847func (s *Service) activateDeployedContainer(ctx context.Context, domainName string, container *domain.Container) bool {
848	s.mu.Lock()
849	_, wasTracked := s.containers[domainName]
850	s.containers[domainName] = container
851	if !wasTracked {
852		s.managedCount++
853	}
854	s.mu.Unlock()
855
856	// Track managed container count (only increment for new domains, not replacements)
857	if !wasTracked && s.metrics != nil {
858		s.metrics.ManagedContainers.Add(ctx, 1)
859	}
860
861	s.publishContainerDeployed(ctx, domainName, container.ID)
862
863	s.mu.RLock()
864	inv := s.cacheInvalidator
865	s.mu.RUnlock()
866	if inv != nil {
867		inv.InvalidateTarget(ctx, domainName)
868		return true
869	}
870
871	return false
872}
873
874// stabilizeNewContainer monitors the new container briefly after traffic switch.
875// If it crashes during this window, rolls back to old container.
876// Returns true if stabilization succeeded, false if rollback was performed.
877// Returns an error only when both new and old containers are dead.
878func (s *Service) stabilizeNewContainer(ctx context.Context, domainName string, newContainer, oldContainer *domain.Container) (bool, error) {
879	log := zerowrap.FromCtx(ctx)
880
881	if oldContainer == nil {
882		return true, nil
883	}
884
885	s.mu.RLock()
886	delay := s.config.StabilizationDelay
887	s.mu.RUnlock()
888	if delay == 0 {
889		delay = 2 * time.Second
890	}
891
892	// Brief stabilization: verify new container is still running after delay
893	select {
894	case <-time.After(delay):
895	case <-ctx.Done():
896		return true, nil
897	}
898
899	running, err := s.runtime.IsContainerRunning(ctx, newContainer.ID)
900	if err != nil || !running {
901		log.Error().
902			Str("new_container", newContainer.ID).
903			Str("old_container", oldContainer.ID).
904			Msg("new container crashed during stabilization, rolling back to old")
905
906		// Verify old container is still running before restoring it.
907		// If both old and new are dead (e.g., OOM), returning a dead
908		// container as "existing" would leave the domain in a broken state.
909		oldRunning, oldErr := s.runtime.IsContainerRunning(ctx, oldContainer.ID)
910		if oldErr != nil || !oldRunning {
911			log.Error().
912				Str("old_container", oldContainer.ID).
913				Msg("old container is also not running, cannot rollback")
914
915			// Cleanup failed new container
916			if stopErr := s.runtime.StopContainer(ctx, newContainer.ID); stopErr != nil {
917				log.WrapErrWithFields(stopErr, "failed to stop failed new container during rollback", map[string]any{zerowrap.FieldEntityID: newContainer.ID})
918			}
919			if removeErr := s.runtime.RemoveContainer(ctx, newContainer.ID, true); removeErr != nil {
920				log.WrapErrWithFields(removeErr, "failed to remove failed new container during rollback", map[string]any{zerowrap.FieldEntityID: newContainer.ID})
921			}
922
923			return false, fmt.Errorf("stabilization failed: new container crashed and old container %s is also not running", oldContainer.ID)
924		}
925
926		// Rollback: restore old container as tracked
927		s.mu.Lock()
928		s.containers[domainName] = oldContainer
929		s.mu.Unlock()
930
931		// Re-invalidate proxy cache to point back to old
932		s.mu.RLock()
933		inv := s.cacheInvalidator
934		s.mu.RUnlock()
935		if inv != nil {
936			inv.InvalidateTarget(ctx, domainName)
937		}
938
939		// Cleanup failed new container
940		if stopErr := s.runtime.StopContainer(ctx, newContainer.ID); stopErr != nil {
941			log.WrapErrWithFields(stopErr, "failed to stop failed new container during rollback", map[string]any{zerowrap.FieldEntityID: newContainer.ID})
942		}
943		if removeErr := s.runtime.RemoveContainer(ctx, newContainer.ID, true); removeErr != nil {
944			log.WrapErrWithFields(removeErr, "failed to remove failed new container during rollback", map[string]any{zerowrap.FieldEntityID: newContainer.ID})
945		}
946
947		return false, nil
948	}
949
950	return true, nil
951}
952
953func (s *Service) finalizePreviousContainer(ctx context.Context, domainName string, existing *domain.Container, hasExisting, invalidated bool, newContainerID string) {
954	if !hasExisting {
955		return
956	}
957
958	if invalidated {
959		s.waitForDrain(ctx, existing.ID)
960	}
961
962	s.cleanupOldContainer(ctx, existing, newContainerID, domainName)
963}
964
965func (s *Service) waitForDrain(ctx context.Context, oldContainerID string) {
966	log := zerowrap.FromCtx(ctx)
967
968	s.mu.RLock()
969	cfg := s.config
970	waiter := s.drainWaiter
971	s.mu.RUnlock()
972
973	drainMode := cfg.DrainMode
974	if drainMode == "" {
975		drainMode = "delay"
976	}
977
978	shouldUseInFlight := (drainMode == "inflight" || drainMode == "auto") && waiter != nil
979	if shouldUseInFlight {
980		timeout := cfg.DrainTimeout
981		if timeout == 0 {
982			timeout = 30 * time.Second
983		}
984		drained := waiter.WaitForNoInFlight(ctx, oldContainerID, timeout)
985		if !drained {
986			log.Warn().
987				Str("old_container_id", oldContainerID).
988				Dur("drain_timeout", timeout).
989				Msg("drain wait timed out; old container may still have in-flight traffic")
990		}
991		return
992	}
993
994	drainDelay := 2 * time.Second
995	if cfg.DrainDelayConfigured {
996		drainDelay = cfg.DrainDelay
997	}
998	if drainDelay <= 0 {
999		return
1000	}
1001	select {
1002	case <-time.After(drainDelay):
1003	case <-ctx.Done():
1004	}
1005}
1006
1007// restartContainerWithRecovery restarts a container, reconciling stale state if needed.
1008// It returns the (possibly refreshed) container and updated attachment IDs.
1009func (s *Service) restartContainerWithRecovery(ctx context.Context, domainName string, container *domain.Container, attachmentIDs []string) (*domain.Container, []string, error) {
1010	log := zerowrap.FromCtx(ctx)
1011
1012	if err := s.runtime.RestartContainer(ctx, container.ID); err != nil {
1013		if !isContainerNotFoundError(err) {
1014			return nil, nil, log.WrapErr(err, "failed to restart container")
1015		}
1016
1017		// In-memory container ID can become stale after external runtime changes.
1018		// Re-sync and retry once with the latest tracked container.
1019		log.Warn().
1020			Err(err).
1021			Str(zerowrap.FieldEntityID, container.ID).
1022			Msg("tracked container missing during restart, attempting state reconciliation")
1023
1024		if syncErr := s.SyncContainers(ctx); syncErr != nil {
1025			log.Warn().Err(syncErr).Msg("failed to sync container state during restart recovery")
1026		}
1027
1028		s.mu.RLock()
1029		refreshed, refreshedExists := s.containers[domainName]
1030		attachmentIDs = append([]string{}, s.attachments[domainName]...)
1031		s.mu.RUnlock()
1032
1033		if !refreshedExists || refreshed == nil {
1034			return nil, nil, domain.ErrContainerNotFound
1035		}
1036		if err := s.runtime.RestartContainer(ctx, refreshed.ID); err != nil {
1037			return nil, nil, log.WrapErr(err, "failed to restart container after state reconciliation")
1038		}
1039		return refreshed, attachmentIDs, nil
1040	}
1041
1042	return container, attachmentIDs, nil
1043}
1044
1045// Restart restarts a running container for the given domain.
1046func (s *Service) Restart(ctx context.Context, domainName string, withAttachments bool) error {
1047	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1048		zerowrap.FieldLayer:   "usecase",
1049		zerowrap.FieldUseCase: "Restart",
1050		"domain":              domainName,
1051	})
1052	log := zerowrap.FromCtx(ctx)
1053
1054	s.mu.RLock()
1055	container, exists := s.containers[domainName]
1056	attachments := s.attachments[domainName]
1057	var attachmentIDs []string
1058	if len(attachments) > 0 {
1059		attachmentIDs = make([]string, len(attachments))
1060		copy(attachmentIDs, attachments)
1061	}
1062	s.mu.RUnlock()
1063
1064	if !exists || container == nil {
1065		return domain.ErrContainerNotFound
1066	}
1067
1068	// When attachments are requested, sync runtime state first to get accurate
1069	// attachment tracking, then check if configured attachments are deployed.
1070	// If fewer attachment containers are tracked than configured, fail early with
1071	// guidance rather than restarting the main container into a broken state
1072	// (e.g., missing database hostname).
1073	if withAttachments {
1074		if syncErr := s.SyncContainers(ctx); syncErr != nil {
1075			log.Warn().Err(syncErr).Msg("failed to sync container state before attachment check")
1076		}
1077
1078		// Re-read attachment IDs after sync
1079		s.mu.RLock()
1080		if freshAttachments := s.attachments[domainName]; len(freshAttachments) > 0 {
1081			attachmentIDs = make([]string, len(freshAttachments))
1082			copy(attachmentIDs, freshAttachments)
1083		} else {
1084			attachmentIDs = nil
1085		}
1086		s.mu.RUnlock()
1087
1088		configuredAttachments := s.resolveAttachmentsForDomain(domainName)
1089		if len(configuredAttachments) > 0 && len(attachmentIDs) < len(configuredAttachments) {
1090			missing := len(configuredAttachments) - len(attachmentIDs)
1091			return fmt.Errorf("%w: domain %q has %d configured attachment(s) but only %d deployed (%d missing)",
1092				domain.ErrAttachmentNotDeployed, domainName, len(configuredAttachments), len(attachmentIDs), missing)
1093		}
1094	}
1095
1096	// Restart the main container (with stale-state recovery)
1097	container, attachmentIDs, err := s.restartContainerWithRecovery(ctx, domainName, container, attachmentIDs)
1098	if err != nil {
1099		return err
1100	}
1101	log.Info().Str(zerowrap.FieldEntityID, container.ID).Msg("container restarted")
1102
1103	// Record restart metric
1104	if s.metrics != nil {
1105		s.metrics.ContainerRestarts.Add(ctx, 1, metric.WithAttributes(
1106			attribute.String("domain", domainName),
1107			attribute.String("source", "api"),
1108		))
1109	}
1110
1111	// Restart attachments if requested
1112	if withAttachments && len(attachmentIDs) > 0 {
1113		for _, attachID := range attachmentIDs {
1114			if err := s.runtime.RestartContainer(ctx, attachID); err != nil {
1115				log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to restart attachment")
1116				continue // Don't fail the whole operation for one attachment
1117			}
1118			log.Info().Str("attachment_id", attachID).Msg("attachment restarted")
1119		}
1120	}
1121
1122	return nil
1123}
1124
1125// Stop stops a running container.
1126func (s *Service) Stop(ctx context.Context, containerID string) error {
1127	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1128		zerowrap.FieldLayer:    "usecase",
1129		zerowrap.FieldUseCase:  "Stop",
1130		zerowrap.FieldEntityID: containerID,
1131	})
1132	log := zerowrap.FromCtx(ctx)
1133
1134	// Stop log collection before stopping container
1135	if s.logWriter != nil {
1136		if err := s.logWriter.StopLogging(containerID); err != nil {
1137			log.Warn().Err(err).Msg("failed to stop container log collection")
1138		}
1139	}
1140
1141	if err := s.runtime.StopContainer(ctx, containerID); err != nil {
1142		return log.WrapErr(err, "failed to stop container")
1143	}
1144
1145	log.Info().Msg("container stopped")
1146	return nil
1147}
1148
1149// Remove removes a container.
1150func (s *Service) Remove(ctx context.Context, containerID string, force bool) error {
1151	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1152		zerowrap.FieldLayer:    "usecase",
1153		zerowrap.FieldUseCase:  "Remove",
1154		zerowrap.FieldEntityID: containerID,
1155	})
1156	log := zerowrap.FromCtx(ctx)
1157
1158	// Stop log collection before removing container
1159	if s.logWriter != nil {
1160		if err := s.logWriter.StopLogging(containerID); err != nil {
1161			log.Warn().Err(err).Msg("failed to stop container log collection")
1162		}
1163	}
1164
1165	// Find domain and attachment IDs for cleanup
1166	containerDomain, attachmentIDs, cfg := s.findContainerContext(containerID)
1167
1168	if err := s.runtime.RemoveContainer(ctx, containerID, force); err != nil {
1169		return log.WrapErr(err, "failed to remove container")
1170	}
1171
1172	// Cleanup volumes
1173	if containerDomain != "" && !cfg.VolumePreserve {
1174		if err := s.cleanupVolumesForDomain(ctx, containerDomain); err != nil {
1175			log.WrapErrWithFields(err, "failed to cleanup volumes", map[string]any{"domain": containerDomain})
1176		}
1177	}
1178
1179	// Clean up attachment containers
1180	s.removeAttachments(ctx, attachmentIDs)
1181
1182	// Remove from tracking
1183	s.mu.Lock()
1184	var removedDomain string
1185	for d, c := range s.containers {
1186		if c.ID == containerID {
1187			delete(s.containers, d)
1188			removedDomain = d
1189			s.managedCount--
1190			log.Info().Str("domain", d).Msg("container removed")
1191			break
1192		}
1193	}
1194	if removedDomain != "" {
1195		delete(s.attachments, removedDomain)
1196	}
1197	s.mu.Unlock()
1198
1199	// Decrement managed container count
1200	if removedDomain != "" && s.metrics != nil {
1201		s.metrics.ManagedContainers.Add(ctx, -1)
1202	}
1203
1204	// Note: We intentionally do NOT clean up deployMu entries to avoid a race condition:
1205	// If Remove deletes the mutex entry while a concurrent Deploy is acquiring the lock,
1206	// the Deploy would create a fresh mutex, breaking serialization and allowing concurrent
1207	// deploys to the same domain. The memory footprint is acceptable (one small struct per
1208	// domain ever deployed), and this is the safer choice for correctness.
1209
1210	// Cleanup network
1211	if removedDomain != "" && cfg.NetworkIsolation {
1212		networkName := s.getNetworkForApp(removedDomain)
1213		if err := s.cleanupNetworkIfEmpty(ctx, networkName); err != nil {
1214			log.WrapErrWithFields(err, "failed to cleanup network", map[string]any{"network": networkName})
1215		}
1216	}
1217
1218	return nil
1219}
1220
1221// findContainerContext looks up the domain, attachment IDs, and config snapshot for a container.
1222func (s *Service) findContainerContext(containerID string) (string, []string, Config) {
1223	s.mu.RLock()
1224	defer s.mu.RUnlock()
1225	cfg := s.config
1226	var containerDomain string
1227	for d, c := range s.containers {
1228		if c.ID == containerID {
1229			containerDomain = d
1230			break
1231		}
1232	}
1233	var attachmentIDs []string
1234	if containerDomain != "" {
1235		attachmentIDs = append(attachmentIDs, s.attachments[containerDomain]...)
1236	}
1237	return containerDomain, attachmentIDs, cfg
1238}
1239
1240// removeAttachments stops and removes attachment containers best-effort.
1241func (s *Service) removeAttachments(ctx context.Context, attachmentIDs []string) {
1242	log := zerowrap.FromCtx(ctx)
1243	for _, attachID := range attachmentIDs {
1244		if s.logWriter != nil {
1245			if err := s.logWriter.StopLogging(attachID); err != nil {
1246				log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to stop attachment log collection")
1247			}
1248		}
1249		if err := s.runtime.StopContainer(ctx, attachID); err != nil {
1250			log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to stop attachment container")
1251		}
1252		if err := s.runtime.RemoveContainer(ctx, attachID, true); err != nil {
1253			log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to remove attachment container")
1254		}
1255	}
1256}
1257
1258// Get retrieves a container by domain.
1259func (s *Service) Get(_ context.Context, domainName string) (*domain.Container, bool) {
1260	s.mu.RLock()
1261	defer s.mu.RUnlock()
1262	container, exists := s.containers[domainName]
1263	return container, exists
1264}
1265
1266// List returns all managed containers.
1267func (s *Service) List(_ context.Context) map[string]*domain.Container {
1268	s.mu.RLock()
1269	defer s.mu.RUnlock()
1270
1271	result := make(map[string]*domain.Container, len(s.containers))
1272	maps.Copy(result, s.containers)
1273	return result
1274}
1275
1276// ListRoutesWithDetails returns routes with network and attachment info.
1277// Note: Container map is copied under lock, then external calls are made without lock.
1278// If containers are removed between copy and runtime calls, errors are handled gracefully
1279// (network becomes empty string). This trade-off avoids holding locks during I/O.
1280func (s *Service) ListRoutesWithDetails(ctx context.Context) []domain.RouteInfo {
1281	s.mu.RLock()
1282	containers := make(map[string]*domain.Container, len(s.containers))
1283	maps.Copy(containers, s.containers)
1284	s.mu.RUnlock()
1285
1286	// Fetch all attachments once to avoid N+1 queries
1287	attachmentsByDomain := s.getAllAttachments(ctx)
1288
1289	results := make([]domain.RouteInfo, 0, len(containers))
1290	for domainName, container := range containers {
1291		network := ""
1292		image := ""
1293		containerID := ""
1294		status := ""
1295		if container != nil {
1296			containerID = container.ID
1297			status = container.Status
1298			image = container.Image
1299			if container.Labels != nil {
1300				if labelImage, ok := container.Labels[domain.LabelImage]; ok && labelImage != "" {
1301					image = labelImage
1302				}
1303			}
1304			// Strip registry domain prefix from image for cleaner display
1305			image = s.stripRegistryPrefix(image)
1306			if networkName, err := s.runtime.GetContainerNetwork(ctx, container.ID); err == nil {
1307				network = networkName
1308			}
1309		}
1310
1311		results = append(results, domain.RouteInfo{
1312			Domain:          domainName,
1313			Image:           image,
1314			ContainerID:     containerID,
1315			ContainerStatus: status,
1316			Network:         network,
1317			Attachments:     attachmentsByDomain[domainName],
1318		})
1319	}
1320
1321	return results
1322}
1323
1324// stripRegistryPrefix removes the configured registry domain prefix from an image reference.
1325// For example, "reg.example.com/myapp:latest" becomes "myapp:latest" if registry domain is "reg.example.com".
1326func (s *Service) stripRegistryPrefix(image string) string {
1327	s.mu.RLock()
1328	cfg := s.config
1329	s.mu.RUnlock()
1330
1331	if cfg.RegistryDomain == "" {
1332		return image
1333	}
1334	prefix := strings.TrimSuffix(cfg.RegistryDomain, "/") + "/"
1335	if strings.HasPrefix(image, prefix) {
1336		return strings.TrimPrefix(image, prefix)
1337	}
1338	return image
1339}
1340
1341// ListAttachments returns attachments for a domain.
1342func (s *Service) ListAttachments(ctx context.Context, domainName string) []domain.Attachment {
1343	return s.getAttachmentsForDomain(ctx, domainName)
1344}
1345
1346// ListNetworks returns Gordon-managed networks.
1347func (s *Service) ListNetworks(ctx context.Context) ([]*domain.NetworkInfo, error) {
1348	s.mu.RLock()
1349	cfg := s.config
1350	s.mu.RUnlock()
1351
1352	networks, err := s.runtime.ListNetworks(ctx)
1353	if err != nil {
1354		return nil, err
1355	}
1356
1357	var filtered []*domain.NetworkInfo
1358	for _, network := range networks {
1359		if strings.HasPrefix(network.Name, cfg.NetworkPrefix+"-") && network.Labels[domain.LabelManaged] == "true" {
1360			filtered = append(filtered, network)
1361		}
1362	}
1363
1364	return filtered, nil
1365}
1366
1367// HealthCheck performs health checks on all containers.
1368func (s *Service) HealthCheck(ctx context.Context) map[string]bool {
1369	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1370		zerowrap.FieldLayer:   "usecase",
1371		zerowrap.FieldUseCase: "HealthCheck",
1372	})
1373	log := zerowrap.FromCtx(ctx)
1374
1375	// Copy container IDs under lock, then release before making Docker API calls
1376	s.mu.RLock()
1377	snapshot := make(map[string]string, len(s.containers))
1378	for d, c := range s.containers {
1379		snapshot[d] = c.ID
1380	}
1381	s.mu.RUnlock()
1382
1383	health := make(map[string]bool, len(snapshot))
1384	for d, id := range snapshot {
1385		running, err := s.runtime.IsContainerRunning(ctx, id)
1386		if err != nil {
1387			log.WrapErrWithFields(err, "health check failed", map[string]any{"domain": d, zerowrap.FieldEntityID: id})
1388			health[d] = false
1389		} else {
1390			health[d] = running
1391		}
1392	}
1393	return health
1394}
1395
1396// SyncContainers synchronizes containers with runtime state.
1397func (s *Service) SyncContainers(ctx context.Context) error {
1398	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1399		zerowrap.FieldLayer:   "usecase",
1400		zerowrap.FieldUseCase: "SyncContainers",
1401	})
1402	log := zerowrap.FromCtx(ctx)
1403
1404	// List containers without holding lock to avoid blocking during Docker API call
1405	allContainers, err := s.runtime.ListContainers(ctx, false)
1406	if err != nil {
1407		return log.WrapErr(err, "failed to list containers")
1408	}
1409
1410	managed := make(map[string]*domain.Container)
1411	attachments := make(map[string][]string)
1412	for _, c := range allContainers {
1413		if c.Labels == nil || c.Labels[domain.LabelManaged] != "true" {
1414			continue
1415		}
1416		if c.Labels[domain.LabelAttachment] == "true" {
1417			owner := c.Labels[domain.LabelAttachedTo]
1418			if owner != "" {
1419				attachments[owner] = append(attachments[owner], c.ID)
1420			}
1421			continue
1422		}
1423		if d, ok := c.Labels[domain.LabelDomain]; ok {
1424			managed[d] = c
1425		}
1426	}
1427
1428	s.mu.Lock()
1429	s.containers = managed
1430	s.attachments = attachments
1431	newCount := int64(len(managed))
1432	delta := newCount - s.managedCount
1433	s.managedCount = newCount
1434	s.mu.Unlock()
1435
1436	// Report initial/delta managed container count to OTel UpDownCounter.
1437	if delta != 0 && s.metrics != nil {
1438		s.metrics.ManagedContainers.Add(ctx, delta)
1439	}
1440
1441	log.Info().Int(zerowrap.FieldCount, len(managed)).Msg("container state synchronized")
1442	return nil
1443}
1444
1445// AutoStart starts containers for the provided routes that aren't running.
1446// It skips readiness checks to avoid blocking boot; the background monitor
1447// handles crash recovery.
1448func (s *Service) AutoStart(ctx context.Context, routes []domain.Route) error {
1449	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1450		zerowrap.FieldLayer:   "usecase",
1451		zerowrap.FieldUseCase: "AutoStart",
1452	})
1453	log := zerowrap.FromCtx(ctx)
1454
1455	log.Info().Int("route_count", len(routes)).Msg("auto-starting containers for configured routes")
1456
1457	// Filter out routes that already have a running container.
1458	var pending []domain.Route
1459	skipped := 0
1460	for _, route := range routes {
1461		if _, exists := s.Get(ctx, route.Domain); exists {
1462			log.Debug().Str("domain", route.Domain).Msg("container already running, skipping")
1463			skipped++
1464		} else {
1465			pending = append(pending, route)
1466		}
1467	}
1468
1469	if len(pending) == 0 {
1470		log.Info().Int("skipped", skipped).Msg("auto-start completed, all containers already running")
1471		return nil
1472	}
1473
1474	// Deploy all pending routes concurrently with readiness checks skipped.
1475	deployCtx := domain.WithSkipReadiness(ctx)
1476
1477	type result struct {
1478		route domain.Route
1479		err   error
1480	}
1481	results := make(chan result, len(pending))
1482	sem := make(chan struct{}, 4) // limit concurrency
1483
1484	for _, route := range pending {
1485		sem <- struct{}{}
1486		go func(r domain.Route) {
1487			defer func() { <-sem }()
1488
1489			log.Info().Str("domain", r.Domain).Str("image", r.Image).Msg("auto-starting container for route")
1490
1491			if _, err := s.Deploy(deployCtx, r); err != nil {
1492				results <- result{route: r, err: err}
1493				return
1494			}
1495			results <- result{route: r}
1496		}(route)
1497	}
1498
1499	var started, errCount int
1500	for i := 0; i < len(pending); i++ {
1501		res := <-results
1502		if res.err != nil {
1503			log.Warn().Err(res.err).Str("domain", res.route.Domain).Msg("failed to auto-start container")
1504			errCount++
1505		} else {
1506			started++
1507		}
1508	}
1509
1510	log.Info().
1511		Int("started", started).
1512		Int("skipped", skipped).
1513		Int("errors", errCount).
1514		Msg("auto-start completed")
1515
1516	if errCount > 0 {
1517		return fmt.Errorf("auto-start completed with %d errors", errCount)
1518	}
1519	return nil
1520}
1521
1522// Shutdown gracefully shuts down all managed containers.
1523func (s *Service) Shutdown(ctx context.Context) error {
1524	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1525		zerowrap.FieldLayer:   "usecase",
1526		zerowrap.FieldUseCase: "Shutdown",
1527	})
1528	log := zerowrap.FromCtx(ctx)
1529	log.Info().Msg("shutting down container manager...")
1530
1531	// Wait for any in-flight background cleanup goroutines to finish
1532	s.cleanupWg.Wait()
1533
1534	// Containers are left running across Gordon restarts.
1535	// SyncContainers + AutoStart will pick them back up on next boot.
1536
1537	// Close log writer to stop all log collection
1538	if s.logWriter != nil {
1539		if err := s.logWriter.Close(); err != nil {
1540			log.Warn().Err(err).Msg("failed to close container log writer")
1541		}
1542	}
1543
1544	log.Info().Msg("container manager shutdown complete")
1545	return nil
1546}
1547
1548// WaitForCleanup blocks until all background cleanup goroutines complete.
1549// Intended for use in tests to avoid mock assertion races.
1550func (s *Service) WaitForCleanup() {
1551	s.cleanupWg.Wait()
1552}
1553
1554// StartMonitor begins background monitoring of tracked containers.
1555// The monitor restarts crashed containers and detects crash loops.
1556// Safe to call multiple times; subsequent calls are no-ops.
1557func (s *Service) StartMonitor(ctx context.Context) {
1558	s.mu.Lock()
1559	if s.monitor != nil {
1560		s.mu.Unlock()
1561		return
1562	}
1563	s.monitor = newMonitor(s)
1564	m := s.monitor
1565	m.Start(ctx)
1566	s.mu.Unlock()
1567}
1568
1569// StopMonitor stops the background container monitor.
1570// Safe to call multiple times; subsequent calls are no-ops.
1571func (s *Service) StopMonitor() {
1572	s.mu.Lock()
1573	m := s.monitor
1574	s.monitor = nil
1575	s.mu.Unlock()
1576	if m != nil {
1577		m.Stop()
1578	}
1579}
1580
1581// UpdateConfig updates the service configuration.
1582func (s *Service) UpdateConfig(config Config) {
1583	s.mu.Lock()
1584	s.config = config
1585	s.mu.Unlock()
1586}
1587
1588// UpdateAttachments updates only the attachment configuration in the service.
1589// This is called after a config reload to propagate attachment changes without restart.
1590// The incoming map is deep-copied so external callers cannot mutate service state.
1591func (s *Service) UpdateAttachments(attachments map[string][]string) {
1592	var copied map[string][]string
1593	if attachments != nil {
1594		copied = make(map[string][]string, len(attachments))
1595		for k, v := range attachments {
1596			sl := make([]string, len(v))
1597			copy(sl, v)
1598			copied[k] = sl
1599		}
1600	}
1601	s.mu.Lock()
1602	s.config.Attachments = copied
1603	s.mu.Unlock()
1604}
1605
1606// Helper methods
1607
1608// cleanupFailedContainer stops and removes a container that failed to start properly.
1609func (s *Service) cleanupFailedContainer(ctx context.Context, containerID string) {
1610	log := zerowrap.FromCtx(ctx)
1611	if err := s.runtime.StopContainer(ctx, containerID); err != nil {
1612		log.Warn().Err(err).Str(zerowrap.FieldEntityID, containerID).Msg("failed to stop container after failure")
1613	}
1614	if err := s.runtime.RemoveContainer(ctx, containerID, true); err != nil {
1615		log.Warn().Err(err).Str(zerowrap.FieldEntityID, containerID).Msg("failed to remove container after failure")
1616	}
1617}
1618
1619// cleanupOldContainer stops and removes an old container after zero-downtime switch.
1620// It also renames the new container to the canonical name.
1621func (s *Service) cleanupOldContainer(ctx context.Context, old *domain.Container, newContainerID, domainName string) {
1622	log := zerowrap.FromCtx(ctx)
1623	log.Info().Str(zerowrap.FieldEntityID, old.ID).Msg("stopping old container after zero-downtime switch")
1624
1625	if s.logWriter != nil {
1626		if err := s.logWriter.StopLogging(old.ID); err != nil {
1627			log.Warn().Err(err).Str(zerowrap.FieldEntityID, old.ID).Msg("failed to stop logging for old container")
1628		}
1629	}
1630
1631	if err := s.runtime.StopContainer(ctx, old.ID); err != nil {
1632		log.Warn().Err(err).Str(zerowrap.FieldEntityID, old.ID).Msg("failed to stop old container")
1633	}
1634
1635	if err := s.runtime.RemoveContainer(ctx, old.ID, true); err != nil {
1636		log.Warn().Err(err).Str(zerowrap.FieldEntityID, old.ID).Msg("failed to remove old container")
1637	}
1638
1639	// Rename new container to canonical name
1640	canonicalName := fmt.Sprintf("gordon-%s", domainName)
1641	if err := s.runtime.RenameContainer(ctx, newContainerID, canonicalName); err != nil {
1642		log.Warn().Err(err).Str("canonical_name", canonicalName).Msg("failed to rename container to canonical name")
1643	}
1644}
1645
1646// startLogCollection starts log collection for a container in the background.
1647// Errors are logged but don't fail the calling operation.
1648func (s *Service) startLogCollection(ctx context.Context, containerID, domainName string) {
1649	if s.logWriter == nil {
1650		return
1651	}
1652
1653	log := zerowrap.FromCtx(ctx)
1654
1655	// Use context.WithoutCancel so the log stream outlives the HTTP request.
1656	// Without this, the Docker log stream closes when the deploy response completes.
1657	bgCtx := context.WithoutCancel(ctx)
1658
1659	logStream, err := s.runtime.GetContainerLogs(bgCtx, containerID, true)
1660	if err != nil {
1661		log.Warn().Err(err).Msg("failed to get container logs for collection")
1662		return
1663	}
1664
1665	if err := s.logWriter.StartLogging(bgCtx, containerID, domainName, logStream); err != nil {
1666		log.Warn().Err(err).Msg("failed to start container log collection")
1667		logStream.Close()
1668	}
1669}
1670
1671func (s *Service) buildImageRef(image string) string {
1672	s.mu.RLock()
1673	cfg := s.config
1674	s.mu.RUnlock()
1675
1676	if !cfg.RegistryAuthEnabled || cfg.RegistryDomain == "" {
1677		return image
1678	}
1679
1680	// Normalize registry domain by trimming any trailing slash
1681	reg := strings.TrimSuffix(cfg.RegistryDomain, "/")
1682
1683	// Check if image already has the registry domain prefix
1684	prefix := reg + "/"
1685	if strings.HasPrefix(image, prefix) {
1686		return image
1687	}
1688
1689	// Check if image already has an explicit registry (host:port/ or host.domain/).
1690	// We need to detect patterns like:
1691	// - "docker.io/library/nginx:latest" (has dot and slash)
1692	// - "localhost:5001/myapp:latest" (has host:port/ pattern)
1693	// - "myregistry:5000/app:v1" (has host:port/ pattern)
1694	// - "[fd00::1]:5000/app:v1" (IPv6 with port)
1695	if hasExplicitRegistry(image) {
1696		return image
1697	}
1698
1699	return fmt.Sprintf("%s/%s", reg, image)
1700}
1701
1702// hasExplicitRegistry checks if an image reference already includes an explicit registry.
1703// Returns true for patterns like "docker.io/image", "localhost:5000/image", "localhost/image",
1704// "registry:8080/image", "[fd00::1]/image", "[fd00::1]:5000/image".
1705func hasExplicitRegistry(image string) bool {
1706	// Find the first slash which separates registry from image name
1707	slashIdx := strings.Index(image, "/")
1708	if slashIdx == -1 {
1709		return false // No slash means no registry prefix (e.g., "myapp:latest")
1710	}
1711
1712	// Extract the part before the first slash (potential registry)
1713	registryPart := image[:slashIdx]
1714
1715	// Check for bracketed IPv6 address (e.g., "[fd00::1]" or "[fd00::1]:5000")
1716	if strings.HasPrefix(registryPart, "[") {
1717		// Look for closing bracket
1718		if closeBracket := strings.Index(registryPart, "]"); closeBracket != -1 {
1719			// Valid bracketed IPv6: either ends at ] or has ]:port
1720			return true
1721		}
1722	}
1723
1724	// Check for localhost (with or without port)
1725	if registryPart == "localhost" {
1726		return true
1727	}
1728
1729	// Check for host:port pattern (e.g., "localhost:5000", "registry:8080")
1730	if colonIdx := strings.LastIndex(registryPart, ":"); colonIdx != -1 {
1731		port := registryPart[colonIdx+1:]
1732		// If everything after colon is digits, it's a port
1733		if len(port) > 0 && isNumeric(port) {
1734			return true
1735		}
1736	}
1737
1738	// Check for domain pattern (has a dot, e.g., "docker.io", "gcr.io")
1739	if strings.Contains(registryPart, ".") {
1740		return true
1741	}
1742
1743	return false
1744}
1745
1746// isNumeric checks if a string contains only digits.
1747func isNumeric(s string) bool {
1748	for _, c := range s {
1749		if c < '0' || c > '9' {
1750			return false
1751		}
1752	}
1753	return true
1754}
1755
1756func normalizePullPolicy(policy string) string {
1757	switch strings.ToLower(strings.TrimSpace(policy)) {
1758	case PullPolicyAlways:
1759		return PullPolicyAlways
1760	case PullPolicyIfTagChanged:
1761		return PullPolicyIfTagChanged
1762	case PullPolicyIfNotPresent:
1763		return PullPolicyIfNotPresent
1764	default:
1765		return PullPolicyIfNotPresent
1766	}
1767}
1768
1769func isDigestRef(imageRef string) bool {
1770	return strings.Contains(imageRef, "@sha256:")
1771}
1772
1773func (s *Service) pullRefForDeploy(ctx context.Context, imageRef string) (string, bool) {
1774	if !domain.IsInternalDeploy(ctx) {
1775		return imageRef, false
1776	}
1777	s.mu.RLock()
1778	cfg := s.config
1779	s.mu.RUnlock()
1780	return rewriteToLocalRegistry(imageRef, cfg.RegistryDomain, cfg.RegistryPort), true
1781}
1782
1783// ensureImage ensures the image is available locally, pulling if needed.
1784// Returns the image reference to use for container operations:
1785//   - For digest references (@sha256:...), returns the pullRef since Docker can't tag digests
1786//   - For tagged images, returns the original imageRef after tagging the pulled image
1787func (s *Service) ensureImage(ctx context.Context, imageRef string) (string, error) {
1788	ctx, span := tracer.Start(ctx, "container.ensure_image",
1789		trace.WithAttributes(attribute.String("image", imageRef)))
1790	defer span.End()
1791
1792	ctx = zerowrap.CtxWithField(ctx, "image", imageRef)
1793	log := zerowrap.FromCtx(ctx)
1794
1795	// Determine if this is an internal deploy and what reference to use for pulls.
1796	pullRef, isInternal := s.pullRefForDeploy(ctx, imageRef)
1797	if pullRef != imageRef {
1798		log.Info().
1799			Str("original_ref", imageRef).
1800			Str("pull_ref", pullRef).
1801			Msg("internal deploy: using localhost registry for pull")
1802	}
1803
1804	found, err := s.ensureLocalImage(ctx, imageRef, pullRef)
1805	if err != nil {
1806		return "", err
1807	}
1808	if found {
1809		return imageRef, nil
1810	}
1811
1812	// Pull image
1813	log.Info().Msg("pulling image from registry")
1814
1815	if err := s.pullImage(ctx, pullRef, isInternal); err != nil {
1816		return "", err
1817	}
1818
1819	// For digest references, we can't create a tag (Docker doesn't allow it).
1820	// In this case, use the pullRef directly since the image is available under that reference.
1821	if strings.Contains(imageRef, "@sha256:") {
1822		log.Info().
1823			Str("pull_ref", pullRef).
1824			Msg("digest reference: using pull reference for container operations")
1825		return pullRef, nil
1826	}
1827
1828	if err := s.tagImageIfNeeded(ctx, pullRef, imageRef); err != nil {
1829		return "", err
1830	}
1831
1832	// Clean up the temporary pull reference tag to avoid duplicate entries
1833	if pullRef != imageRef {
1834		if err := s.runtime.UntagImage(ctx, pullRef); err != nil {
1835			// Log but don't fail - the canonical tag is already applied
1836			log.Debug().Err(err).Str("pull_ref", pullRef).Msg("failed to remove temporary pull tag")
1837		}
1838	}
1839
1840	log.Info().Msg("image pulled successfully")
1841	return imageRef, nil
1842}
1843
1844func (s *Service) ensureLocalImage(ctx context.Context, imageRef, pullRef string) (bool, error) {
1845	log := zerowrap.FromCtx(ctx)
1846
1847	// For internal deploys (image push events), always pull fresh image
1848	// because the same tag may reference new content (e.g., latest tag updated)
1849	if domain.IsInternalDeploy(ctx) {
1850		log.Info().Msg("internal deploy detected, forcing image pull to ensure latest content")
1851		return false, nil
1852	}
1853
1854	s.mu.RLock()
1855	cfg := s.config
1856	s.mu.RUnlock()
1857
1858	pullPolicy := normalizePullPolicy(cfg.PullPolicy)
1859	switch pullPolicy {
1860	case PullPolicyAlways:
1861		log.Info().Str("pull_policy", pullPolicy).Msg("pull policy forces image pull")
1862		return false, nil
1863	case PullPolicyIfTagChanged:
1864		if !isDigestRef(imageRef) {
1865			log.Info().Str("pull_policy", pullPolicy).Msg("tag reference detected, pulling to check for updates")
1866			return false, nil
1867		}
1868	}
1869
1870	localImages, err := s.runtime.ListImages(ctx)
1871	if err != nil {
1872		log.WrapErr(err, "failed to list local images, will attempt pull")
1873		return false, nil
1874	}
1875
1876	normalizedRef := normalizeImageRef(imageRef)
1877	normalizedPullRef := normalizeImageRef(pullRef)
1878	for _, img := range localImages {
1879		normalizedImage := normalizeImageRef(img)
1880		if normalizedImage == normalizedRef {
1881			log.Info().Msg("image found locally, skipping pull")
1882			return true, nil
1883		}
1884		if normalizedImage == normalizedPullRef {
1885			if err := s.tagImageIfNeeded(ctx, pullRef, imageRef); err != nil {
1886				return false, err
1887			}
1888			// Clean up the temporary pull reference tag
1889			if pullRef != imageRef {
1890				if err := s.runtime.UntagImage(ctx, pullRef); err != nil {
1891					log.Debug().Err(err).Str("pull_ref", pullRef).Msg("failed to remove temporary pull tag")
1892				}
1893			}
1894			log.Info().Msg("image found locally, skipping pull")
1895			return true, nil
1896		}
1897	}
1898
1899	return false, nil
1900}
1901
1902func (s *Service) pullImage(ctx context.Context, pullRef string, isInternal bool) error {
1903	s.mu.RLock()
1904	cfg := s.config
1905	s.mu.RUnlock()
1906
1907	log := zerowrap.FromCtx(ctx)
1908
1909	switch {
1910	case isInternal && cfg.RegistryAuthEnabled:
1911		if cfg.InternalRegistryUsername == "" || cfg.InternalRegistryPassword == "" {
1912			return log.WrapErr(fmt.Errorf("internal registry auth not configured"), "failed to pull image for internal deploy")
1913		}
1914		if err := s.pullImageWithRetry(ctx, func(ctx context.Context) error {
1915			return s.runtime.PullImageWithAuth(ctx, pullRef, cfg.InternalRegistryUsername, cfg.InternalRegistryPassword)
1916		}, isInternal); err != nil {
1917			return log.WrapErr(fmt.Errorf("%w: %w", domain.ErrImagePullFailed, err), "failed to pull image with internal auth")
1918		}
1919	case isInternal:
1920		if err := s.pullImageWithRetry(ctx, func(ctx context.Context) error {
1921			return s.runtime.PullImage(ctx, pullRef)
1922		}, isInternal); err != nil {
1923			return log.WrapErr(fmt.Errorf("%w: %w", domain.ErrImagePullFailed, err), "failed to pull image")
1924		}
1925	case cfg.RegistryAuthEnabled:
1926		if cfg.ServiceTokenUsername == "" || cfg.ServiceToken == "" {
1927			return log.WrapErr(fmt.Errorf("registry service token not configured"), "failed to pull image for registry auth")
1928		}
1929		if err := s.runtime.PullImageWithAuth(ctx, pullRef, cfg.ServiceTokenUsername, cfg.ServiceToken); err != nil {
1930			return log.WrapErr(fmt.Errorf("%w: %w", domain.ErrImagePullFailed, err), "failed to pull image with auth")
1931		}
1932	default:
1933		if err := s.runtime.PullImage(ctx, pullRef); err != nil {
1934			return log.WrapErr(fmt.Errorf("%w: %w", domain.ErrImagePullFailed, err), "failed to pull image")
1935		}
1936	}
1937
1938	return nil
1939}
1940
1941func (s *Service) pullImageWithRetry(ctx context.Context, pullFn func(context.Context) error, isInternal bool) error {
1942	log := zerowrap.FromCtx(ctx)
1943
1944	for attempt := 1; attempt <= internalPullMaxAttempts; attempt++ {
1945		err := pullFn(ctx)
1946		if err == nil {
1947			return nil
1948		}
1949		if !isInternal || !isConnectionRefusedError(err) || attempt == internalPullMaxAttempts {
1950			return fmt.Errorf("internal image pull failed after %d attempts: %w", attempt, err)
1951		}
1952
1953		backoff := time.Duration(attempt) * time.Second
1954		log.Warn().
1955			Err(err).
1956			Int("attempt", attempt).
1957			Dur("backoff", backoff).
1958			Msg("internal image pull failed with connection refused, retrying")
1959
1960		select {
1961		case <-time.After(backoff):
1962		case <-ctx.Done():
1963			return ctx.Err()
1964		}
1965	}
1966	return nil
1967}
1968
1969func (s *Service) tagImageIfNeeded(ctx context.Context, sourceRef, targetRef string) error {
1970	if sourceRef == targetRef {
1971		return nil
1972	}
1973
1974	// Cannot create a tag with a digest reference - Docker/Podman doesn't allow it.
1975	// When using digest references (image@sha256:...), skip tagging as the image
1976	// is already available by its digest.
1977	if strings.Contains(targetRef, "@sha256:") {
1978		return nil
1979	}
1980
1981	log := zerowrap.FromCtx(ctx)
1982	if err := s.runtime.TagImage(ctx, sourceRef, targetRef); err != nil {
1983		return log.WrapErr(err, "failed to tag image from pull reference")
1984	}
1985	return nil
1986}
1987
1988func (s *Service) loadEnvironment(ctx context.Context, preResolved []string, domainName, imageRef string) ([]string, error) {
1989	log := zerowrap.FromCtx(ctx)
1990
1991	var userEnvVars []string
1992	if len(preResolved) > 0 {
1993		userEnvVars = preResolved
1994	} else {
1995		var err error
1996		userEnvVars, err = s.envLoader.LoadEnv(ctx, domainName)
1997		if err != nil {
1998			return nil, log.WrapErr(err, "failed to load environment variables")
1999		}
2000	}
2001
2002	dockerfileEnvVars, err := s.runtime.InspectImageEnv(ctx, imageRef)
2003	if err != nil {
2004		log.WrapErr(err, "failed to inspect image environment")
2005		dockerfileEnvVars = []string{}
2006	}
2007
2008	return mergeEnvironmentVariables(dockerfileEnvVars, userEnvVars), nil
2009}
2010
2011func (s *Service) setupVolumes(ctx context.Context, domainName, imageRef string) (map[string]string, error) {
2012	s.mu.RLock()
2013	cfg := s.config
2014	s.mu.RUnlock()
2015
2016	log := zerowrap.FromCtx(ctx)
2017	volumes := make(map[string]string)
2018
2019	if !cfg.VolumeAutoCreate {
2020		return volumes, nil
2021	}
2022
2023	volumePaths, err := s.runtime.InspectImageVolumes(ctx, imageRef)
2024	if err != nil {
2025		log.WrapErr(err, "failed to inspect image volumes")
2026		return volumes, nil
2027	}
2028
2029	for _, path := range volumePaths {
2030		name := generateVolumeName(cfg.VolumePrefix, domainName, path)
2031
2032		exists, err := s.runtime.VolumeExists(ctx, name)
2033		if err != nil {
2034			log.WrapErrWithFields(err, "failed to check volume", map[string]any{"volume": name})
2035			continue
2036		}
2037
2038		if !exists {
2039			if err := s.runtime.CreateVolume(ctx, name); err != nil {
2040				log.WrapErrWithFields(err, "failed to create volume", map[string]any{"volume": name})
2041				continue
2042			}
2043			log.Info().Str("volume", name).Str(zerowrap.FieldPath, path).Msg("created volume")
2044		}
2045
2046		volumes[path] = name
2047	}
2048
2049	return volumes, nil
2050}
2051
2052func (s *Service) getNetworkForApp(domainName string) string {
2053	var networkIsolation bool
2054	var networkGroups map[string][]string
2055
2056	if s.configProvider != nil {
2057		snap := s.configProvider.GetAttachmentConfig()
2058		networkGroups = snap.NetworkGroups
2059		s.mu.RLock()
2060		networkIsolation = s.config.NetworkIsolation
2061		s.mu.RUnlock()
2062	} else {
2063		s.mu.RLock()
2064		networkIsolation = s.config.NetworkIsolation
2065		networkGroups = s.config.NetworkGroups
2066		s.mu.RUnlock()
2067	}
2068
2069	if !networkIsolation {
2070		return "bridge"
2071	}
2072
2073	for groupName, domains := range networkGroups {
2074		if slices.Contains(domains, domainName) {
2075			return s.generateNetworkName(groupName)
2076		}
2077	}
2078
2079	return s.generateNetworkName(domainName)
2080}
2081
2082func (s *Service) generateNetworkName(identifier string) string {
2083	s.mu.RLock()
2084	cfg := s.config
2085	s.mu.RUnlock()
2086
2087	return fmt.Sprintf("%s-%s", cfg.NetworkPrefix, strings.ReplaceAll(identifier, ".", "-"))
2088}
2089
2090func (s *Service) createNetworkIfNeeded(ctx context.Context, networkName string) error {
2091	if networkName == "bridge" || networkName == "default" {
2092		return nil
2093	}
2094
2095	ctx = zerowrap.CtxWithField(ctx, "network", networkName)
2096	log := zerowrap.FromCtx(ctx)
2097
2098	exists, err := s.runtime.NetworkExists(ctx, networkName)
2099	if err != nil {
2100		return log.WrapErr(err, "failed to check network existence")
2101	}
2102
2103	if !exists {
2104		if err := s.runtime.CreateNetwork(ctx, networkName, map[string]string{"driver": "bridge"}); err != nil {
2105			return log.WrapErr(err, "failed to create network")
2106		}
2107		log.Info().Msg("created network for app isolation")
2108	}
2109
2110	return nil
2111}
2112
2113func (s *Service) cleanupOrphanedContainers(ctx context.Context, domainName string, skipContainerID string) error {
2114	log := zerowrap.FromCtx(ctx)
2115	expectedName := fmt.Sprintf("gordon-%s", domainName)
2116	expectedNewName := expectedName + "-new"
2117	expectedNextName := expectedName + "-next"
2118
2119	allContainers, err := s.runtime.ListContainers(ctx, true)
2120	if err != nil {
2121		return err
2122	}
2123
2124	for _, c := range allContainers {
2125		if (c.Name == expectedName || c.Name == expectedNewName || c.Name == expectedNextName) && c.ID != skipContainerID {
2126			// Skip any running container regardless of name — a running
2127			// temp container may be actively serving traffic while the
2128			// new container stabilizes, or may be from a concurrent deploy.
2129			if c.Status == "running" {
2130				log.Debug().
2131					Str(zerowrap.FieldEntityID, c.ID).
2132					Str("container_name", c.Name).
2133					Str(zerowrap.FieldStatus, c.Status).
2134					Msg("skipping running container during orphan cleanup")
2135				continue
2136			}
2137
2138			log.Info().Str(zerowrap.FieldEntityID, c.ID).Str("container_name", c.Name).Str(zerowrap.FieldStatus, c.Status).Msg("found orphaned container, removing")
2139
2140			if err := s.runtime.StopContainer(ctx, c.ID); err != nil {
2141				log.WrapErrWithFields(err, "failed to stop orphaned container", map[string]any{zerowrap.FieldEntityID: c.ID})
2142			}
2143
2144			if err := s.runtime.RemoveContainer(ctx, c.ID, true); err != nil {
2145				log.WrapErrWithFields(err, "failed to remove orphaned container", map[string]any{zerowrap.FieldEntityID: c.ID})
2146			}
2147		}
2148	}
2149
2150	return nil
2151}
2152
2153func (s *Service) cleanupVolumesForDomain(_ context.Context, _ string) error {
2154	return nil
2155}
2156
2157func (s *Service) cleanupNetworkIfEmpty(ctx context.Context, networkName string) error {
2158	if networkName == "bridge" || networkName == "default" {
2159		return nil
2160	}
2161
2162	ctx = zerowrap.CtxWithField(ctx, "network", networkName)
2163	log := zerowrap.FromCtx(ctx)
2164
2165	networks, err := s.runtime.ListNetworks(ctx)
2166	if err != nil {
2167		return log.WrapErr(err, "failed to list networks")
2168	}
2169
2170	for _, n := range networks {
2171		if n.Name == networkName && len(n.Containers) == 0 {
2172			if err := s.runtime.RemoveNetwork(ctx, networkName); err != nil {
2173				return err
2174			}
2175			log.Info().Msg("cleaned up empty network")
2176			break
2177		}
2178	}
2179
2180	return nil
2181}
2182
2183func (s *Service) deployAttachments(ctx context.Context, domainName, networkName string) error {
2184	// Collect attachments from both domain-specific and network group configs
2185	attachments := s.resolveAttachmentsForDomain(domainName)
2186	if len(attachments) == 0 {
2187		return nil
2188	}
2189
2190	log := zerowrap.FromCtx(ctx)
2191	var deployed []string // track successfully deployed attachment IDs for rollback
2192	for _, svc := range attachments {
2193		if err := s.deployAttachedService(ctx, domainName, svc, networkName); err != nil {
2194			log.WrapErrWithFields(err, "failed to deploy attachment", map[string]any{zerowrap.FieldService: svc, "domain": domainName})
2195
2196			// Rollback: clean up already-deployed attachments
2197			for _, id := range deployed {
2198				if stopErr := s.runtime.StopContainer(ctx, id); stopErr != nil {
2199					log.Warn().Err(stopErr).Str("attachment_id", id).Msg("rollback: failed to stop attachment")
2200				}
2201				if rmErr := s.runtime.RemoveContainer(ctx, id, true); rmErr != nil {
2202					log.Warn().Err(rmErr).Str("attachment_id", id).Msg("rollback: failed to remove attachment")
2203				}
2204			}
2205			// Deregister rolled-back attachments
2206			if len(deployed) > 0 {
2207				s.mu.Lock()
2208				remaining := s.attachments[domainName]
2209				filtered := remaining[:0]
2210				rollbackSet := make(map[string]bool, len(deployed))
2211				for _, id := range deployed {
2212					rollbackSet[id] = true
2213				}
2214				for _, id := range remaining {
2215					if !rollbackSet[id] {
2216						filtered = append(filtered, id)
2217					}
2218				}
2219				s.attachments[domainName] = filtered
2220				s.mu.Unlock()
2221			}
2222
2223			return fmt.Errorf("failed to deploy attachment %q (rolled back %d already-deployed)", svc, len(deployed))
2224		}
2225		// Collect IDs of attachments tracked under this domain after successful deploy
2226		s.mu.RLock()
2227		ids := s.attachments[domainName]
2228		if len(ids) > 0 {
2229			latest := ids[len(ids)-1]
2230			deployed = append(deployed, latest)
2231		}
2232		s.mu.RUnlock()
2233	}
2234	return nil
2235}
2236
2237// resolveAttachmentsForDomain returns attachments for a domain by checking:
2238// 1. Direct domain attachments (attachments[domain])
2239// 2. Network group attachments (attachments[group] where domain is in network_groups[group])
2240func (s *Service) resolveAttachmentsForDomain(domainName string) []string {
2241	var attachments map[string][]string
2242	var networkGroups map[string][]string
2243
2244	if s.configProvider != nil {
2245		snap := s.configProvider.GetAttachmentConfig()
2246		attachments = snap.Attachments
2247		networkGroups = snap.NetworkGroups
2248	} else {
2249		s.mu.RLock()
2250		attachments = s.config.Attachments
2251		networkGroups = s.config.NetworkGroups
2252		s.mu.RUnlock()
2253	}
2254
2255	seen := make(map[string]bool)
2256	var result []string
2257
2258	// First, add domain-specific attachments
2259	if domainAttachments, ok := attachments[domainName]; ok {
2260		for _, img := range domainAttachments {
2261			if !seen[img] {
2262				seen[img] = true
2263				result = append(result, img)
2264			}
2265		}
2266	}
2267
2268	// Then, find which network group this domain belongs to and add group attachments
2269	for groupName, domains := range networkGroups {
2270		if slices.Contains(domains, domainName) {
2271			if groupAttachments, ok := attachments[groupName]; ok {
2272				for _, img := range groupAttachments {
2273					if !seen[img] {
2274						seen[img] = true
2275						result = append(result, img)
2276					}
2277				}
2278			}
2279			break // Domain can only be in one network group
2280		}
2281	}
2282
2283	return result
2284}
2285
2286func (s *Service) getAttachmentsForDomain(ctx context.Context, domainName string) []domain.Attachment {
2287	containers, err := s.runtime.ListContainers(ctx, true)
2288	if err != nil {
2289		return nil
2290	}
2291
2292	return s.filterAttachments(ctx, containers, domainName)
2293}
2294
2295// getAllAttachments fetches all containers once and returns attachments grouped by domain.
2296// This avoids N+1 queries when listing multiple routes.
2297func (s *Service) getAllAttachments(ctx context.Context) map[string][]domain.Attachment {
2298	containers, err := s.runtime.ListContainers(ctx, true)
2299	if err != nil {
2300		return nil
2301	}
2302
2303	result := make(map[string][]domain.Attachment)
2304	for _, container := range containers {
2305		if container.Labels == nil {
2306			continue
2307		}
2308		if container.Labels[domain.LabelAttachment] != "true" {
2309			continue
2310		}
2311		ownerDomain := container.Labels[domain.LabelAttachedTo]
2312		if ownerDomain == "" {
2313			continue
2314		}
2315		image := container.Image
2316		serviceName := container.Name
2317		if labelImage, ok := container.Labels[domain.LabelImage]; ok && labelImage != "" {
2318			image = labelImage
2319			serviceName = extractServiceName(labelImage)
2320		}
2321		// Get the container's network for display in the routes table
2322		network := ""
2323		if networkName, err := s.runtime.GetContainerNetwork(ctx, container.ID); err == nil {
2324			network = networkName
2325		}
2326		attachment := domain.Attachment{
2327			Name:        serviceName,
2328			Image:       s.stripRegistryPrefix(image),
2329			ContainerID: container.ID,
2330			Status:      container.Status,
2331			Network:     network,
2332			Ports:       append([]int(nil), container.Ports...),
2333		}
2334		result[ownerDomain] = append(result[ownerDomain], attachment)
2335	}
2336
2337	return result
2338}
2339
2340// filterAttachments extracts attachments for a specific domain from a container list.
2341func (s *Service) filterAttachments(ctx context.Context, containers []*domain.Container, domainName string) []domain.Attachment {
2342	attachments := make([]domain.Attachment, 0)
2343	for _, container := range containers {
2344		if container.Labels == nil {
2345			continue
2346		}
2347		if container.Labels[domain.LabelAttachment] != "true" {
2348			continue
2349		}
2350		if container.Labels[domain.LabelAttachedTo] != domainName {
2351			continue
2352		}
2353		image := container.Image
2354		serviceName := container.Name
2355		if labelImage, ok := container.Labels[domain.LabelImage]; ok && labelImage != "" {
2356			image = labelImage
2357			serviceName = extractServiceName(labelImage)
2358		}
2359		// Get the container's network for display in the routes table
2360		network := ""
2361		if networkName, err := s.runtime.GetContainerNetwork(ctx, container.ID); err == nil {
2362			network = networkName
2363		}
2364		attachment := domain.Attachment{
2365			Name:        serviceName,
2366			Image:       s.stripRegistryPrefix(image),
2367			ContainerID: container.ID,
2368			Status:      container.Status,
2369			Network:     network,
2370			Ports:       append([]int(nil), container.Ports...),
2371		}
2372		attachments = append(attachments, attachment)
2373	}
2374
2375	return attachments
2376}
2377
2378func (s *Service) deployAttachedService(ctx context.Context, ownerDomain, serviceImage, networkName string) error {
2379	log := zerowrap.FromCtx(ctx)
2380
2381	// Parse service name from image (e.g., "my-postgres:latest" → "postgres")
2382	serviceName := extractServiceName(serviceImage)
2383	containerName := fmt.Sprintf("gordon-%s-%s", sanitizeName(ownerDomain), serviceName)
2384
2385	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
2386		"attachment":     serviceName,
2387		"container_name": containerName,
2388		"owner_domain":   ownerDomain,
2389	})
2390
2391	// Check if already running (idempotent)
2392	// Try new name first, then fall back to legacy name for backwards compatibility
2393	existingContainer, err := s.resolveExistingAttachment(ctx, ownerDomain, serviceName)
2394	if err != nil {
2395		return err
2396	}
2397
2398	if existingContainer != nil {
2399		shouldSkip, err := s.handleRunningAttachment(ctx, existingContainer, containerName, serviceImage)
2400		if err != nil {
2401			return err
2402		}
2403		if shouldSkip {
2404			return nil
2405		}
2406		if existingContainer.Status == string(domain.ContainerStatusRunning) {
2407			existingContainer = nil
2408		}
2409	}
2410
2411	if err := s.removeStoppedAttachment(ctx, existingContainer, containerName); err != nil {
2412		return err
2413	}
2414
2415	log.Info().Str(zerowrap.FieldService, serviceImage).Msg("deploying attached service")
2416
2417	// Ensure image (canonical ref is returned; internal pulls may use the local registry).
2418	imageRef := s.buildImageRef(serviceImage)
2419	actualImageRef, err := s.ensureImage(ctx, imageRef)
2420	if err != nil {
2421		return err
2422	}
2423
2424	// Get image metadata
2425	exposedPorts, err := s.runtime.GetImageExposedPorts(ctx, actualImageRef)
2426	if err != nil {
2427		log.WrapErr(err, "failed to get exposed ports for attachment, using defaults")
2428		exposedPorts = []int{}
2429	}
2430
2431	// Setup volumes (attachments need persistent data)
2432	volumes, err := s.setupVolumes(ctx, containerName, actualImageRef)
2433	if err != nil {
2434		log.WrapErr(err, "failed to setup volumes for attachment")
2435		volumes = make(map[string]string)
2436	}
2437
2438	// Load environment (attachment-specific env file)
2439	envVars, err := s.loadEnvironment(ctx, nil, containerName, actualImageRef)
2440	if err != nil {
2441		log.WrapErr(err, "failed to load environment for attachment")
2442		envVars = []string{}
2443	}
2444	envHash := hashEnvironment(envVars)
2445
2446	s.mu.RLock()
2447	cfg := s.config
2448	s.mu.RUnlock()
2449
2450	// Create container with attachment labels (use actual ref, track original in labels)
2451	config := &domain.ContainerConfig{
2452		Image:       actualImageRef,
2453		Name:        containerName,
2454		Hostname:    serviceName, // Internal DNS: postgres, redis, etc.
2455		Aliases:     []string{serviceName},
2456		Ports:       exposedPorts,
2457		Env:         envVars,
2458		Volumes:     volumes,
2459		NetworkMode: networkName, // Same network as main app
2460		Labels: map[string]string{
2461			domain.LabelManaged:    "true",
2462			domain.LabelAttachment: "true",
2463			domain.LabelAttachedTo: ownerDomain,
2464			domain.LabelEnvHash:    envHash,
2465			domain.LabelImage:      serviceImage,
2466		},
2467		MemoryLimit: cfg.DefaultMemoryLimit,
2468		NanoCPUs:    cfg.DefaultNanoCPUs,
2469		PidsLimit:   cfg.DefaultPidsLimit,
2470	}
2471
2472	container, err := s.runtime.CreateContainer(ctx, config)
2473	if err != nil {
2474		return log.WrapErr(err, "failed to create attachment container")
2475	}
2476
2477	// Start container
2478	if err := s.runtime.StartContainer(ctx, container.ID); err != nil {
2479		s.runtime.RemoveContainer(ctx, container.ID, true)
2480		return log.WrapErr(err, "failed to start attachment container")
2481	}
2482
2483	// Wait for attachment to be ready before proceeding
2484	if err := s.waitForAttachmentReady(ctx, container.ID, config); err != nil {
2485		log.WrapErr(err, "attachment readiness check failed, cleaning up")
2486		if stopErr := s.runtime.StopContainer(ctx, container.ID); stopErr != nil {
2487			log.Warn().Err(stopErr).Str(zerowrap.FieldEntityID, container.ID).Msg("failed to stop unready attachment")
2488		}
2489		s.runtime.RemoveContainer(ctx, container.ID, true)
2490		return fmt.Errorf("attachment %q not ready: %w", serviceName, err)
2491	}
2492
2493	// Track attachment
2494	s.mu.Lock()
2495	s.attachments[ownerDomain] = append(s.attachments[ownerDomain], container.ID)
2496	s.mu.Unlock()
2497
2498	// Start log collection for attachment
2499	s.startLogCollection(ctx, container.ID, containerName)
2500
2501	log.Info().Str(zerowrap.FieldEntityID, container.ID).Msg("attachment deployed successfully")
2502	return nil
2503}
2504
2505func (s *Service) attachmentEnvDrifted(ctx context.Context, existing *domain.Container, containerName, serviceImage string) (bool, error) {
2506	currentEnv, err := s.loadEnvironment(ctx, nil, containerName, s.buildImageRef(serviceImage))
2507	if err != nil {
2508		return false, err
2509	}
2510
2511	currentHash := hashEnvironment(currentEnv)
2512	existingHash, ok := existing.Labels[domain.LabelEnvHash]
2513	if ok && existingHash == currentHash {
2514		return false, nil
2515	}
2516
2517	return true, nil
2518}
2519
2520func (s *Service) handleRunningAttachment(ctx context.Context, existing *domain.Container, containerName, serviceImage string) (bool, error) {
2521	if existing.Status != string(domain.ContainerStatusRunning) {
2522		return false, nil
2523	}
2524
2525	log := zerowrap.FromCtx(ctx)
2526
2527	// Check image drift
2528	existingImage := existing.Labels[domain.LabelImage]
2529	imageDrifted := existingImage != "" && existingImage != serviceImage
2530
2531	// Check env drift
2532	envDrifted, err := s.attachmentEnvDrifted(ctx, existing, containerName, serviceImage)
2533	if err != nil {
2534		return false, log.WrapErr(err, "failed to check attachment env drift")
2535	}
2536
2537	if !envDrifted && !imageDrifted {
2538		log.Debug().Str("container_name", containerName).Msg("attachment already running with current env and image, skipping")
2539		return true, nil
2540	}
2541
2542	if imageDrifted {
2543		log.Info().Str("container_name", containerName).Str("existing_image", existingImage).Str("desired_image", serviceImage).Msg("attachment image changed, recreating")
2544	}
2545	if envDrifted {
2546		log.Info().Str("container_name", containerName).Msg("attachment env changed, recreating")
2547	}
2548
2549	if err := s.runtime.StopContainer(ctx, existing.ID); err != nil {
2550		return false, log.WrapErr(err, "failed to stop attachment for drift update")
2551	}
2552	if err := s.runtime.RemoveContainer(ctx, existing.ID, true); err != nil {
2553		return false, log.WrapErr(err, "failed to remove attachment for drift update")
2554	}
2555
2556	return false, nil
2557}
2558
2559func (s *Service) removeStoppedAttachment(ctx context.Context, existing *domain.Container, containerName string) error {
2560	if existing == nil {
2561		return nil
2562	}
2563
2564	log := zerowrap.FromCtx(ctx)
2565	log.Info().Str("container_name", containerName).Msg("removing stopped attachment container")
2566	if err := s.runtime.RemoveContainer(ctx, existing.ID, true); err != nil {
2567		return log.WrapErr(err, "failed to remove existing attachment container")
2568	}
2569
2570	return nil
2571}
2572
2573func (s *Service) resolveExistingAttachment(ctx context.Context, ownerDomain, serviceName string) (*domain.Container, error) {
2574	log := zerowrap.FromCtx(ctx)
2575	containerName := fmt.Sprintf("gordon-%s-%s", sanitizeName(ownerDomain), serviceName)
2576	existingContainer := s.findContainerByName(ctx, containerName)
2577	if existingContainer != nil {
2578		return existingContainer, nil
2579	}
2580
2581	containerNameLegacy := fmt.Sprintf("gordon-%s-%s", sanitizeNameLegacy(ownerDomain), serviceName)
2582	existingContainer = s.findContainerByName(ctx, containerNameLegacy)
2583	if existingContainer == nil {
2584		return nil, nil
2585	}
2586
2587	log.Info().Str("container_name", containerNameLegacy).Msg("found attachment with legacy naming, will be replaced")
2588	if err := s.runtime.StopContainer(ctx, existingContainer.ID); err != nil {
2589		return nil, log.WrapErr(err, "failed to stop legacy attachment container")
2590	}
2591	if err := s.runtime.RemoveContainer(ctx, existingContainer.ID, true); err != nil {
2592		return nil, log.WrapErr(err, "failed to remove legacy attachment container")
2593	}
2594
2595	return nil, nil
2596}
2597
2598// Utility functions
2599
2600func normalizeImageRef(image string) string {
2601	// Extract repository from image reference, handling port numbers correctly.
2602	// Examples:
2603	//   nginx:latest -> docker.io/library/nginx
2604	//   user/repo:tag -> docker.io/user/repo
2605	//   localhost:5000/image:tag -> localhost:5000/image
2606	//   registry.example.com/image:tag -> registry.example.com/image
2607	//
2608	// Note: This function assumes valid Docker image references. Edge cases like
2609	// "registry.com:5000" (registry with port but no image name) are not valid
2610	// image references per Docker's naming conventions, which require at least
2611	// one path component after the registry (e.g., "registry.com:5000/image").
2612
2613	// Find the tag separator: the last colon that isn't part of a port number.
2614	// A colon is part of a port if there's no slash after it until the next colon.
2615	repo := image
2616	lastColon := strings.LastIndex(image, ":")
2617	if lastColon != -1 {
2618		afterColon := image[lastColon+1:]
2619		// If there's no slash after the colon, it's the tag separator
2620		if !strings.Contains(afterColon, "/") {
2621			repo = image[:lastColon]
2622		}
2623	}
2624
2625	if !strings.Contains(repo, "/") {
2626		return "docker.io/library/" + repo
2627	}
2628
2629	if strings.Count(repo, "/") == 1 && !strings.Contains(strings.Split(repo, "/")[0], ".") && !strings.Contains(strings.Split(repo, "/")[0], ":") {
2630		return "docker.io/" + repo
2631	}
2632
2633	return repo
2634}
2635
2636func generateVolumeName(prefix, domainName, volumePath string) string {
2637	return fmt.Sprintf("%s-%s-%s",
2638		prefix,
2639		strings.ReplaceAll(domainName, ".", "-"),
2640		strings.ReplaceAll(strings.Trim(volumePath, "/"), "/", "-"))
2641}
2642
2643func mergeEnvironmentVariables(dockerfileEnv, userEnv []string) []string {
2644	envMap := make(map[string]string)
2645
2646	for _, env := range dockerfileEnv {
2647		if k, v, ok := strings.Cut(env, "="); ok {
2648			envMap[k] = v
2649		}
2650	}
2651
2652	for _, env := range userEnv {
2653		if k, v, ok := strings.Cut(env, "="); ok {
2654			envMap[k] = v
2655		}
2656	}
2657
2658	keys := make([]string, 0, len(envMap))
2659	for k := range envMap {
2660		keys = append(keys, k)
2661	}
2662	slices.Sort(keys)
2663
2664	result := make([]string, 0, len(envMap))
2665	for _, k := range keys {
2666		result = append(result, k+"="+envMap[k])
2667	}
2668
2669	return result
2670}
2671
2672// extractServiceName gets service name from image reference.
2673// "my-postgres:latest" → "postgres", "redis:7" → "redis"
2674func extractServiceName(image string) string {
2675	// Remove tag
2676	parts := strings.Split(image, ":")
2677	name := parts[0]
2678
2679	// Remove registry prefix if present
2680	if strings.Contains(name, "/") {
2681		nameParts := strings.Split(name, "/")
2682		name = nameParts[len(nameParts)-1]
2683	}
2684
2685	// Remove common prefixes like "my-"
2686	name = strings.TrimPrefix(name, "my-")
2687
2688	return name
2689}
2690
2691// sanitizeName is a convenience wrapper around domain.SanitizeDomainForContainer.
2692func sanitizeName(d string) string {
2693	return domain.SanitizeDomainForContainer(d)
2694}
2695
2696// sanitizeNameLegacy is a convenience wrapper around domain.SanitizeDomainForContainerLegacy.
2697func sanitizeNameLegacy(d string) string {
2698	return domain.SanitizeDomainForContainerLegacy(d)
2699}
2700
2701func isContainerNotFoundError(err error) bool {
2702	if err == nil {
2703		return false
2704	}
2705	if errors.Is(err, domain.ErrContainerNotFound) {
2706		return true
2707	}
2708	msg := strings.ToLower(err.Error())
2709	return strings.Contains(msg, "no such container") ||
2710		strings.Contains(msg, "no container with name or id") ||
2711		strings.Contains(msg, "container not found")
2712}
2713
2714func isConnectionRefusedError(err error) bool {
2715	if err == nil {
2716		return false
2717	}
2718	return strings.Contains(strings.ToLower(err.Error()), "connection refused")
2719}
2720
2721// findContainerByName finds a container by its name.
2722func (s *Service) findContainerByName(ctx context.Context, name string) *domain.Container {
2723	containers, err := s.runtime.ListContainers(ctx, true)
2724	if err != nil {
2725		return nil
2726	}
2727
2728	for _, c := range containers {
2729		if c.Name == name {
2730			return c
2731		}
2732	}
2733	return nil
2734}
2735
2736func (s *Service) waitForReady(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig) error {
2737	if err := s.pollContainerRunning(ctx, containerID); err != nil {
2738		return err
2739	}
2740
2741	s.mu.RLock()
2742	cfg := s.config
2743	s.mu.RUnlock()
2744
2745	readinessMode := cfg.ReadinessMode
2746	if readinessMode == "" {
2747		readinessMode = "delay"
2748	}
2749
2750	switch readinessMode {
2751	case "delay":
2752		return s.waitForReadyByDelay(ctx, containerID)
2753	case "docker-health":
2754		_, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(ctx, containerID)
2755		if err != nil {
2756			return err
2757		}
2758		if !hasHealthcheck {
2759			return errors.New("no healthcheck detected")
2760		}
2761		return s.waitForHealthy(ctx, containerID, cfg.HealthTimeout)
2762	default: // auto
2763		return s.readinessCascade(ctx, containerID, containerConfig, cfg)
2764	}
2765}
2766
2767// readinessCascade auto-detects the strongest available readiness signal:
2768//  1. Docker healthcheck (if present) → wait for healthy status
2769//  2. HTTP probe (if gordon.health label set) → GET until 2xx/3xx
2770//  3. Default HTTP probe (GET / on exposed port) → wait for 2xx/3xx
2771//  4. TCP probe (if port info available) → connect until accepted
2772//  5. Delay fallback (last resort) → waitForReadyByDelay
2773func (s *Service) readinessCascade(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) error {
2774	log := zerowrap.FromCtx(ctx)
2775
2776	// 1. Docker healthcheck
2777	_, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(ctx, containerID)
2778	if err != nil {
2779		return err
2780	}
2781	if hasHealthcheck {
2782		log.Info().Msg("readiness cascade: using Docker healthcheck")
2783		return s.waitForHealthy(ctx, containerID, cfg.HealthTimeout)
2784	}
2785
2786	// 2. HTTP probe via gordon.health label
2787	if probed, probeErr := s.tryHTTPProbe(ctx, containerID, containerConfig, cfg); probed {
2788		return probeErr
2789	}
2790
2791	// 3. Default HTTP probe on root path (if port info available)
2792	if probed, probeErr := s.tryDefaultHTTPProbe(ctx, containerID, containerConfig, cfg); probed {
2793		return probeErr
2794	}
2795
2796	// 4. TCP probe (if port info available)
2797	if probed, probeErr := s.tryTCPProbe(ctx, containerID, containerConfig, cfg); probed {
2798		return probeErr
2799	}
2800
2801	// 5. Delay fallback
2802	log.Info().Msg("readiness cascade: using delay fallback")
2803	return s.waitForReadyByDelay(ctx, containerID)
2804}
2805
2806// waitForAttachmentReady runs a reduced readiness cascade for attachments:
2807// Docker healthcheck → TCP probe → delay fallback.
2808// HTTP probes are skipped — attachments are typically databases/caches.
2809func (s *Service) waitForAttachmentReady(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig) error {
2810	if err := s.pollContainerRunning(ctx, containerID); err != nil {
2811		return err
2812	}
2813
2814	log := zerowrap.FromCtx(ctx)
2815
2816	s.mu.RLock()
2817	cfg := s.config
2818	s.mu.RUnlock()
2819
2820	// 1. Docker healthcheck (if present)
2821	_, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(ctx, containerID)
2822	if err != nil {
2823		return err
2824	}
2825	if hasHealthcheck {
2826		log.Info().Msg("attachment readiness: using Docker healthcheck")
2827		timeout := cfg.HealthTimeout
2828		if timeout == 0 {
2829			timeout = 90 * time.Second
2830		}
2831		return s.waitForHealthy(ctx, containerID, timeout)
2832	}
2833
2834	// 2. TCP probe (if port info available)
2835	if containerConfig != nil && len(containerConfig.Ports) > 0 {
2836		ip, port, probeErr := s.resolveProbeEndpoint(ctx, containerID, containerConfig)
2837		if probeErr == nil && ip != "" && port > 0 {
2838			addr := fmt.Sprintf("%s:%d", ip, port)
2839			timeout := cfg.AttachmentReadinessTimeout
2840			if timeout == 0 {
2841				timeout = 30 * time.Second
2842			}
2843			log.Info().Str("addr", addr).Dur("timeout", timeout).Msg("attachment readiness: using TCP probe")
2844			return tcpProbe(ctx, addr, timeout)
2845		}
2846		log.Debug().Err(probeErr).Msg("attachment readiness: TCP probe skipped, could not resolve endpoint")
2847	}
2848
2849	// 3. Delay fallback
2850	log.Info().Msg("attachment readiness: using delay fallback")
2851	return s.waitForReadyByDelay(ctx, containerID)
2852}
2853
2854// tryHTTPProbe attempts an HTTP probe if the gordon.health label is set.
2855// Returns (true, err) if the probe was attempted, (false, nil) if skipped.
2856func (s *Service) tryHTTPProbe(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) (bool, error) {
2857	log := zerowrap.FromCtx(ctx)
2858	if containerConfig == nil {
2859		return false, nil
2860	}
2861	healthPath, ok := containerConfig.Labels[domain.LabelHealth]
2862	if !ok || healthPath == "" {
2863		return false, nil
2864	}
2865	ip, port, probeErr := s.resolveProbeEndpoint(ctx, containerID, containerConfig)
2866	if probeErr != nil || ip == "" || port <= 0 {
2867		log.Debug().Err(probeErr).Msg("readiness cascade: HTTP probe skipped, could not resolve container endpoint")
2868		return false, nil
2869	}
2870	url := fmt.Sprintf("http://%s:%d%s", ip, port, healthPath)
2871	timeout := cfg.HTTPProbeTimeout
2872	if timeout == 0 {
2873		timeout = 60 * time.Second
2874	}
2875	log.Info().Str("url", url).Dur("timeout", timeout).Msg("readiness cascade: using HTTP probe")
2876	return true, httpProbe(ctx, url, timeout)
2877}
2878
2879// tryDefaultHTTPProbe attempts an HTTP GET on "/" using the container's
2880// exposed port. This is the default readiness check for web containers —
2881// it verifies the app is actually serving HTTP, not just accepting TCP.
2882// Returns (true, err) if the probe was attempted, (false, nil) if skipped.
2883func (s *Service) tryDefaultHTTPProbe(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) (bool, error) {
2884	log := zerowrap.FromCtx(ctx)
2885	if containerConfig == nil || len(containerConfig.Ports) == 0 {
2886		return false, nil
2887	}
2888	ip, port, probeErr := s.resolveProbeEndpoint(ctx, containerID, containerConfig)
2889	if probeErr != nil || ip == "" || port <= 0 {
2890		log.Debug().Err(probeErr).Msg("readiness cascade: default HTTP probe skipped, could not resolve container endpoint")
2891		return false, nil
2892	}
2893	url := fmt.Sprintf("http://%s:%d/", ip, port)
2894	timeout := cfg.HTTPProbeTimeout
2895	if timeout == 0 {
2896		timeout = 60 * time.Second
2897	}
2898	log.Info().Str("url", url).Dur("timeout", timeout).Msg("readiness cascade: using default HTTP alive probe")
2899	return true, httpAliveProbe(ctx, url, timeout)
2900}
2901
2902// tryTCPProbe attempts a TCP probe if port info is available.
2903// Returns (true, err) if the probe was attempted, (false, nil) if skipped.
2904func (s *Service) tryTCPProbe(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) (bool, error) {
2905	log := zerowrap.FromCtx(ctx)
2906	if containerConfig == nil || len(containerConfig.Ports) == 0 {
2907		return false, nil
2908	}
2909	ip, port, probeErr := s.resolveProbeEndpoint(ctx, containerID, containerConfig)
2910	if probeErr != nil || ip == "" || port <= 0 {
2911		log.Debug().Err(probeErr).Msg("readiness cascade: TCP probe skipped, could not resolve container endpoint")
2912		return false, nil
2913	}
2914	addr := fmt.Sprintf("%s:%d", ip, port)
2915	timeout := cfg.TCPProbeTimeout
2916	if timeout == 0 {
2917		timeout = 30 * time.Second
2918	}
2919	log.Info().Str("addr", addr).Dur("timeout", timeout).Msg("readiness cascade: using TCP probe")
2920	return true, tcpProbe(ctx, addr, timeout)
2921}
2922
2923// resolveContainerEndpoint returns a host-reachable address for probing.
2924// In rootless podman/Docker setups, container internal IPs are not routable
2925// from the host. We use the host port binding (127.0.0.1:<mapped_port>)
2926// which is always reachable. Falls back to the container's internal IP
2927// only if no host port mapping exists (e.g. host-network mode).
2928func (s *Service) resolveProbeEndpoint(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig) (string, int, error) {
2929	log := zerowrap.FromCtx(ctx)
2930
2931	internalPort := 0
2932	if containerConfig != nil {
2933		if labels := containerConfig.Labels; labels != nil {
2934			for _, key := range []string{domain.LabelProxyPort, domain.LabelPort} {
2935				if portStr, ok := labels[key]; ok && portStr != "" {
2936					port, err := strconv.Atoi(portStr)
2937					if err == nil && port > 0 && port <= 65535 {
2938						internalPort = port
2939						break
2940					}
2941					log.Warn().Str("label", key).Str("port_value", portStr).Msg("invalid probe port label value")
2942				}
2943			}
2944		}
2945	}
2946
2947	if internalPort == 0 {
2948		_, resolvedPort, err := s.runtime.GetContainerNetworkInfo(ctx, containerID)
2949		if err != nil {
2950			return "", 0, err
2951		}
2952		internalPort = resolvedPort
2953	}
2954
2955	hostPort, hostErr := s.runtime.GetContainerPort(ctx, containerID, internalPort)
2956	if hostErr == nil && hostPort > 0 {
2957		log.Debug().
2958			Int("internal_port", internalPort).
2959			Int("host_port", hostPort).
2960			Msg("resolved container endpoint via host port binding")
2961		return "127.0.0.1", hostPort, nil
2962	}
2963
2964	ip, _, fallbackErr := s.runtime.GetContainerNetworkInfo(ctx, containerID)
2965	if fallbackErr != nil {
2966		return "", 0, fallbackErr
2967	}
2968	log.Debug().
2969		Str("ip", ip).
2970		Int("port", internalPort).
2971		Msg("resolved container endpoint via internal IP (no host port binding)")
2972	return ip, internalPort, nil
2973}
2974
2975// waitForReadyByDelay waits using the legacy running+delay strategy.
2976func (s *Service) waitForReadyByDelay(ctx context.Context, containerID string) error {
2977	log := zerowrap.FromCtx(ctx)
2978
2979	s.mu.RLock()
2980	cfg := s.config
2981	s.mu.RUnlock()
2982
2983	delay := cfg.ReadinessDelay
2984	if delay == 0 {
2985		delay = 5 * time.Second
2986	}
2987
2988	log.Debug().Dur("delay", delay).Msg("waiting for container readiness")
2989	select {
2990	case <-time.After(delay):
2991	case <-ctx.Done():
2992		return ctx.Err()
2993	}
2994
2995	// Verify still running after delay
2996	running, err := s.runtime.IsContainerRunning(ctx, containerID)
2997	if err != nil {
2998		return err
2999	}
3000	if running {
3001		return nil
3002	}
3003
3004	return s.waitForRecovery(ctx, containerID)
3005}
3006
3007func (s *Service) waitForHealthy(ctx context.Context, containerID string, timeout time.Duration) error {
3008	if timeout == 0 {
3009		timeout = 90 * time.Second
3010	}
3011	waitCtx, cancel := context.WithTimeout(ctx, timeout)
3012	defer cancel()
3013
3014	var lastStatus string
3015	for {
3016		status, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(waitCtx, containerID)
3017		if err != nil {
3018			return err
3019		}
3020		if !hasHealthcheck {
3021			return errors.New("no healthcheck detected")
3022		}
3023		// Treat empty health status as transitional startup. Some runtimes can
3024		// temporarily report an empty status before first probe results.
3025		if status == "" {
3026			status = "starting"
3027		}
3028		if status == "healthy" {
3029			return nil
3030		}
3031		if status == "unhealthy" {
3032			return errors.New("container reported unhealthy")
3033		}
3034		lastStatus = status
3035
3036		select {
3037		case <-time.After(time.Second):
3038		case <-waitCtx.Done():
3039			return fmt.Errorf("container healthcheck timeout after %s (last status: %s)", timeout, lastStatus)
3040		}
3041	}
3042}
3043
3044// pollContainerRunning polls until the container is running (max 30 seconds).
3045func (s *Service) pollContainerRunning(ctx context.Context, containerID string) error {
3046	for i := 0; i < 30; i++ {
3047		running, err := s.runtime.IsContainerRunning(ctx, containerID)
3048		if err != nil {
3049			return err
3050		}
3051		if running {
3052			return nil
3053		}
3054		// Check if the container has already exited (crash) rather than still starting up.
3055		if info, inspectErr := s.runtime.InspectContainer(ctx, containerID); inspectErr == nil && info != nil && info.Status == string(domain.ContainerStatusExited) {
3056			return fmt.Errorf("container exited immediately with code %d: %w", info.ExitCode, domain.ErrContainerExited)
3057		}
3058		if i == 29 {
3059			return fmt.Errorf("container did not start within 30 seconds")
3060		}
3061		select {
3062		case <-time.After(time.Second):
3063		case <-ctx.Done():
3064			return ctx.Err()
3065		}
3066	}
3067	return fmt.Errorf("container did not start within 30 seconds")
3068}
3069
3070// waitForRecovery waits for a container to recover within the recovery window.
3071func (s *Service) waitForRecovery(ctx context.Context, containerID string) error {
3072	log := zerowrap.FromCtx(ctx)
3073	log.Warn().
3074		Dur("recovery_window", readinessRecoveryWindow).
3075		Msg("container not running after readiness delay, waiting for recovery")
3076
3077	deadline := time.Now().Add(readinessRecoveryWindow)
3078	for time.Now().Before(deadline) {
3079		select {
3080		case <-time.After(time.Second):
3081		case <-ctx.Done():
3082			return ctx.Err()
3083		}
3084
3085		running, err := s.runtime.IsContainerRunning(ctx, containerID)
3086		if err != nil {
3087			return err
3088		}
3089		if running {
3090			log.Info().Msg("container recovered during readiness recovery window")
3091			return nil
3092		}
3093	}
3094
3095	return fmt.Errorf("container not running after readiness delay and recovery window")
3096}
3097
3098// publishContainerDeployed publishes a container.deployed event.
3099func (s *Service) publishContainerDeployed(ctx context.Context, domainName, containerID string) {
3100	payload := &domain.ContainerEventPayload{
3101		ContainerID: containerID,
3102		Domain:      domainName,
3103		Action:      "deployed",
3104	}
3105
3106	if err := s.eventBus.Publish(domain.EventContainerDeployed, payload); err != nil {
3107		log := zerowrap.FromCtx(ctx)
3108		log.Warn().Err(err).Msg("failed to publish container deployed event")
3109	}
3110}
3111
3112// rewriteToRegistryDomain rewrites an image reference to use the configured registry domain.
3113// e.g., "myapp:latest" -> "registry.example.com/myapp:latest"
3114func rewriteToRegistryDomain(imageRef, registryDomain string) string {
3115	if registryDomain == "" {
3116		return imageRef
3117	}
3118
3119	prefix := registryDomain + "/"
3120	if strings.HasPrefix(imageRef, prefix) {
3121		return imageRef
3122	}
3123
3124	return prefix + imageRef
3125}
3126
3127// rewriteToLocalRegistry rewrites an image reference to use the local registry address.
3128// e.g., "registry.example.com/myapp:latest" -> "localhost:5000/myapp:latest"
3129func rewriteToLocalRegistry(imageRef, registryDomain string, registryPort int) string {
3130	if imageRef == "" {
3131		return imageRef
3132	}
3133
3134	localRegistry := fmt.Sprintf("localhost:%d", registryPort)
3135	localPrefix := localRegistry + "/"
3136	imageRef = strings.TrimPrefix(imageRef, localPrefix)
3137
3138	registryDomain = strings.TrimSuffix(registryDomain, "/")
3139	if registryDomain != "" {
3140		prefix := registryDomain + "/"
3141		imageRef = strings.TrimPrefix(imageRef, prefix)
3142	}
3143
3144	return localPrefix + imageRef
3145}