service.go RawHistoryFolder

1// Package container implements the container management use case.
2package container
3
4import (
5	"bufio"
6	"context"
7	"encoding/binary"
8	"encoding/hex"
9	"errors"
10	"fmt"
11	"io"
12	"maps"
13	"net"
14	"slices"
15	"strconv"
16	"strings"
17	"sync"
18	"time"
19
20	"github.com/bnema/zerowrap"
21	"go.opentelemetry.io/otel"
22	"go.opentelemetry.io/otel/attribute"
23	"go.opentelemetry.io/otel/codes"
24	"go.opentelemetry.io/otel/metric"
25	"go.opentelemetry.io/otel/trace"
26
27	"github.com/bnema/gordon/internal/adapters/out/telemetry"
28	"github.com/bnema/gordon/internal/boundaries/out"
29	"github.com/bnema/gordon/internal/domain"
30)
31
32// Config holds configuration needed by the container service.
33type Config struct {
34	RegistryAuthEnabled        bool
35	RegistryDomain             string
36	LegacyRegistryDomains      []string
37	RegistryPort               int
38	ServiceTokenUsername       string
39	ServiceToken               string
40	InternalRegistryUsername   string
41	InternalRegistryPassword   string
42	PullPolicy                 string
43	VolumeAutoCreate           bool
44	VolumePrefix               string
45	VolumePreserve             bool
46	NetworkIsolation           bool
47	NetworkPrefix              string
48	NetworkGroups              map[string][]string
49	NetworkInternal            bool
50	Attachments                map[string][]string
51	AllowedRegistries          []string
52	RequireImageDigest         bool
53	SecurityProfile            string
54	ReadinessDelay             time.Duration // Delay after container starts before considering it ready
55	ReadinessMode              string        // Readiness strategy: auto, docker-health, delay
56	HealthTimeout              time.Duration // Max wait for health-based readiness
57	DrainDelay                 time.Duration // Grace period after cache invalidation before stopping old container
58	DrainDelayConfigured       bool          // True when deploy.drain_delay was explicitly configured
59	DrainMode                  string        // Drain strategy: auto, inflight, delay
60	DrainTimeout               time.Duration // Max wait for in-flight requests to drain
61	StabilizationDelay         time.Duration // Post-switch monitoring window (default 2s)
62	TCPProbeTimeout            time.Duration // TCP probe timeout (default 30s)
63	HTTPProbeTimeout           time.Duration // HTTP probe timeout (default 60s)
64	AttachmentReadinessTimeout time.Duration // Max wait for attachment readiness (default 30s)
65	DefaultMemoryLimit         int64         // Default memory limit in bytes for containers (0 = no limit)
66	DefaultNanoCPUs            int64         // Default CPU quota in nanoseconds for containers (0 = no limit)
67	DefaultPidsLimit           int64         // Default max PIDs for containers (0 = no limit)
68}
69
70var tracer = otel.Tracer("gordon.container")
71
72const (
73	PullPolicyAlways       = "always"
74	PullPolicyIfNotPresent = "if-not-present"
75	PullPolicyIfTagChanged = "if-tag-changed"
76
77	// readinessRecoveryWindow is an additional grace window used when a container
78	// is briefly not running at the end of readiness delay. This avoids false
79	// negatives during short startup flaps.
80	readinessRecoveryWindow = 30 * time.Second
81	internalPullMaxAttempts = 3
82	deployFailureLogLimit   = 20
83)
84
85// Service implements the ContainerService interface.
86type Service struct {
87	runtime          out.ContainerRuntime
88	envLoader        out.EnvLoader
89	eventBus         out.EventPublisher
90	logWriter        out.ContainerLogWriter
91	cacheInvalidator out.ProxyCacheInvalidator
92	drainWaiter      out.ProxyDrainWaiter
93	config           Config
94	configProvider   AttachmentConfigProvider // live config reads for attachments/networks (may be nil)
95	metrics          *telemetry.Metrics
96	containers       map[string]*domain.Container
97	attachments      map[string][]string // ownerDomain → []containerIDs
98	managedCount     int64               // tracks UpDownCounter value for delta computation
99	mu               sync.RWMutex
100	deployMu         sync.Map       // per-domain deploy locks (domain → *domainDeployLock)
101	cleanupWg        sync.WaitGroup // tracks background old-container cleanup goroutines
102	monitor          *Monitor
103}
104
105// domainDeployLock is a context-aware mutex using a buffered channel.
106// It allows Lock() to be interrupted by context cancellation.
107type domainDeployLock struct {
108	ch chan struct{}
109}
110
111// newDomainDeployLock creates a new lock that is initially unlocked.
112func newDomainDeployLock() *domainDeployLock {
113	l := &domainDeployLock{ch: make(chan struct{}, 1)}
114	l.ch <- struct{}{}
115	return l
116}
117
118// Lock acquires the lock or returns an error if the context is cancelled.
119func (l *domainDeployLock) Lock(ctx context.Context) error {
120	// Fast path: honor already-canceled contexts before blocking.
121	if err := ctx.Err(); err != nil {
122		return err
123	}
124	select {
125	case <-ctx.Done():
126		return ctx.Err()
127	case <-l.ch:
128		// Re-check context after acquisition: if canceled while we were
129		// in the select, release the token so the lock isn't consumed.
130		if err := ctx.Err(); err != nil {
131			l.ch <- struct{}{}
132			return err
133		}
134		return nil
135	}
136}
137
138// Unlock releases the lock.
139func (l *domainDeployLock) Unlock() {
140	select {
141	case l.ch <- struct{}{}:
142	default:
143		// This should never happen if Lock/Unlock are used correctly.
144		// Panic to detect double-unlocks or other misuse early.
145		panic("domainDeployLock: double unlock detected")
146	}
147}
148
149// NewService creates a new container service.
150func NewService(
151	runtime out.ContainerRuntime,
152	envLoader out.EnvLoader,
153	eventBus out.EventPublisher,
154	logWriter out.ContainerLogWriter,
155	config Config,
156	configProvider AttachmentConfigProvider,
157) *Service {
158	return &Service{
159		runtime:        runtime,
160		envLoader:      envLoader,
161		eventBus:       eventBus,
162		logWriter:      logWriter,
163		config:         config,
164		configProvider: configProvider,
165		containers:     make(map[string]*domain.Container),
166		attachments:    make(map[string][]string),
167	}
168}
169
170// SetMetrics sets the telemetry metrics for the container service.
171func (s *Service) SetMetrics(m *telemetry.Metrics) {
172	s.metrics = m
173}
174
175// SetProxyCacheInvalidator sets the proxy cache invalidator for synchronous
176// cache invalidation during zero-downtime deployments. This must be called
177// after construction because the proxy service is created after the container
178// service during application initialization.
179func (s *Service) SetProxyCacheInvalidator(inv out.ProxyCacheInvalidator) {
180	s.mu.Lock()
181	s.cacheInvalidator = inv
182	s.mu.Unlock()
183}
184
185// SetProxyDrainWaiter sets the proxy in-flight drain waiter used during
186// zero-downtime replacement before stopping the old container.
187func (s *Service) SetProxyDrainWaiter(waiter out.ProxyDrainWaiter) {
188	s.mu.Lock()
189	s.drainWaiter = waiter
190	s.mu.Unlock()
191}
192
193// domainDeployMu returns a per-domain lock for serializing deploy operations.
194func (s *Service) domainDeployMu(domain string) *domainDeployLock {
195	v, _ := s.deployMu.LoadOrStore(domain, newDomainDeployLock())
196	// Type assertion is safe: LoadOrStore always stores *domainDeployLock values,
197	// and any concurrent LoadOrStore calls will also store *domainDeployLock.
198	return v.(*domainDeployLock)
199}
200
201// acquireDomainDeployLock acquires the per-domain deploy lock with context cancellation support.
202// Returns an unlock function that must be called to release the lock.
203func (s *Service) acquireDomainDeployLock(ctx context.Context, domain string) (func(), error) {
204	lock := s.domainDeployMu(domain)
205	if err := lock.Lock(ctx); err != nil {
206		return nil, err
207	}
208	return func() { lock.Unlock() }, nil
209}
210
211// buildContainerConfig constructs the container configuration for deployment.
212func (s *Service) deploymentContainerName(containerDomain string, existing *domain.Container) string {
213	canonicalName := fmt.Sprintf("gordon-%s", containerDomain)
214	if existing == nil {
215		return canonicalName
216	}
217
218	// Alternate temporary names if the tracked container was left with a temp suffix.
219	// This prevents name collisions after interrupted zero-downtime deploys.
220	newName := canonicalName + "-new"
221	nextName := canonicalName + "-next"
222	switch existing.Name {
223	case newName:
224		return nextName
225	case nextName:
226		return newName
227	default:
228		return newName
229	}
230}
231
232// containerConfigInput groups the parameters for building a container configuration.
233type containerConfigInput struct {
234	Domain       string
235	Image        string
236	ImageRef     string
237	ExposedPorts []int
238	EnvVars      []string
239	EnvHash      string
240	Volumes      map[string]string
241	NetworkName  string
242	ImageLabels  map[string]string
243	Existing     *domain.Container
244}
245
246func (s *Service) buildContainerConfig(in containerConfigInput) *domain.ContainerConfig {
247	s.mu.RLock()
248	cfg := s.config
249	s.mu.RUnlock()
250
251	containerName := s.deploymentContainerName(in.Domain, in.Existing)
252	labels := map[string]string{
253		domain.LabelDomain:  in.Domain,
254		domain.LabelEnvHash: in.EnvHash,
255		domain.LabelImage:   in.Image,
256		domain.LabelManaged: "true",
257		domain.LabelRoute:   in.Domain,
258	}
259
260	// Propagate proxy/port labels from image so readiness probes can find them
261	for _, key := range []string{domain.LabelProxyPort, domain.LabelPort, domain.LabelHealth} {
262		if v, ok := in.ImageLabels[key]; ok && v != "" {
263			labels[key] = v
264		}
265	}
266
267	// Ensure the label-specified proxy port is included in exposed ports
268	// so Docker creates a host port binding for it.
269	ports := in.ExposedPorts
270	if portStr, ok := labels[domain.LabelProxyPort]; ok {
271		if port, err := strconv.Atoi(portStr); err == nil && port > 0 && port <= 65535 {
272			if !slices.Contains(ports, port) {
273				ports = append(ports, port)
274			}
275		}
276	}
277
278	containerConfig := &domain.ContainerConfig{
279		Image:         in.ImageRef,
280		Name:          containerName,
281		Ports:         ports,
282		Env:           in.EnvVars,
283		Volumes:       in.Volumes,
284		NetworkMode:   in.NetworkName,
285		Hostname:      in.Domain,
286		Labels:        labels,
287		AutoRemove:    false,
288		RestartPolicy: domain.RestartPolicyAlways,
289		MemoryLimit:   cfg.DefaultMemoryLimit,
290		NanoCPUs:      cfg.DefaultNanoCPUs,
291		PidsLimit:     cfg.DefaultPidsLimit,
292	}
293	applySecurityProfile(containerConfig, cfg)
294	return containerConfig
295}
296
297// Deploy creates and starts a container for the given route.
298// Implements zero-downtime deployment: new container starts before old one stops.
299func (s *Service) Deploy(ctx context.Context, route domain.Route) (*domain.Container, error) {
300	ctx, span := tracer.Start(ctx, "container.deploy",
301		trace.WithAttributes(
302			attribute.String("domain", route.Domain),
303			attribute.String("image", route.Image),
304		))
305	defer span.End()
306
307	deployStart := time.Now()
308
309	// Serialize deploys for the same domain to prevent race conditions
310	// (e.g. multiple image.pushed events + explicit deploy call from CLI).
311	unlock, err := s.acquireDomainDeployLock(ctx, route.Domain)
312	if err != nil {
313		return nil, err
314	}
315	defer unlock()
316
317	// Enrich context with use case fields for all downstream logs
318	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
319		zerowrap.FieldLayer:   "usecase",
320		zerowrap.FieldUseCase: "Deploy",
321		"domain":              route.Domain,
322	})
323	log := zerowrap.FromCtx(ctx)
324
325	// Record deploy metrics and trace status
326	defer s.recordDeployMetrics(ctx, span, route, deployStart, &err)
327
328	existing, hasExisting := s.resolveExistingContainer(ctx, route.Domain)
329
330	resources, err := s.prepareDeployResources(ctx, route, existing)
331	if err != nil {
332		if deployErr := s.wrapPullDeployFailure(route, existing, err); deployErr != nil {
333			err = deployErr
334			return nil, err
335		}
336		return nil, err
337	}
338
339	// Skip redundant deploy: if the existing container is already running
340	// the exact same image (by Docker image ID), return it immediately.
341	// This prevents the double-deploy caused by the event-based deploy
342	// (triggered by image.pushed) racing with the explicit CLI deploy call.
343	if hasExisting {
344		existingForSkip := existing
345		if existingForSkip.ImageID == "" && existingForSkip.Image != "" && normalizeImageRef(existingForSkip.Image) == normalizeImageRef(route.Image) {
346			existingForSkip = s.containerForRedundantCheck(ctx, existingForSkip)
347		}
348		if existingForSkip.ImageID != "" {
349			if skip, container := s.skipRedundantDeploy(ctx, existingForSkip, resources.actualImageRef, resources.envHash); skip {
350				return container, nil
351			}
352		}
353	}
354
355	newContainer, err := s.createStartedContainer(ctx, route, existing, resources)
356	if err != nil {
357		return nil, err
358	}
359
360	invalidated := s.activateDeployedContainer(ctx, route.Domain, newContainer)
361
362	// Post-switch stabilization: verify new container stays running
363	if hasExisting {
364		stable, stabilizeErr := s.stabilizeNewContainer(ctx, route.Domain, newContainer, existing)
365		if stabilizeErr != nil {
366			// Both old and new containers are dead; assign to named return so
367			// the deferred recordDeployMetrics and span see the failure.
368			err = stabilizeErr
369			return nil, err
370		}
371		if !stable {
372			// Rollback performed — old container is restored
373			return existing, nil
374		}
375	}
376
377	// Finalize old container in the background — the new container is already
378	// serving traffic, so there's no reason to block the deploy response while
379	// waiting for the old container to stop (which can take 20s if the app
380	// doesn't handle SIGTERM).
381	s.cleanupWg.Add(1)
382	go func() {
383		defer s.cleanupWg.Done()
384		s.finalizePreviousContainer(context.WithoutCancel(ctx), route.Domain, existing, hasExisting, invalidated, newContainer.ID)
385	}()
386
387	// Start container log collection (non-blocking, errors don't fail deployment)
388	s.startLogCollection(ctx, newContainer.ID, route.Domain)
389
390	log.Info().
391		Str("image", route.Image).
392		Str(zerowrap.FieldEntityID, newContainer.ID).
393		Ints("ports", newContainer.Ports).
394		Str("network", resources.networkName).
395		Bool("zero_downtime", hasExisting).
396		Msg("container deployed successfully")
397
398	return newContainer, nil
399}
400
401// recordDeployMetrics records span error status and deploy metrics at the end of a Deploy call.
402// It is called via defer and receives a pointer to the named return error so it can observe
403// the final error value after all deferred functions have run.
404func (s *Service) recordDeployMetrics(ctx context.Context, span trace.Span, route domain.Route, start time.Time, errPtr *error) {
405	err := *errPtr
406	if err != nil {
407		span.RecordError(err)
408		span.SetStatus(codes.Error, err.Error())
409	}
410	if s.metrics == nil {
411		return
412	}
413	attrs := metric.WithAttributes(
414		attribute.String("domain", route.Domain),
415		attribute.String("image", route.Image),
416	)
417	s.metrics.DeployTotal.Add(ctx, 1, attrs)
418	s.metrics.DeployDuration.Record(ctx, time.Since(start).Seconds(), attrs)
419	if err != nil {
420		s.metrics.DeployErrors.Add(ctx, 1, attrs)
421	}
422}
423
424// skipRedundantDeploy checks whether the existing container is already running
425// the same image (by Docker image ID) as the one we are about to deploy.
426// When a push triggers both an event-based deploy and an explicit CLI deploy,
427// the second one arrives after the first has already completed; this avoids
428// a full redundant create-start-readiness cycle.
429func (s *Service) skipRedundantDeploy(ctx context.Context, existing *domain.Container, actualImageRef, envHash string) (bool, *domain.Container) {
430	log := zerowrap.FromCtx(ctx)
431
432	newImageID, err := s.runtime.GetImageID(ctx, actualImageRef)
433	if err != nil {
434		log.Debug().Err(err).Msg("cannot resolve image ID for redundancy check, proceeding with deploy")
435		return false, nil
436	}
437
438	if existing.ImageID != newImageID {
439		return false, nil
440	}
441
442	// Verify the container is actually still running before skipping.
443	running, err := s.runtime.IsContainerRunning(ctx, existing.ID)
444	if err != nil || !running {
445		return false, nil
446	}
447
448	if existing.Labels != nil {
449		existingEnvHash, hasEnvHash := existing.Labels[domain.LabelEnvHash]
450		if !hasEnvHash || existingEnvHash != envHash {
451			log.Info().
452				Str("container_id", existing.ID).
453				Bool("has_env_hash", hasEnvHash).
454				Msg("env changed, proceeding with deploy despite same image")
455			return false, nil
456		}
457	} else {
458		return false, nil
459	}
460
461	log.Info().
462		Str("container_id", existing.ID).
463		Str("image_id", newImageID).
464		Msg("skipping redundant deploy: container already running this image")
465
466	return true, existing
467}
468
469func (s *Service) containerForRedundantCheck(ctx context.Context, existing *domain.Container) *domain.Container {
470	if existing == nil || existing.ImageID != "" || existing.ID == "" {
471		return existing
472	}
473
474	log := zerowrap.FromCtx(ctx)
475	inspected, err := s.runtime.InspectContainer(ctx, existing.ID)
476	if err != nil {
477		log.Debug().Err(err).Str("container_id", existing.ID).Msg("cannot inspect existing container for redundancy check")
478		return existing
479	}
480	if inspected == nil || inspected.ImageID == "" {
481		return existing
482	}
483
484	existingCopy := *existing
485	existingCopy.ImageID = inspected.ImageID
486	return &existingCopy
487}
488
489type deployResources struct {
490	networkName    string
491	actualImageRef string
492	exposedPorts   []int
493	imageLabels    map[string]string
494	envVars        []string
495	envHash        string
496	volumes        map[string]string
497}
498
499// resolveExistingContainer returns the currently running container for a domain.
500// It first checks the in-memory map (fast path). If not found, it queries the
501// runtime for a running container with matching name and managed label. This
502// handles cases where Gordon restarted and in-memory state is stale.
503//
504// The slow path checks canonical, -new, and -next names because a previous
505// deploy may have been interrupted (e.g. eventbus timeout, Gordon restart)
506// leaving the active container under a temp name. Canonical is preferred when
507// multiple matches exist.
508func (s *Service) resolveExistingContainer(ctx context.Context, domainName string) (*domain.Container, bool) {
509	// Fast path: check in-memory state
510	s.mu.RLock()
511	container, ok := s.containers[domainName]
512	s.mu.RUnlock()
513	if ok {
514		return container, true
515	}
516
517	// Slow path: query runtime for running containers
518	log := zerowrap.FromCtx(ctx)
519	canonicalName := fmt.Sprintf("gordon-%s", domainName)
520	candidateNames := map[string]bool{
521		canonicalName:           true,
522		canonicalName + "-new":  true,
523		canonicalName + "-next": true,
524	}
525
526	running, err := s.runtime.ListContainers(ctx, false)
527	if err != nil {
528		log.Warn().Err(err).Msg("failed to list running containers for existing container resolution")
529		return nil, false
530	}
531
532	// Prefer canonical name; fall back to any running managed temp container.
533	var best *domain.Container
534	for _, c := range running {
535		if !candidateNames[c.Name] || c.Labels[domain.LabelManaged] != "true" {
536			continue
537		}
538		if best == nil || c.Name == canonicalName {
539			best = c
540		}
541	}
542
543	if best != nil {
544		log.Info().
545			Str("container_id", best.ID).
546			Str("container_name", best.Name).
547			Msg("resolved existing container from runtime (in-memory state was stale)")
548
549		// Update in-memory state so subsequent lookups are fast
550		s.mu.Lock()
551		if _, alreadyTracked := s.containers[domainName]; !alreadyTracked {
552			s.managedCount++
553		}
554		s.containers[domainName] = best
555		s.mu.Unlock()
556
557		return best, true
558	}
559
560	return nil, false
561}
562
563func (s *Service) prepareDeployResources(ctx context.Context, route domain.Route, existing *domain.Container) (*deployResources, error) {
564	log := zerowrap.FromCtx(ctx)
565
566	existingID := ""
567	if existing != nil {
568		existingID = existing.ID
569	}
570	if err := s.cleanupOrphanedContainers(ctx, route.Domain, existingID); err != nil {
571		log.WrapErr(err, "failed to cleanup orphaned containers")
572	}
573
574	imageRef, err := s.buildValidatedImageRef(ctx, route.Image)
575	if err != nil {
576		return nil, err
577	}
578	actualImageRef, err := s.ensureImage(ctx, imageRef)
579	if err != nil {
580		return nil, err
581	}
582
583	networkName := s.getNetworkForApp(route.Domain)
584	if err := s.createNetworkIfNeeded(ctx, networkName); err != nil {
585		return nil, log.WrapErr(err, "failed to create network")
586	}
587	if err := s.deployAttachments(ctx, route.Domain, networkName); err != nil {
588		return nil, log.WrapErr(err, "failed to deploy attachments")
589	}
590
591	exposedPorts, err := s.runtime.GetImageExposedPorts(ctx, actualImageRef)
592	if err != nil {
593		log.WrapErr(err, "failed to get exposed ports, using defaults")
594		exposedPorts = []int{80, 8080, 3000}
595	}
596
597	imageLabels, err := s.runtime.GetImageLabels(ctx, actualImageRef)
598	if err != nil {
599		log.Warn().Err(err).Msg("failed to get image labels, skipping label propagation")
600		imageLabels = nil
601	}
602
603	envVars, err := s.loadEnvironment(ctx, route.Env, route.Domain, actualImageRef)
604	if err != nil {
605		return nil, err
606	}
607	envHash := hashEnvironment(envVars)
608
609	volumes, err := s.setupVolumes(ctx, route.Domain, actualImageRef)
610	if err != nil {
611		return nil, err
612	}
613
614	return &deployResources{
615		networkName:    networkName,
616		actualImageRef: actualImageRef,
617		exposedPorts:   exposedPorts,
618		imageLabels:    imageLabels,
619		envVars:        envVars,
620		envHash:        envHash,
621		volumes:        volumes,
622	}, nil
623}
624
625func (s *Service) createStartedContainer(ctx context.Context, route domain.Route, existing *domain.Container, resources *deployResources) (*domain.Container, error) {
626	ctx, span := tracer.Start(ctx, "container.create_and_start")
627	defer span.End()
628
629	log := zerowrap.FromCtx(ctx)
630
631	containerConfig := s.buildContainerConfig(containerConfigInput{
632		Domain:       route.Domain,
633		Image:        route.Image,
634		ImageRef:     resources.actualImageRef,
635		ExposedPorts: resources.exposedPorts,
636		EnvVars:      resources.envVars,
637		EnvHash:      resources.envHash,
638		Volumes:      resources.volumes,
639		NetworkName:  resources.networkName,
640		ImageLabels:  resources.imageLabels,
641		Existing:     existing,
642	})
643
644	newContainer, err := s.runtime.CreateContainer(ctx, containerConfig)
645	if err != nil {
646		return nil, log.WrapErr(err, "failed to create container")
647	}
648	if err := s.runtime.StartContainer(ctx, newContainer.ID); err != nil {
649		s.runtime.RemoveContainer(ctx, newContainer.ID, true)
650		return nil, log.WrapErr(err, "failed to start container")
651	}
652	if !domain.IsSkipReadiness(ctx) {
653		if err := s.waitForReady(ctx, newContainer.ID, containerConfig); err != nil {
654			deployErr := s.newDeployFailure(newContainer, "container failed readiness check", err, s.captureRecentContainerLogs(ctx, newContainer.ID))
655			s.cleanupFailedContainer(ctx, newContainer.ID)
656			return nil, deployErr
657		}
658	}
659
660	inspected, err := s.runtime.InspectContainer(ctx, newContainer.ID)
661	if err != nil {
662		s.cleanupFailedContainer(ctx, newContainer.ID)
663		return nil, log.WrapErr(err, "failed to inspect started container")
664	}
665
666	return inspected, nil
667}
668
669func (s *Service) newDeployFailure(container *domain.Container, cause string, err error, logs []string) error {
670	deployErr := &domain.DeployFailureError{
671		Summary: "failed to deploy",
672		Cause:   cause,
673		Hint:    inferDeployFailureHint(err),
674		Logs:    logs,
675		Err:     err,
676	}
677	if container != nil {
678		deployErr.ContainerName = container.Name
679		deployErr.ContainerID = container.ID
680	}
681
682	return deployErr
683}
684
685func (s *Service) wrapPullDeployFailure(route domain.Route, existing *domain.Container, err error) error {
686	if !isPullFailure(err) {
687		return nil
688	}
689
690	return s.newDeployFailure(&domain.Container{Name: s.deploymentContainerName(route.Domain, existing)}, "failed to pull image", err, nil)
691}
692
693func (s *Service) captureRecentContainerLogs(ctx context.Context, containerID string) []string {
694	if ctx.Err() != nil {
695		return nil
696	}
697
698	logStream, err := s.runtime.GetContainerLogs(ctx, containerID, false)
699	if err != nil {
700		return nil
701	}
702	defer func() {
703		_ = logStream.Close()
704	}()
705
706	logs, err := parseDockerLogLines(logStream, deployFailureLogLimit)
707	if err != nil {
708		return nil
709	}
710
711	return logs
712}
713
714func parseDockerLogLines(r io.Reader, maxLines int) ([]string, error) {
715	reader := bufio.NewReader(r)
716	collector := newLogLineCollector(maxLines)
717
718	header, err := reader.Peek(8)
719	if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, bufio.ErrBufferFull) {
720		return nil, err
721	}
722
723	if len(header) >= 8 && looksLikeDockerLogStream(header) {
724		return parseDockerFramedLogLines(reader, collector)
725	}
726
727	return parsePlainLogLines(reader, collector)
728}
729
730func parseDockerFramedLogLines(reader *bufio.Reader, collector *logLineCollector) ([]string, error) {
731	for {
732		frameHeader := make([]byte, 8)
733		if _, err := io.ReadFull(reader, frameHeader); err != nil {
734			if isLogStreamEOF(err) {
735				collector.flush()
736				return collector.lines, nil
737			}
738			return nil, err
739		}
740
741		size := int(binary.BigEndian.Uint32(frameHeader[4:8]))
742		if size <= 0 {
743			continue
744		}
745
746		payload, err := readLogPayload(reader, size)
747		if err != nil {
748			collector.addChunk(string(payload))
749			collector.flush()
750			if isLogStreamEOF(err) {
751				return collector.lines, nil
752			}
753			return nil, err
754		}
755
756		collector.addChunk(string(payload))
757	}
758}
759
760func parsePlainLogLines(reader *bufio.Reader, collector *logLineCollector) ([]string, error) {
761
762	scanner := bufio.NewScanner(reader)
763	for scanner.Scan() {
764		collector.addLine(scanner.Text())
765	}
766	if err := scanner.Err(); err != nil {
767		return nil, err
768	}
769
770	collector.flush()
771	return collector.lines, nil
772}
773
774func readLogPayload(reader *bufio.Reader, size int) ([]byte, error) {
775	payload := make([]byte, size)
776	_, err := io.ReadFull(reader, payload)
777	return payload, err
778}
779
780func isLogStreamEOF(err error) bool {
781	return errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF)
782}
783
784func looksLikeDockerLogStream(data []byte) bool {
785	return len(data) >= 8 && data[1] == 0 && data[2] == 0 && data[3] == 0
786}
787
788type logLineCollector struct {
789	lines    []string
790	pending  string
791	maxLines int
792}
793
794func newLogLineCollector(maxLines int) *logLineCollector {
795	return &logLineCollector{maxLines: maxLines}
796}
797
798func (c *logLineCollector) addChunk(chunk string) {
799	for _, part := range strings.SplitAfter(chunk, "\n") {
800		c.pending += part
801		if strings.HasSuffix(part, "\n") {
802			c.flush()
803		}
804	}
805}
806
807func (c *logLineCollector) addLine(line string) {
808	line = strings.TrimRight(line, "\r")
809	if line == "" {
810		return
811	}
812	c.lines = append(c.lines, line)
813	if c.maxLines > 0 && len(c.lines) > c.maxLines {
814		c.lines = c.lines[len(c.lines)-c.maxLines:]
815	}
816}
817
818func (c *logLineCollector) flush() {
819	if c.pending == "" {
820		return
821	}
822	c.addLine(strings.TrimRight(c.pending, "\r\n"))
823	c.pending = ""
824}
825
826func inferDeployFailureHint(err error) string {
827	if err == nil {
828		return ""
829	}
830
831	msg := strings.ToLower(err.Error())
832	switch {
833	case strings.Contains(msg, "healthcheck timeout"):
834		return "check startup logs and confirm the health endpoint becomes ready within the configured timeout"
835	case strings.Contains(msg, "unhealthy"):
836		return "inspect recent app logs and verify the healthcheck command reflects actual readiness"
837	case strings.Contains(msg, "no healthcheck detected"):
838		return "add a Docker healthcheck or switch readiness mode"
839	case isPullFailure(err), strings.Contains(msg, "access denied"), strings.Contains(msg, "unauthorized"):
840		return "verify registry auth and confirm the image exists at the requested tag"
841	default:
842		return ""
843	}
844}
845
846func isPullFailure(err error) bool {
847	if err == nil {
848		return false
849	}
850
851	if errors.Is(err, domain.ErrImagePullFailed) {
852		return true
853	}
854
855	msg := strings.ToLower(err.Error())
856	return strings.Contains(msg, "failed to pull image") ||
857		strings.Contains(msg, "pull access denied")
858}
859
860func (s *Service) activateDeployedContainer(ctx context.Context, domainName string, container *domain.Container) bool {
861	s.mu.Lock()
862	_, wasTracked := s.containers[domainName]
863	s.containers[domainName] = container
864	if !wasTracked {
865		s.managedCount++
866	}
867	s.mu.Unlock()
868
869	// Track managed container count (only increment for new domains, not replacements)
870	if !wasTracked && s.metrics != nil {
871		s.metrics.ManagedContainers.Add(ctx, 1)
872	}
873
874	s.publishContainerDeployed(ctx, domainName, container.ID)
875
876	s.mu.RLock()
877	inv := s.cacheInvalidator
878	s.mu.RUnlock()
879	if inv != nil {
880		inv.InvalidateTarget(ctx, domainName)
881		return true
882	}
883
884	return false
885}
886
887// stabilizeNewContainer monitors the new container briefly after traffic switch.
888// If it crashes during this window, rolls back to old container.
889// Returns true if stabilization succeeded, false if rollback was performed.
890// Returns an error only when both new and old containers are dead.
891func (s *Service) stabilizeNewContainer(ctx context.Context, domainName string, newContainer, oldContainer *domain.Container) (bool, error) {
892	log := zerowrap.FromCtx(ctx)
893
894	if oldContainer == nil {
895		return true, nil
896	}
897
898	s.mu.RLock()
899	delay := s.config.StabilizationDelay
900	s.mu.RUnlock()
901	if delay == 0 {
902		delay = 2 * time.Second
903	}
904
905	// Brief stabilization: verify new container is still running after delay
906	select {
907	case <-time.After(delay):
908	case <-ctx.Done():
909		return true, nil
910	}
911
912	running, err := s.runtime.IsContainerRunning(ctx, newContainer.ID)
913	if err != nil || !running {
914		log.Error().
915			Str("new_container", newContainer.ID).
916			Str("old_container", oldContainer.ID).
917			Msg("new container crashed during stabilization, rolling back to old")
918
919		// Verify old container is still running before restoring it.
920		// If both old and new are dead (e.g., OOM), returning a dead
921		// container as "existing" would leave the domain in a broken state.
922		oldRunning, oldErr := s.runtime.IsContainerRunning(ctx, oldContainer.ID)
923		if oldErr != nil || !oldRunning {
924			log.Error().
925				Str("old_container", oldContainer.ID).
926				Msg("old container is also not running, cannot rollback")
927
928			// Cleanup failed new container
929			if stopErr := s.runtime.StopContainer(ctx, newContainer.ID); stopErr != nil {
930				log.WrapErrWithFields(stopErr, "failed to stop failed new container during rollback", map[string]any{zerowrap.FieldEntityID: newContainer.ID})
931			}
932			if removeErr := s.runtime.RemoveContainer(ctx, newContainer.ID, true); removeErr != nil {
933				log.WrapErrWithFields(removeErr, "failed to remove failed new container during rollback", map[string]any{zerowrap.FieldEntityID: newContainer.ID})
934			}
935
936			return false, fmt.Errorf("stabilization failed: new container crashed and old container %s is also not running", oldContainer.ID)
937		}
938
939		// Rollback: restore old container as tracked
940		s.mu.Lock()
941		s.containers[domainName] = oldContainer
942		s.mu.Unlock()
943
944		// Re-invalidate proxy cache to point back to old
945		s.mu.RLock()
946		inv := s.cacheInvalidator
947		s.mu.RUnlock()
948		if inv != nil {
949			inv.InvalidateTarget(ctx, domainName)
950		}
951
952		// Cleanup failed new container
953		if stopErr := s.runtime.StopContainer(ctx, newContainer.ID); stopErr != nil {
954			log.WrapErrWithFields(stopErr, "failed to stop failed new container during rollback", map[string]any{zerowrap.FieldEntityID: newContainer.ID})
955		}
956		if removeErr := s.runtime.RemoveContainer(ctx, newContainer.ID, true); removeErr != nil {
957			log.WrapErrWithFields(removeErr, "failed to remove failed new container during rollback", map[string]any{zerowrap.FieldEntityID: newContainer.ID})
958		}
959
960		return false, nil
961	}
962
963	return true, nil
964}
965
966func (s *Service) finalizePreviousContainer(ctx context.Context, domainName string, existing *domain.Container, hasExisting, invalidated bool, newContainerID string) {
967	if !hasExisting {
968		return
969	}
970
971	if invalidated {
972		s.waitForDrain(ctx, existing.ID)
973	}
974
975	s.cleanupOldContainer(ctx, existing, newContainerID, domainName)
976}
977
978func (s *Service) waitForDrain(ctx context.Context, oldContainerID string) {
979	log := zerowrap.FromCtx(ctx)
980
981	s.mu.RLock()
982	cfg := s.config
983	waiter := s.drainWaiter
984	s.mu.RUnlock()
985
986	drainMode := cfg.DrainMode
987	if drainMode == "" {
988		drainMode = "delay"
989	}
990
991	shouldUseInFlight := (drainMode == "inflight" || drainMode == "auto") && waiter != nil
992	if shouldUseInFlight {
993		timeout := cfg.DrainTimeout
994		if timeout == 0 {
995			timeout = 30 * time.Second
996		}
997		drained := waiter.WaitForNoInFlight(ctx, oldContainerID, timeout)
998		if !drained {
999			log.Warn().
1000				Str("old_container_id", oldContainerID).
1001				Dur("drain_timeout", timeout).
1002				Msg("drain wait timed out; old container may still have in-flight traffic")
1003		}
1004		return
1005	}
1006
1007	drainDelay := 2 * time.Second
1008	if cfg.DrainDelayConfigured {
1009		drainDelay = cfg.DrainDelay
1010	}
1011	if drainDelay <= 0 {
1012		return
1013	}
1014	select {
1015	case <-time.After(drainDelay):
1016	case <-ctx.Done():
1017	}
1018}
1019
1020// restartContainerWithRecovery restarts a container, reconciling stale state if needed.
1021// It returns the (possibly refreshed) container and updated attachment IDs.
1022func (s *Service) restartContainerWithRecovery(ctx context.Context, domainName string, container *domain.Container, attachmentIDs []string) (*domain.Container, []string, error) {
1023	log := zerowrap.FromCtx(ctx)
1024
1025	if err := s.runtime.RestartContainer(ctx, container.ID); err != nil {
1026		if !isContainerNotFoundError(err) {
1027			return nil, nil, log.WrapErr(err, "failed to restart container")
1028		}
1029
1030		// In-memory container ID can become stale after external runtime changes.
1031		// Re-sync and retry once with the latest tracked container.
1032		log.Warn().
1033			Err(err).
1034			Str(zerowrap.FieldEntityID, container.ID).
1035			Msg("tracked container missing during restart, attempting state reconciliation")
1036
1037		if syncErr := s.SyncContainers(ctx); syncErr != nil {
1038			log.Warn().Err(syncErr).Msg("failed to sync container state during restart recovery")
1039		}
1040
1041		s.mu.RLock()
1042		refreshed, refreshedExists := s.containers[domainName]
1043		attachmentIDs = append([]string{}, s.attachments[domainName]...)
1044		s.mu.RUnlock()
1045
1046		if !refreshedExists || refreshed == nil {
1047			return nil, nil, domain.ErrContainerNotFound
1048		}
1049		if err := s.runtime.RestartContainer(ctx, refreshed.ID); err != nil {
1050			return nil, nil, log.WrapErr(err, "failed to restart container after state reconciliation")
1051		}
1052		return refreshed, attachmentIDs, nil
1053	}
1054
1055	return container, attachmentIDs, nil
1056}
1057
1058// Restart restarts a running container for the given domain.
1059func (s *Service) Restart(ctx context.Context, domainName string, withAttachments bool) error {
1060	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1061		zerowrap.FieldLayer:   "usecase",
1062		zerowrap.FieldUseCase: "Restart",
1063		"domain":              domainName,
1064	})
1065	log := zerowrap.FromCtx(ctx)
1066
1067	s.mu.RLock()
1068	container, exists := s.containers[domainName]
1069	attachments := s.attachments[domainName]
1070	var attachmentIDs []string
1071	if len(attachments) > 0 {
1072		attachmentIDs = make([]string, len(attachments))
1073		copy(attachmentIDs, attachments)
1074	}
1075	s.mu.RUnlock()
1076
1077	if !exists || container == nil {
1078		return domain.ErrContainerNotFound
1079	}
1080
1081	// When attachments are requested, sync runtime state first to get accurate
1082	// attachment tracking, then check if configured attachments are deployed.
1083	// If fewer attachment containers are tracked than configured, fail early with
1084	// guidance rather than restarting the main container into a broken state
1085	// (e.g., missing database hostname).
1086	if withAttachments {
1087		if syncErr := s.SyncContainers(ctx); syncErr != nil {
1088			log.Warn().Err(syncErr).Msg("failed to sync container state before attachment check")
1089		}
1090
1091		// Re-read attachment IDs after sync
1092		s.mu.RLock()
1093		if freshAttachments := s.attachments[domainName]; len(freshAttachments) > 0 {
1094			attachmentIDs = make([]string, len(freshAttachments))
1095			copy(attachmentIDs, freshAttachments)
1096		} else {
1097			attachmentIDs = nil
1098		}
1099		s.mu.RUnlock()
1100
1101		configuredAttachments := s.resolveAttachmentsForDomain(domainName)
1102		if len(configuredAttachments) > 0 && len(attachmentIDs) < len(configuredAttachments) {
1103			missing := len(configuredAttachments) - len(attachmentIDs)
1104			return fmt.Errorf("%w: domain %q has %d configured attachment(s) but only %d deployed (%d missing)",
1105				domain.ErrAttachmentNotDeployed, domainName, len(configuredAttachments), len(attachmentIDs), missing)
1106		}
1107	}
1108
1109	// Restart the main container (with stale-state recovery)
1110	container, attachmentIDs, err := s.restartContainerWithRecovery(ctx, domainName, container, attachmentIDs)
1111	if err != nil {
1112		return err
1113	}
1114	log.Info().Str(zerowrap.FieldEntityID, container.ID).Msg("container restarted")
1115
1116	// Record restart metric
1117	if s.metrics != nil {
1118		s.metrics.ContainerRestarts.Add(ctx, 1, metric.WithAttributes(
1119			attribute.String("domain", domainName),
1120			attribute.String("source", "api"),
1121		))
1122	}
1123
1124	// Restart attachments if requested
1125	if withAttachments && len(attachmentIDs) > 0 {
1126		for _, attachID := range attachmentIDs {
1127			if err := s.runtime.RestartContainer(ctx, attachID); err != nil {
1128				log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to restart attachment")
1129				continue // Don't fail the whole operation for one attachment
1130			}
1131			log.Info().Str("attachment_id", attachID).Msg("attachment restarted")
1132		}
1133	}
1134
1135	return nil
1136}
1137
1138// Stop stops a running container.
1139func (s *Service) Stop(ctx context.Context, containerID string) error {
1140	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1141		zerowrap.FieldLayer:    "usecase",
1142		zerowrap.FieldUseCase:  "Stop",
1143		zerowrap.FieldEntityID: containerID,
1144	})
1145	log := zerowrap.FromCtx(ctx)
1146
1147	// Stop log collection before stopping container
1148	if s.logWriter != nil {
1149		if err := s.logWriter.StopLogging(containerID); err != nil {
1150			log.Warn().Err(err).Msg("failed to stop container log collection")
1151		}
1152	}
1153
1154	if err := s.runtime.StopContainer(ctx, containerID); err != nil {
1155		return log.WrapErr(err, "failed to stop container")
1156	}
1157
1158	log.Info().Msg("container stopped")
1159	return nil
1160}
1161
1162// Remove removes a container.
1163func (s *Service) Remove(ctx context.Context, containerID string, force bool) error {
1164	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1165		zerowrap.FieldLayer:    "usecase",
1166		zerowrap.FieldUseCase:  "Remove",
1167		zerowrap.FieldEntityID: containerID,
1168	})
1169	log := zerowrap.FromCtx(ctx)
1170
1171	// Stop log collection before removing container
1172	if s.logWriter != nil {
1173		if err := s.logWriter.StopLogging(containerID); err != nil {
1174			log.Warn().Err(err).Msg("failed to stop container log collection")
1175		}
1176	}
1177
1178	// Find domain and attachment IDs for cleanup
1179	containerDomain, attachmentIDs, cfg := s.findContainerContext(containerID)
1180
1181	if err := s.runtime.RemoveContainer(ctx, containerID, force); err != nil {
1182		return log.WrapErr(err, "failed to remove container")
1183	}
1184
1185	// Cleanup volumes
1186	if containerDomain != "" && !cfg.VolumePreserve {
1187		if err := s.cleanupVolumesForDomain(ctx, containerDomain); err != nil {
1188			log.WrapErrWithFields(err, "failed to cleanup volumes", map[string]any{"domain": containerDomain})
1189		}
1190	}
1191
1192	// Clean up attachment containers
1193	s.removeAttachments(ctx, attachmentIDs)
1194
1195	// Remove from tracking
1196	s.mu.Lock()
1197	var removedDomain string
1198	for d, c := range s.containers {
1199		if c.ID == containerID {
1200			delete(s.containers, d)
1201			removedDomain = d
1202			s.managedCount--
1203			log.Info().Str("domain", d).Msg("container removed")
1204			break
1205		}
1206	}
1207	if removedDomain != "" {
1208		delete(s.attachments, removedDomain)
1209	}
1210	s.mu.Unlock()
1211
1212	// Decrement managed container count
1213	if removedDomain != "" && s.metrics != nil {
1214		s.metrics.ManagedContainers.Add(ctx, -1)
1215	}
1216
1217	// Note: We intentionally do NOT clean up deployMu entries to avoid a race condition:
1218	// If Remove deletes the mutex entry while a concurrent Deploy is acquiring the lock,
1219	// the Deploy would create a fresh mutex, breaking serialization and allowing concurrent
1220	// deploys to the same domain. The memory footprint is acceptable (one small struct per
1221	// domain ever deployed), and this is the safer choice for correctness.
1222
1223	// Cleanup network
1224	if removedDomain != "" && cfg.NetworkIsolation {
1225		networkName := s.getNetworkForApp(removedDomain)
1226		if err := s.cleanupNetworkIfEmpty(ctx, networkName); err != nil {
1227			log.WrapErrWithFields(err, "failed to cleanup network", map[string]any{"network": networkName})
1228		}
1229	}
1230
1231	return nil
1232}
1233
1234// findContainerContext looks up the domain, attachment IDs, and config snapshot for a container.
1235func (s *Service) findContainerContext(containerID string) (string, []string, Config) {
1236	s.mu.RLock()
1237	defer s.mu.RUnlock()
1238	cfg := s.config
1239	var containerDomain string
1240	for d, c := range s.containers {
1241		if c.ID == containerID {
1242			containerDomain = d
1243			break
1244		}
1245	}
1246	var attachmentIDs []string
1247	if containerDomain != "" {
1248		attachmentIDs = append(attachmentIDs, s.attachments[containerDomain]...)
1249	}
1250	return containerDomain, attachmentIDs, cfg
1251}
1252
1253// removeAttachments stops and removes attachment containers best-effort.
1254func (s *Service) removeAttachments(ctx context.Context, attachmentIDs []string) {
1255	log := zerowrap.FromCtx(ctx)
1256	for _, attachID := range attachmentIDs {
1257		if s.logWriter != nil {
1258			if err := s.logWriter.StopLogging(attachID); err != nil {
1259				log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to stop attachment log collection")
1260			}
1261		}
1262		if err := s.runtime.StopContainer(ctx, attachID); err != nil {
1263			log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to stop attachment container")
1264		}
1265		if err := s.runtime.RemoveContainer(ctx, attachID, true); err != nil {
1266			log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to remove attachment container")
1267		}
1268	}
1269}
1270
1271// Get retrieves a container by domain.
1272func (s *Service) Get(_ context.Context, domainName string) (*domain.Container, bool) {
1273	s.mu.RLock()
1274	defer s.mu.RUnlock()
1275	container, exists := s.containers[domainName]
1276	return container, exists
1277}
1278
1279// List returns all managed containers.
1280func (s *Service) List(_ context.Context) map[string]*domain.Container {
1281	s.mu.RLock()
1282	defer s.mu.RUnlock()
1283
1284	result := make(map[string]*domain.Container, len(s.containers))
1285	maps.Copy(result, s.containers)
1286	return result
1287}
1288
1289// ListRoutesWithDetails returns routes with network and attachment info.
1290// Note: Container map is copied under lock, then external calls are made without lock.
1291// If containers are removed between copy and runtime calls, errors are handled gracefully
1292// (network becomes empty string). This trade-off avoids holding locks during I/O.
1293func (s *Service) ListRoutesWithDetails(ctx context.Context) []domain.RouteInfo {
1294	s.mu.RLock()
1295	containers := make(map[string]*domain.Container, len(s.containers))
1296	maps.Copy(containers, s.containers)
1297	s.mu.RUnlock()
1298
1299	// Fetch all attachments once to avoid N+1 queries
1300	attachmentsByDomain := s.getAllAttachments(ctx)
1301
1302	results := make([]domain.RouteInfo, 0, len(containers))
1303	for domainName, container := range containers {
1304		network := ""
1305		image := ""
1306		containerID := ""
1307		status := ""
1308		if container != nil {
1309			containerID = container.ID
1310			status = container.Status
1311			image = container.Image
1312			if container.Labels != nil {
1313				if labelImage, ok := container.Labels[domain.LabelImage]; ok && labelImage != "" {
1314					image = labelImage
1315				}
1316			}
1317			// Strip registry domain prefix from image for cleaner display
1318			image = s.stripRegistryPrefix(image)
1319			if networkName, err := s.runtime.GetContainerNetwork(ctx, container.ID); err == nil {
1320				network = networkName
1321			}
1322		}
1323
1324		results = append(results, domain.RouteInfo{
1325			Domain:          domainName,
1326			Image:           image,
1327			ContainerID:     containerID,
1328			ContainerStatus: status,
1329			Network:         network,
1330			Attachments:     attachmentsByDomain[domainName],
1331		})
1332	}
1333
1334	return results
1335}
1336
1337// stripRegistryPrefix removes the configured current or legacy Gordon registry
1338// domain prefix from an image reference. For example,
1339// "reg.example.com/myapp:latest" becomes "myapp:latest" when
1340// "reg.example.com" is a current or legacy Gordon registry domain.
1341func (s *Service) stripRegistryPrefix(image string) string {
1342	s.mu.RLock()
1343	cfg := s.config
1344	s.mu.RUnlock()
1345
1346	return domain.StripKnownGordonRegistry(image, cfg.RegistryDomain, cfg.LegacyRegistryDomains)
1347}
1348
1349// ListAttachments returns attachments for a domain.
1350func (s *Service) ListAttachments(ctx context.Context, domainName string) []domain.Attachment {
1351	return s.getAttachmentsForDomain(ctx, domainName)
1352}
1353
1354// ListNetworks returns Gordon-managed networks.
1355func (s *Service) ListNetworks(ctx context.Context) ([]*domain.NetworkInfo, error) {
1356	s.mu.RLock()
1357	cfg := s.config
1358	s.mu.RUnlock()
1359
1360	networks, err := s.runtime.ListNetworks(ctx)
1361	if err != nil {
1362		return nil, err
1363	}
1364
1365	var filtered []*domain.NetworkInfo
1366	for _, network := range networks {
1367		if strings.HasPrefix(network.Name, cfg.NetworkPrefix+"-") && network.Labels[domain.LabelManaged] == "true" {
1368			filtered = append(filtered, network)
1369		}
1370	}
1371
1372	return filtered, nil
1373}
1374
1375// HealthCheck performs health checks on all containers.
1376func (s *Service) HealthCheck(ctx context.Context) map[string]bool {
1377	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1378		zerowrap.FieldLayer:   "usecase",
1379		zerowrap.FieldUseCase: "HealthCheck",
1380	})
1381	log := zerowrap.FromCtx(ctx)
1382
1383	// Copy container IDs under lock, then release before making Docker API calls
1384	s.mu.RLock()
1385	snapshot := make(map[string]string, len(s.containers))
1386	for d, c := range s.containers {
1387		snapshot[d] = c.ID
1388	}
1389	s.mu.RUnlock()
1390
1391	health := make(map[string]bool, len(snapshot))
1392	for d, id := range snapshot {
1393		running, err := s.runtime.IsContainerRunning(ctx, id)
1394		if err != nil {
1395			log.WrapErrWithFields(err, "health check failed", map[string]any{"domain": d, zerowrap.FieldEntityID: id})
1396			health[d] = false
1397		} else {
1398			health[d] = running
1399		}
1400	}
1401	return health
1402}
1403
1404// SyncContainers synchronizes containers with runtime state.
1405func (s *Service) SyncContainers(ctx context.Context) error {
1406	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1407		zerowrap.FieldLayer:   "usecase",
1408		zerowrap.FieldUseCase: "SyncContainers",
1409	})
1410	log := zerowrap.FromCtx(ctx)
1411
1412	// List containers without holding lock to avoid blocking during the runtime call.
1413	// Use all=true so we can rebuild the running-container maps from a single
1414	// snapshot without a second list call.
1415	allContainers, err := s.runtime.ListContainers(ctx, true)
1416	if err != nil {
1417		return log.WrapErr(err, "failed to list containers")
1418	}
1419
1420	managed := make(map[string]*domain.Container)
1421	attachments := make(map[string][]string)
1422	for _, c := range allContainers {
1423		if c.Status != "" && c.Status != string(domain.ContainerStatusRunning) {
1424			continue
1425		}
1426		if c.Labels == nil || c.Labels[domain.LabelManaged] != "true" {
1427			continue
1428		}
1429		if c.Labels[domain.LabelAttachment] == "true" {
1430			owner := c.Labels[domain.LabelAttachedTo]
1431			if owner != "" {
1432				attachments[owner] = append(attachments[owner], c.ID)
1433			}
1434			continue
1435		}
1436		if d, ok := c.Labels[domain.LabelDomain]; ok {
1437			managed[d] = c
1438		}
1439	}
1440
1441	s.mu.Lock()
1442	s.containers = managed
1443	s.attachments = attachments
1444	newCount := int64(len(managed))
1445	delta := newCount - s.managedCount
1446	s.managedCount = newCount
1447	s.mu.Unlock()
1448
1449	// Report initial/delta managed container count to OTel UpDownCounter.
1450	if delta != 0 && s.metrics != nil {
1451		s.metrics.ManagedContainers.Add(ctx, delta)
1452	}
1453
1454	log.Info().Int(zerowrap.FieldCount, len(managed)).Msg("container state synchronized")
1455	return nil
1456}
1457
1458// AutoStart starts containers for the provided routes that aren't running.
1459// It skips readiness checks to avoid blocking boot; the background monitor
1460// handles crash recovery.
1461func (s *Service) AutoStart(ctx context.Context, routes []domain.Route) error {
1462	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1463		zerowrap.FieldLayer:   "usecase",
1464		zerowrap.FieldUseCase: "AutoStart",
1465	})
1466	log := zerowrap.FromCtx(ctx)
1467
1468	log.Info().Int("route_count", len(routes)).Msg("auto-starting containers for configured routes")
1469
1470	// Filter out routes that already have a running container.
1471	var pending []domain.Route
1472	skipped := 0
1473	for _, route := range routes {
1474		if _, exists := s.Get(ctx, route.Domain); exists {
1475			log.Debug().Str("domain", route.Domain).Msg("container already running, skipping")
1476			skipped++
1477		} else {
1478			pending = append(pending, route)
1479		}
1480	}
1481
1482	if len(pending) == 0 {
1483		log.Info().Int("skipped", skipped).Msg("auto-start completed, all containers already running")
1484		return nil
1485	}
1486
1487	// Deploy all pending routes concurrently with readiness checks skipped.
1488	deployCtx := domain.WithSkipReadiness(ctx)
1489
1490	type result struct {
1491		route domain.Route
1492		err   error
1493	}
1494	results := make(chan result, len(pending))
1495	sem := make(chan struct{}, 4) // limit concurrency
1496
1497	for _, route := range pending {
1498		sem <- struct{}{}
1499		go func(r domain.Route) {
1500			defer func() { <-sem }()
1501
1502			log.Info().Str("domain", r.Domain).Str("image", r.Image).Msg("auto-starting container for route")
1503
1504			if _, err := s.Deploy(deployCtx, r); err != nil {
1505				results <- result{route: r, err: err}
1506				return
1507			}
1508			results <- result{route: r}
1509		}(route)
1510	}
1511
1512	var started, errCount int
1513	for i := 0; i < len(pending); i++ {
1514		res := <-results
1515		if res.err != nil {
1516			log.Warn().Err(res.err).Str("domain", res.route.Domain).Msg("failed to auto-start container")
1517			errCount++
1518		} else {
1519			started++
1520		}
1521	}
1522
1523	log.Info().
1524		Int("started", started).
1525		Int("skipped", skipped).
1526		Int("errors", errCount).
1527		Msg("auto-start completed")
1528
1529	if errCount > 0 {
1530		return fmt.Errorf("auto-start completed with %d errors", errCount)
1531	}
1532	return nil
1533}
1534
1535// Shutdown gracefully shuts down all managed containers.
1536func (s *Service) Shutdown(ctx context.Context) error {
1537	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1538		zerowrap.FieldLayer:   "usecase",
1539		zerowrap.FieldUseCase: "Shutdown",
1540	})
1541	log := zerowrap.FromCtx(ctx)
1542	log.Info().Msg("shutting down container manager...")
1543
1544	// Wait for any in-flight background cleanup goroutines to finish
1545	s.cleanupWg.Wait()
1546
1547	// Containers are left running across Gordon restarts.
1548	// SyncContainers + AutoStart will pick them back up on next boot.
1549
1550	// Close log writer to stop all log collection
1551	if s.logWriter != nil {
1552		if err := s.logWriter.Close(); err != nil {
1553			log.Warn().Err(err).Msg("failed to close container log writer")
1554		}
1555	}
1556
1557	log.Info().Msg("container manager shutdown complete")
1558	return nil
1559}
1560
1561// WaitForCleanup blocks until all background cleanup goroutines complete.
1562// Intended for use in tests to avoid mock assertion races.
1563func (s *Service) WaitForCleanup() {
1564	s.cleanupWg.Wait()
1565}
1566
1567// StartMonitor begins background monitoring of tracked containers.
1568// The monitor restarts crashed containers and detects crash loops.
1569// Safe to call multiple times; subsequent calls are no-ops.
1570func (s *Service) StartMonitor(ctx context.Context) {
1571	s.mu.Lock()
1572	if s.monitor != nil {
1573		s.mu.Unlock()
1574		return
1575	}
1576	s.monitor = newMonitor(s)
1577	m := s.monitor
1578	m.Start(ctx)
1579	s.mu.Unlock()
1580}
1581
1582// StopMonitor stops the background container monitor.
1583// Safe to call multiple times; subsequent calls are no-ops.
1584func (s *Service) StopMonitor() {
1585	s.mu.Lock()
1586	m := s.monitor
1587	s.monitor = nil
1588	s.mu.Unlock()
1589	if m != nil {
1590		m.Stop()
1591	}
1592}
1593
1594// UpdateConfig updates the service configuration.
1595func (s *Service) UpdateConfig(config Config) {
1596	s.mu.Lock()
1597	s.config = config
1598	s.mu.Unlock()
1599}
1600
1601// UpdateAttachments updates only the attachment configuration in the service.
1602// This is called after a config reload to propagate attachment changes without restart.
1603// The incoming map is deep-copied so external callers cannot mutate service state.
1604func (s *Service) UpdateAttachments(attachments map[string][]string) {
1605	var copied map[string][]string
1606	if attachments != nil {
1607		copied = make(map[string][]string, len(attachments))
1608		for k, v := range attachments {
1609			sl := make([]string, len(v))
1610			copy(sl, v)
1611			copied[k] = sl
1612		}
1613	}
1614	s.mu.Lock()
1615	s.config.Attachments = copied
1616	s.mu.Unlock()
1617}
1618
1619// Helper methods
1620
1621// cleanupFailedContainer stops and removes a container that failed to start properly.
1622func (s *Service) cleanupFailedContainer(ctx context.Context, containerID string) {
1623	log := zerowrap.FromCtx(ctx)
1624	if err := s.runtime.StopContainer(ctx, containerID); err != nil {
1625		log.Warn().Err(err).Str(zerowrap.FieldEntityID, containerID).Msg("failed to stop container after failure")
1626	}
1627	if err := s.runtime.RemoveContainer(ctx, containerID, true); err != nil {
1628		log.Warn().Err(err).Str(zerowrap.FieldEntityID, containerID).Msg("failed to remove container after failure")
1629	}
1630}
1631
1632// cleanupOldContainer stops and removes an old container after zero-downtime switch.
1633// It also renames the new container to the canonical name.
1634func (s *Service) cleanupOldContainer(ctx context.Context, old *domain.Container, newContainerID, domainName string) {
1635	log := zerowrap.FromCtx(ctx)
1636	log.Info().Str(zerowrap.FieldEntityID, old.ID).Msg("stopping old container after zero-downtime switch")
1637
1638	if s.logWriter != nil {
1639		if err := s.logWriter.StopLogging(old.ID); err != nil {
1640			log.Warn().Err(err).Str(zerowrap.FieldEntityID, old.ID).Msg("failed to stop logging for old container")
1641		}
1642	}
1643
1644	if err := s.runtime.StopContainer(ctx, old.ID); err != nil {
1645		log.Warn().Err(err).Str(zerowrap.FieldEntityID, old.ID).Msg("failed to stop old container")
1646	}
1647
1648	if err := s.runtime.RemoveContainer(ctx, old.ID, true); err != nil {
1649		log.Warn().Err(err).Str(zerowrap.FieldEntityID, old.ID).Msg("failed to remove old container")
1650	}
1651
1652	// Rename new container to canonical name
1653	canonicalName := fmt.Sprintf("gordon-%s", domainName)
1654	if err := s.runtime.RenameContainer(ctx, newContainerID, canonicalName); err != nil {
1655		log.Warn().Err(err).Str("canonical_name", canonicalName).Msg("failed to rename container to canonical name")
1656	}
1657}
1658
1659// startLogCollection starts log collection for a container in the background.
1660// Errors are logged but don't fail the calling operation.
1661func (s *Service) startLogCollection(ctx context.Context, containerID, domainName string) {
1662	if s.logWriter == nil {
1663		return
1664	}
1665
1666	log := zerowrap.FromCtx(ctx)
1667
1668	// Use context.WithoutCancel so the log stream outlives the HTTP request.
1669	// Without this, the Docker log stream closes when the deploy response completes.
1670	bgCtx := context.WithoutCancel(ctx)
1671
1672	logStream, err := s.runtime.GetContainerLogs(bgCtx, containerID, true)
1673	if err != nil {
1674		log.Warn().Err(err).Msg("failed to get container logs for collection")
1675		return
1676	}
1677
1678	if err := s.logWriter.StartLogging(bgCtx, containerID, domainName, logStream); err != nil {
1679		log.Warn().Err(err).Msg("failed to start container log collection")
1680		logStream.Close()
1681	}
1682}
1683
1684func (s *Service) buildValidatedImageRef(ctx context.Context, image string) (string, error) {
1685	imageRef := s.buildImageRef(image)
1686	if err := s.validateImageRef(ctx, imageRef); err != nil {
1687		return "", err
1688	}
1689	return imageRef, nil
1690}
1691
1692func (s *Service) buildImageRef(image string) string {
1693	s.mu.RLock()
1694	cfg := s.config
1695	s.mu.RUnlock()
1696
1697	if !cfg.RegistryAuthEnabled || cfg.RegistryDomain == "" {
1698		return image
1699	}
1700
1701	// Normalize registry domain by trimming any trailing slash
1702	reg := strings.TrimSuffix(cfg.RegistryDomain, "/")
1703
1704	// Check if image already has the registry domain prefix
1705	prefix := reg + "/"
1706	if strings.HasPrefix(image, prefix) {
1707		return image
1708	}
1709
1710	// Check if image already has an explicit registry (host:port/ or host.domain/).
1711	// We need to detect patterns like:
1712	// - "docker.io/library/nginx:latest" (has dot and slash)
1713	// - "localhost:5001/myapp:latest" (has host:port/ pattern)
1714	// - "myregistry:5000/app:v1" (has host:port/ pattern)
1715	// - "[fd00::1]:5000/app:v1" (IPv6 with port)
1716	if hasExplicitRegistry(image) {
1717		return image
1718	}
1719
1720	return fmt.Sprintf("%s/%s", reg, image)
1721}
1722
1723// hasExplicitRegistry checks if an image reference already includes an explicit registry.
1724// Returns true for patterns like "docker.io/image", "localhost:5000/image", "localhost/image",
1725// "registry:8080/image", "[fd00::1]/image", "[fd00::1]:5000/image".
1726func hasExplicitRegistry(image string) bool {
1727	// Find the first slash which separates registry from image name
1728	slashIdx := strings.Index(image, "/")
1729	if slashIdx == -1 {
1730		return false // No slash means no registry prefix (e.g., "myapp:latest")
1731	}
1732
1733	// Extract the part before the first slash (potential registry)
1734	registryPart := image[:slashIdx]
1735
1736	// Check for bracketed IPv6 address (e.g., "[fd00::1]" or "[fd00::1]:5000")
1737	if strings.HasPrefix(registryPart, "[") {
1738		// Look for closing bracket
1739		if closeBracket := strings.Index(registryPart, "]"); closeBracket != -1 {
1740			// Valid bracketed IPv6: either ends at ] or has ]:port
1741			return true
1742		}
1743	}
1744
1745	// Check for localhost (with or without port)
1746	if registryPart == "localhost" {
1747		return true
1748	}
1749
1750	// Check for host:port pattern (e.g., "localhost:5000", "registry:8080")
1751	if colonIdx := strings.LastIndex(registryPart, ":"); colonIdx != -1 {
1752		port := registryPart[colonIdx+1:]
1753		// If everything after colon is digits, it's a port
1754		if len(port) > 0 && isNumeric(port) {
1755			return true
1756		}
1757	}
1758
1759	// Check for domain pattern (has a dot, e.g., "docker.io", "gcr.io")
1760	if strings.Contains(registryPart, ".") {
1761		return true
1762	}
1763
1764	return false
1765}
1766
1767func (s *Service) validateImageRef(ctx context.Context, imageRef string) error {
1768	if !hasExplicitRegistry(imageRef) {
1769		return nil
1770	}
1771	return s.validateExternalImageRef(ctx, imageRef, imageRegistry(imageRef))
1772}
1773
1774func (s *Service) validateImagePullRef(ctx context.Context, imageRef string) error {
1775	return s.validateExternalImageRef(ctx, imageRef, imageRegistryForPolicy(imageRef))
1776}
1777
1778func (s *Service) validateExternalImageRef(ctx context.Context, imageRef, registry string) error {
1779	s.mu.RLock()
1780	cfg := s.config
1781	s.mu.RUnlock()
1782
1783	if registry == "" {
1784		return fmt.Errorf("image %q rejected: invalid image reference", imageRef)
1785	}
1786
1787	if domain.IsGordonRegistryImageRef(imageRef, cfg.RegistryDomain, cfg.LegacyRegistryDomains) || sameRegistry(registry, cfg.RegistryDomain) {
1788		return nil
1789	}
1790	dangerous, err := isDangerousRegistryHost(ctx, imageRegistryHost(registry))
1791	if err != nil {
1792		return fmt.Errorf("image %q rejected: registry %q could not be verified: %w", imageRef, registry, err)
1793	}
1794	if dangerous {
1795		return fmt.Errorf("image %q rejected: registry %q is not allowed", imageRef, registry)
1796	}
1797	if len(cfg.AllowedRegistries) == 0 {
1798		return fmt.Errorf("image %q rejected: registry %q is not in images.allowed_registries", imageRef, registry)
1799	}
1800	for _, allowed := range cfg.AllowedRegistries {
1801		if sameRegistry(registry, allowed) {
1802			if cfg.RequireImageDigest && !isValidSha256DigestRef(imageRef) {
1803				return fmt.Errorf("image %q rejected: digest reference required", imageRef)
1804			}
1805			return nil
1806		}
1807	}
1808	return fmt.Errorf("image %q rejected: registry %q is not in images.allowed_registries", imageRef, registry)
1809}
1810
1811func imageRegistry(imageRef string) string {
1812	slashIdx := strings.Index(imageRef, "/")
1813	if slashIdx == -1 {
1814		return ""
1815	}
1816	return normalizeRegistry(imageRef[:slashIdx])
1817}
1818
1819func imageRegistryForPolicy(imageRef string) string {
1820	if hasExplicitRegistry(imageRef) {
1821		return imageRegistry(imageRef)
1822	}
1823	return "docker.io"
1824}
1825
1826// normalizeRegistry normalizes a registry string for comparison purposes only.
1827// This function is used by sameRegistry to determine if two registry strings
1828// refer to the same registry. The output intentionally strips IPv6 brackets
1829// (e.g., "[fd00::1]:5000" -> "fd00::1:5000") which is acceptable for equality
1830// checks but the result MUST NOT be used to construct or emit valid registry URLs.
1831func normalizeRegistry(registry string) string {
1832	registry = strings.ToLower(strings.TrimSuffix(strings.TrimSpace(registry), "."))
1833	registry = strings.TrimSuffix(registry, "/")
1834	if strings.HasPrefix(registry, "[") {
1835		if end := strings.Index(registry, "]"); end != -1 {
1836			host := strings.Trim(registry[:end+1], "[]")
1837			port := registry[end+1:]
1838			return host + port
1839		}
1840	}
1841	return registry
1842}
1843
1844func imageRegistryHost(registry string) string {
1845	if ip := net.ParseIP(registry); ip != nil {
1846		return registry
1847	}
1848	if h, _, err := net.SplitHostPort(registry); err == nil {
1849		return strings.Trim(h, "[]")
1850	}
1851	if strings.HasPrefix(registry, "[") && strings.Contains(registry, "]") {
1852		return strings.TrimPrefix(strings.Split(registry, "]")[0], "[")
1853	}
1854	if colon := strings.LastIndex(registry, ":"); colon != -1 && isNumeric(registry[colon+1:]) {
1855		return registry[:colon]
1856	}
1857	return registry
1858}
1859
1860func sameRegistry(a, b string) bool {
1861	return normalizeRegistry(a) == normalizeRegistry(b)
1862}
1863
1864func isDangerousRegistryHost(ctx context.Context, host string) (bool, error) {
1865	host = strings.ToLower(strings.TrimSuffix(host, "."))
1866	if host == "localhost" || host == "metadata.google.internal" {
1867		return true, nil
1868	}
1869	if ip := net.ParseIP(host); ip != nil {
1870		return isDangerousRegistryIP(ip), nil
1871	}
1872	addrs, err := net.DefaultResolver.LookupIPAddr(ctx, host)
1873	if err != nil {
1874		return false, err
1875	}
1876	for _, addr := range addrs {
1877		if isDangerousRegistryIP(addr.IP) {
1878			return true, nil
1879		}
1880	}
1881	return false, nil
1882}
1883
1884func isDangerousRegistryIP(ip net.IP) bool {
1885	return ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsUnspecified()
1886}
1887
1888func applySecurityProfile(config *domain.ContainerConfig, cfg Config) {
1889	if strings.EqualFold(cfg.SecurityProfile, "strict") {
1890		config.ReadOnlyRootFS = true
1891		config.CapDrop = []string{"ALL"}
1892		config.CapAdd = []string{"NET_BIND_SERVICE"}
1893	}
1894}
1895
1896// isNumeric checks if a string contains only digits.
1897func isNumeric(s string) bool {
1898	for _, c := range s {
1899		if c < '0' || c > '9' {
1900			return false
1901		}
1902	}
1903	return true
1904}
1905
1906func normalizePullPolicy(policy string) string {
1907	switch strings.ToLower(strings.TrimSpace(policy)) {
1908	case PullPolicyAlways:
1909		return PullPolicyAlways
1910	case PullPolicyIfTagChanged:
1911		return PullPolicyIfTagChanged
1912	case PullPolicyIfNotPresent:
1913		return PullPolicyIfNotPresent
1914	default:
1915		return PullPolicyIfNotPresent
1916	}
1917}
1918
1919func isDigestRef(imageRef string) bool {
1920	return isValidSha256DigestRef(imageRef)
1921}
1922
1923func isValidSha256DigestRef(imageRef string) bool {
1924	_, digest, ok := strings.Cut(imageRef, "@sha256:")
1925	if !ok || len(digest) != 64 {
1926		return false
1927	}
1928	_, err := hex.DecodeString(digest)
1929	return err == nil
1930}
1931
1932func (s *Service) pullRefForDeploy(ctx context.Context, imageRef string) (string, bool) {
1933	if !domain.IsInternalDeploy(ctx) {
1934		return imageRef, false
1935	}
1936	s.mu.RLock()
1937	cfg := s.config
1938	s.mu.RUnlock()
1939	return rewriteToLocalRegistry(imageRef, cfg.RegistryDomain, cfg.LegacyRegistryDomains, cfg.RegistryPort), true
1940}
1941
1942// ensureImage ensures the image is available locally, pulling if needed.
1943// Returns the image reference to use for container operations:
1944//   - For digest references (@sha256:...), returns the pullRef since Docker can't tag digests
1945//   - For tagged images, returns the original imageRef after tagging the pulled image
1946func (s *Service) ensureImage(ctx context.Context, imageRef string) (string, error) {
1947	ctx, span := tracer.Start(ctx, "container.ensure_image",
1948		trace.WithAttributes(attribute.String("image", imageRef)))
1949	defer span.End()
1950
1951	ctx = zerowrap.CtxWithField(ctx, "image", imageRef)
1952	log := zerowrap.FromCtx(ctx)
1953
1954	// Determine if this is an internal deploy and what reference to use for pulls.
1955	pullRef, isInternal := s.pullRefForDeploy(ctx, imageRef)
1956	if err := s.validateImagePullRef(ctx, imageRef); err != nil {
1957		return "", err
1958	}
1959	if pullRef != imageRef {
1960		log.Info().
1961			Str("original_ref", imageRef).
1962			Str("pull_ref", pullRef).
1963			Msg("internal deploy: using localhost registry for pull")
1964	}
1965
1966	found, err := s.ensureLocalImage(ctx, imageRef, pullRef)
1967	if err != nil {
1968		return "", err
1969	}
1970	if found {
1971		return imageRef, nil
1972	}
1973
1974	// Pull image
1975	log.Info().Msg("pulling image from registry")
1976
1977	if err := s.pullImage(ctx, pullRef, isInternal); err != nil {
1978		return "", err
1979	}
1980
1981	// For digest references, we can't create a tag (Docker doesn't allow it).
1982	// In this case, use the pullRef directly since the image is available under that reference.
1983	if strings.Contains(imageRef, "@sha256:") {
1984		log.Info().
1985			Str("pull_ref", pullRef).
1986			Msg("digest reference: using pull reference for container operations")
1987		return pullRef, nil
1988	}
1989
1990	if err := s.tagImageIfNeeded(ctx, pullRef, imageRef); err != nil {
1991		return "", err
1992	}
1993
1994	// Clean up the temporary pull reference tag to avoid duplicate entries
1995	if pullRef != imageRef {
1996		if err := s.runtime.UntagImage(ctx, pullRef); err != nil {
1997			// Log but don't fail - the canonical tag is already applied
1998			log.Debug().Err(err).Str("pull_ref", pullRef).Msg("failed to remove temporary pull tag")
1999		}
2000	}
2001
2002	log.Info().Msg("image pulled successfully")
2003	return imageRef, nil
2004}
2005
2006func (s *Service) ensureLocalImage(ctx context.Context, imageRef, pullRef string) (bool, error) {
2007	log := zerowrap.FromCtx(ctx)
2008
2009	// For internal deploys (image push events), always pull fresh image
2010	// because the same tag may reference new content (e.g., latest tag updated)
2011	if domain.IsInternalDeploy(ctx) {
2012		log.Info().Msg("internal deploy detected, forcing image pull to ensure latest content")
2013		return false, nil
2014	}
2015
2016	s.mu.RLock()
2017	cfg := s.config
2018	s.mu.RUnlock()
2019
2020	pullPolicy := normalizePullPolicy(cfg.PullPolicy)
2021	switch pullPolicy {
2022	case PullPolicyAlways:
2023		log.Info().Str("pull_policy", pullPolicy).Msg("pull policy forces image pull")
2024		return false, nil
2025	case PullPolicyIfTagChanged:
2026		if !isDigestRef(imageRef) {
2027			log.Info().Str("pull_policy", pullPolicy).Msg("tag reference detected, pulling to check for updates")
2028			return false, nil
2029		}
2030	}
2031
2032	localImages, err := s.runtime.ListImages(ctx)
2033	if err != nil {
2034		log.WrapErr(err, "failed to list local images, will attempt pull")
2035		return false, nil
2036	}
2037
2038	normalizedRef := normalizeImageRef(imageRef)
2039	normalizedPullRef := normalizeImageRef(pullRef)
2040	for _, img := range localImages {
2041		normalizedImage := normalizeImageRef(img)
2042		if normalizedImage == normalizedRef {
2043			log.Info().Msg("image found locally, skipping pull")
2044			return true, nil
2045		}
2046		if normalizedImage == normalizedPullRef {
2047			if err := s.tagImageIfNeeded(ctx, pullRef, imageRef); err != nil {
2048				return false, err
2049			}
2050			// Clean up the temporary pull reference tag
2051			if pullRef != imageRef {
2052				if err := s.runtime.UntagImage(ctx, pullRef); err != nil {
2053					log.Debug().Err(err).Str("pull_ref", pullRef).Msg("failed to remove temporary pull tag")
2054				}
2055			}
2056			log.Info().Msg("image found locally, skipping pull")
2057			return true, nil
2058		}
2059	}
2060
2061	return false, nil
2062}
2063
2064func (s *Service) pullImage(ctx context.Context, pullRef string, isInternal bool) error {
2065	s.mu.RLock()
2066	cfg := s.config
2067	s.mu.RUnlock()
2068
2069	log := zerowrap.FromCtx(ctx)
2070
2071	switch {
2072	case isInternal && cfg.RegistryAuthEnabled:
2073		if cfg.InternalRegistryUsername == "" || cfg.InternalRegistryPassword == "" {
2074			return log.WrapErr(fmt.Errorf("internal registry auth not configured"), "failed to pull image for internal deploy")
2075		}
2076		if err := s.pullImageWithRetry(ctx, func(ctx context.Context) error {
2077			return s.runtime.PullImageWithAuth(ctx, pullRef, cfg.InternalRegistryUsername, cfg.InternalRegistryPassword)
2078		}, isInternal); err != nil {
2079			return log.WrapErr(fmt.Errorf("%w: %w", domain.ErrImagePullFailed, err), "failed to pull image with internal auth")
2080		}
2081	case isInternal:
2082		if err := s.pullImageWithRetry(ctx, func(ctx context.Context) error {
2083			return s.runtime.PullImage(ctx, pullRef)
2084		}, isInternal); err != nil {
2085			return log.WrapErr(fmt.Errorf("%w: %w", domain.ErrImagePullFailed, err), "failed to pull image")
2086		}
2087	case cfg.RegistryAuthEnabled:
2088		if cfg.ServiceTokenUsername == "" || cfg.ServiceToken == "" {
2089			return log.WrapErr(fmt.Errorf("registry service token not configured"), "failed to pull image for registry auth")
2090		}
2091		if err := s.runtime.PullImageWithAuth(ctx, pullRef, cfg.ServiceTokenUsername, cfg.ServiceToken); err != nil {
2092			return log.WrapErr(fmt.Errorf("%w: %w", domain.ErrImagePullFailed, err), "failed to pull image with auth")
2093		}
2094	default:
2095		if err := s.runtime.PullImage(ctx, pullRef); err != nil {
2096			return log.WrapErr(fmt.Errorf("%w: %w", domain.ErrImagePullFailed, err), "failed to pull image")
2097		}
2098	}
2099
2100	return nil
2101}
2102
2103func (s *Service) pullImageWithRetry(ctx context.Context, pullFn func(context.Context) error, isInternal bool) error {
2104	log := zerowrap.FromCtx(ctx)
2105
2106	for attempt := 1; attempt <= internalPullMaxAttempts; attempt++ {
2107		err := pullFn(ctx)
2108		if err == nil {
2109			return nil
2110		}
2111		if !isInternal || !isConnectionRefusedError(err) || attempt == internalPullMaxAttempts {
2112			return fmt.Errorf("internal image pull failed after %d attempts: %w", attempt, err)
2113		}
2114
2115		backoff := time.Duration(attempt) * time.Second
2116		log.Warn().
2117			Err(err).
2118			Int("attempt", attempt).
2119			Dur("backoff", backoff).
2120			Msg("internal image pull failed with connection refused, retrying")
2121
2122		select {
2123		case <-time.After(backoff):
2124		case <-ctx.Done():
2125			return ctx.Err()
2126		}
2127	}
2128	return nil
2129}
2130
2131func (s *Service) tagImageIfNeeded(ctx context.Context, sourceRef, targetRef string) error {
2132	if sourceRef == targetRef {
2133		return nil
2134	}
2135
2136	// Cannot create a tag with a digest reference - Docker/Podman doesn't allow it.
2137	// When using digest references (image@sha256:...), skip tagging as the image
2138	// is already available by its digest.
2139	if strings.Contains(targetRef, "@sha256:") {
2140		return nil
2141	}
2142
2143	log := zerowrap.FromCtx(ctx)
2144	if err := s.runtime.TagImage(ctx, sourceRef, targetRef); err != nil {
2145		return log.WrapErr(err, "failed to tag image from pull reference")
2146	}
2147	return nil
2148}
2149
2150func (s *Service) loadEnvironment(ctx context.Context, preResolved []string, domainName, imageRef string) ([]string, error) {
2151	log := zerowrap.FromCtx(ctx)
2152
2153	var userEnvVars []string
2154	if len(preResolved) > 0 {
2155		userEnvVars = preResolved
2156	} else {
2157		var err error
2158		userEnvVars, err = s.envLoader.LoadEnv(ctx, domainName)
2159		if err != nil {
2160			return nil, log.WrapErr(err, "failed to load environment variables")
2161		}
2162	}
2163
2164	dockerfileEnvVars, err := s.runtime.InspectImageEnv(ctx, imageRef)
2165	if err != nil {
2166		log.WrapErr(err, "failed to inspect image environment")
2167		dockerfileEnvVars = []string{}
2168	}
2169
2170	return mergeEnvironmentVariables(dockerfileEnvVars, userEnvVars), nil
2171}
2172
2173func (s *Service) setupVolumes(ctx context.Context, domainName, imageRef string) (map[string]string, error) {
2174	s.mu.RLock()
2175	cfg := s.config
2176	s.mu.RUnlock()
2177
2178	log := zerowrap.FromCtx(ctx)
2179	volumes := make(map[string]string)
2180
2181	if !cfg.VolumeAutoCreate {
2182		return volumes, nil
2183	}
2184
2185	volumePaths, err := s.runtime.InspectImageVolumes(ctx, imageRef)
2186	if err != nil {
2187		log.WrapErr(err, "failed to inspect image volumes")
2188		return volumes, nil
2189	}
2190
2191	for _, path := range volumePaths {
2192		name := generateVolumeName(cfg.VolumePrefix, domainName, path)
2193
2194		exists, err := s.runtime.VolumeExists(ctx, name)
2195		if err != nil {
2196			log.WrapErrWithFields(err, "failed to check volume", map[string]any{"volume": name})
2197			continue
2198		}
2199
2200		if !exists {
2201			if err := s.runtime.CreateVolume(ctx, name); err != nil {
2202				log.WrapErrWithFields(err, "failed to create volume", map[string]any{"volume": name})
2203				continue
2204			}
2205			log.Info().Str("volume", name).Str(zerowrap.FieldPath, path).Msg("created volume")
2206		}
2207
2208		volumes[path] = name
2209	}
2210
2211	return volumes, nil
2212}
2213
2214func (s *Service) getNetworkForApp(domainName string) string {
2215	var networkIsolation bool
2216	var networkGroups map[string][]string
2217
2218	if s.configProvider != nil {
2219		snap := s.configProvider.GetAttachmentConfig()
2220		networkGroups = snap.NetworkGroups
2221		s.mu.RLock()
2222		networkIsolation = s.config.NetworkIsolation
2223		s.mu.RUnlock()
2224	} else {
2225		s.mu.RLock()
2226		networkIsolation = s.config.NetworkIsolation
2227		networkGroups = s.config.NetworkGroups
2228		s.mu.RUnlock()
2229	}
2230
2231	if !networkIsolation {
2232		return "bridge"
2233	}
2234
2235	for groupName, domains := range networkGroups {
2236		if slices.Contains(domains, domainName) {
2237			return s.generateNetworkName(groupName)
2238		}
2239	}
2240
2241	return s.generateNetworkName(domainName)
2242}
2243
2244func (s *Service) generateNetworkName(identifier string) string {
2245	s.mu.RLock()
2246	cfg := s.config
2247	s.mu.RUnlock()
2248
2249	return fmt.Sprintf("%s-%s", cfg.NetworkPrefix, strings.ReplaceAll(identifier, ".", "-"))
2250}
2251
2252func (s *Service) createNetworkIfNeeded(ctx context.Context, networkName string) error {
2253	if networkName == "bridge" || networkName == "default" {
2254		return nil
2255	}
2256
2257	ctx = zerowrap.CtxWithField(ctx, "network", networkName)
2258	log := zerowrap.FromCtx(ctx)
2259
2260	exists, err := s.runtime.NetworkExists(ctx, networkName)
2261	if err != nil {
2262		return log.WrapErr(err, "failed to check network existence")
2263	}
2264
2265	if !exists {
2266		s.mu.RLock()
2267		internal := s.config.NetworkInternal
2268		s.mu.RUnlock()
2269		if err := s.runtime.CreateNetwork(ctx, networkName, domain.NetworkConfig{Driver: "bridge", Internal: internal}); err != nil {
2270			return log.WrapErr(err, "failed to create network")
2271		}
2272		log.Info().Msg("created network for app isolation")
2273	}
2274
2275	return nil
2276}
2277
2278func (s *Service) cleanupOrphanedContainers(ctx context.Context, domainName string, skipContainerID string) error {
2279	log := zerowrap.FromCtx(ctx)
2280	expectedName := fmt.Sprintf("gordon-%s", domainName)
2281	expectedNewName := expectedName + "-new"
2282	expectedNextName := expectedName + "-next"
2283
2284	allContainers, err := s.runtime.ListContainers(ctx, true)
2285	if err != nil {
2286		return err
2287	}
2288
2289	for _, c := range allContainers {
2290		if (c.Name == expectedName || c.Name == expectedNewName || c.Name == expectedNextName) && c.ID != skipContainerID {
2291			// Skip any running container regardless of name — a running
2292			// temp container may be actively serving traffic while the
2293			// new container stabilizes, or may be from a concurrent deploy.
2294			if c.Status == "running" {
2295				log.Debug().
2296					Str(zerowrap.FieldEntityID, c.ID).
2297					Str("container_name", c.Name).
2298					Str(zerowrap.FieldStatus, c.Status).
2299					Msg("skipping running container during orphan cleanup")
2300				continue
2301			}
2302
2303			log.Info().Str(zerowrap.FieldEntityID, c.ID).Str("container_name", c.Name).Str(zerowrap.FieldStatus, c.Status).Msg("found orphaned container, removing")
2304
2305			if err := s.runtime.StopContainer(ctx, c.ID); err != nil {
2306				log.WrapErrWithFields(err, "failed to stop orphaned container", map[string]any{zerowrap.FieldEntityID: c.ID})
2307			}
2308
2309			if err := s.runtime.RemoveContainer(ctx, c.ID, true); err != nil {
2310				log.WrapErrWithFields(err, "failed to remove orphaned container", map[string]any{zerowrap.FieldEntityID: c.ID})
2311			}
2312		}
2313	}
2314
2315	return nil
2316}
2317
2318func (s *Service) cleanupVolumesForDomain(_ context.Context, _ string) error {
2319	return nil
2320}
2321
2322func (s *Service) cleanupNetworkIfEmpty(ctx context.Context, networkName string) error {
2323	if networkName == "bridge" || networkName == "default" {
2324		return nil
2325	}
2326
2327	ctx = zerowrap.CtxWithField(ctx, "network", networkName)
2328	log := zerowrap.FromCtx(ctx)
2329
2330	networks, err := s.runtime.ListNetworks(ctx)
2331	if err != nil {
2332		return log.WrapErr(err, "failed to list networks")
2333	}
2334
2335	for _, n := range networks {
2336		if n.Name == networkName && len(n.Containers) == 0 {
2337			if err := s.runtime.RemoveNetwork(ctx, networkName); err != nil {
2338				return err
2339			}
2340			log.Info().Msg("cleaned up empty network")
2341			break
2342		}
2343	}
2344
2345	return nil
2346}
2347
2348func (s *Service) deployAttachments(ctx context.Context, domainName, networkName string) error {
2349	// Collect attachments from both domain-specific and network group configs
2350	attachments := s.resolveAttachmentsForDomain(domainName)
2351	if len(attachments) == 0 {
2352		return nil
2353	}
2354
2355	log := zerowrap.FromCtx(ctx)
2356	var deployed []string // track successfully deployed attachment IDs for rollback
2357	for _, svc := range attachments {
2358		if err := s.deployAttachedService(ctx, domainName, svc, networkName); err != nil {
2359			log.WrapErrWithFields(err, "failed to deploy attachment", map[string]any{zerowrap.FieldService: svc, "domain": domainName})
2360
2361			// Rollback: clean up already-deployed attachments
2362			for _, id := range deployed {
2363				if stopErr := s.runtime.StopContainer(ctx, id); stopErr != nil {
2364					log.Warn().Err(stopErr).Str("attachment_id", id).Msg("rollback: failed to stop attachment")
2365				}
2366				if rmErr := s.runtime.RemoveContainer(ctx, id, true); rmErr != nil {
2367					log.Warn().Err(rmErr).Str("attachment_id", id).Msg("rollback: failed to remove attachment")
2368				}
2369			}
2370			// Deregister rolled-back attachments
2371			if len(deployed) > 0 {
2372				s.mu.Lock()
2373				remaining := s.attachments[domainName]
2374				filtered := remaining[:0]
2375				rollbackSet := make(map[string]bool, len(deployed))
2376				for _, id := range deployed {
2377					rollbackSet[id] = true
2378				}
2379				for _, id := range remaining {
2380					if !rollbackSet[id] {
2381						filtered = append(filtered, id)
2382					}
2383				}
2384				s.attachments[domainName] = filtered
2385				s.mu.Unlock()
2386			}
2387
2388			return fmt.Errorf("failed to deploy attachment %q (rolled back %d already-deployed)", svc, len(deployed))
2389		}
2390		// Collect IDs of attachments tracked under this domain after successful deploy
2391		s.mu.RLock()
2392		ids := s.attachments[domainName]
2393		if len(ids) > 0 {
2394			latest := ids[len(ids)-1]
2395			deployed = append(deployed, latest)
2396		}
2397		s.mu.RUnlock()
2398	}
2399	return nil
2400}
2401
2402// resolveAttachmentsForDomain returns attachments for a domain by checking:
2403// 1. Direct domain attachments (attachments[domain])
2404// 2. Network group attachments (attachments[group] where domain is in network_groups[group])
2405func (s *Service) resolveAttachmentsForDomain(domainName string) []string {
2406	var attachments map[string][]string
2407	var networkGroups map[string][]string
2408
2409	if s.configProvider != nil {
2410		snap := s.configProvider.GetAttachmentConfig()
2411		attachments = snap.Attachments
2412		networkGroups = snap.NetworkGroups
2413	} else {
2414		s.mu.RLock()
2415		attachments = s.config.Attachments
2416		networkGroups = s.config.NetworkGroups
2417		s.mu.RUnlock()
2418	}
2419
2420	seen := make(map[string]bool)
2421	var result []string
2422
2423	// First, add domain-specific attachments
2424	if domainAttachments, ok := attachments[domainName]; ok {
2425		for _, img := range domainAttachments {
2426			if !seen[img] {
2427				seen[img] = true
2428				result = append(result, img)
2429			}
2430		}
2431	}
2432
2433	// Then, find which network group this domain belongs to and add group attachments
2434	for groupName, domains := range networkGroups {
2435		if slices.Contains(domains, domainName) {
2436			if groupAttachments, ok := attachments[groupName]; ok {
2437				for _, img := range groupAttachments {
2438					if !seen[img] {
2439						seen[img] = true
2440						result = append(result, img)
2441					}
2442				}
2443			}
2444			break // Domain can only be in one network group
2445		}
2446	}
2447
2448	return result
2449}
2450
2451func (s *Service) getAttachmentsForDomain(ctx context.Context, domainName string) []domain.Attachment {
2452	containers, err := s.runtime.ListContainers(ctx, true)
2453	if err != nil {
2454		return nil
2455	}
2456
2457	return s.filterAttachments(ctx, containers, domainName)
2458}
2459
2460// getAllAttachments fetches all containers once and returns attachments grouped by domain.
2461// This avoids N+1 queries when listing multiple routes.
2462func (s *Service) getAllAttachments(ctx context.Context) map[string][]domain.Attachment {
2463	containers, err := s.runtime.ListContainers(ctx, true)
2464	if err != nil {
2465		return nil
2466	}
2467
2468	result := make(map[string][]domain.Attachment)
2469	for _, container := range containers {
2470		if container.Labels == nil {
2471			continue
2472		}
2473		if container.Labels[domain.LabelAttachment] != "true" {
2474			continue
2475		}
2476		ownerDomain := container.Labels[domain.LabelAttachedTo]
2477		if ownerDomain == "" {
2478			continue
2479		}
2480		image := container.Image
2481		serviceName := container.Name
2482		if labelImage, ok := container.Labels[domain.LabelImage]; ok && labelImage != "" {
2483			image = labelImage
2484			serviceName = extractServiceName(labelImage)
2485		}
2486		// Get the container's network for display in the routes table
2487		network := ""
2488		if networkName, err := s.runtime.GetContainerNetwork(ctx, container.ID); err == nil {
2489			network = networkName
2490		}
2491		attachment := domain.Attachment{
2492			Name:        serviceName,
2493			Image:       s.stripRegistryPrefix(image),
2494			ContainerID: container.ID,
2495			Status:      container.Status,
2496			Network:     network,
2497			Ports:       append([]int(nil), container.Ports...),
2498		}
2499		result[ownerDomain] = append(result[ownerDomain], attachment)
2500	}
2501
2502	return result
2503}
2504
2505// filterAttachments extracts attachments for a specific domain from a container list.
2506func (s *Service) filterAttachments(ctx context.Context, containers []*domain.Container, domainName string) []domain.Attachment {
2507	attachments := make([]domain.Attachment, 0)
2508	for _, container := range containers {
2509		if container.Labels == nil {
2510			continue
2511		}
2512		if container.Labels[domain.LabelAttachment] != "true" {
2513			continue
2514		}
2515		if container.Labels[domain.LabelAttachedTo] != domainName {
2516			continue
2517		}
2518		image := container.Image
2519		serviceName := container.Name
2520		if labelImage, ok := container.Labels[domain.LabelImage]; ok && labelImage != "" {
2521			image = labelImage
2522			serviceName = extractServiceName(labelImage)
2523		}
2524		// Get the container's network for display in the routes table
2525		network := ""
2526		if networkName, err := s.runtime.GetContainerNetwork(ctx, container.ID); err == nil {
2527			network = networkName
2528		}
2529		attachment := domain.Attachment{
2530			Name:        serviceName,
2531			Image:       s.stripRegistryPrefix(image),
2532			ContainerID: container.ID,
2533			Status:      container.Status,
2534			Network:     network,
2535			Ports:       append([]int(nil), container.Ports...),
2536		}
2537		attachments = append(attachments, attachment)
2538	}
2539
2540	return attachments
2541}
2542
2543func (s *Service) deployAttachedService(ctx context.Context, ownerDomain, serviceImage, networkName string) error {
2544	log := zerowrap.FromCtx(ctx)
2545
2546	// Parse service name from image (e.g., "my-postgres:latest" → "postgres")
2547	serviceName := extractServiceName(serviceImage)
2548	containerName := fmt.Sprintf("gordon-%s-%s", sanitizeName(ownerDomain), serviceName)
2549
2550	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
2551		"attachment":     serviceName,
2552		"container_name": containerName,
2553		"owner_domain":   ownerDomain,
2554	})
2555
2556	// Check if already running (idempotent)
2557	// Try new name first, then fall back to legacy name for backwards compatibility
2558	existingContainer, err := s.resolveExistingAttachment(ctx, ownerDomain, serviceName)
2559	if err != nil {
2560		return err
2561	}
2562
2563	if existingContainer != nil {
2564		shouldSkip, err := s.handleRunningAttachment(ctx, existingContainer, containerName, serviceImage)
2565		if err != nil {
2566			return err
2567		}
2568		if shouldSkip {
2569			return nil
2570		}
2571		if existingContainer.Status == string(domain.ContainerStatusRunning) {
2572			existingContainer = nil
2573		}
2574	}
2575
2576	if err := s.removeStoppedAttachment(ctx, existingContainer, containerName); err != nil {
2577		return err
2578	}
2579
2580	log.Info().Str(zerowrap.FieldService, serviceImage).Msg("deploying attached service")
2581
2582	config, err := s.buildAttachmentContainerConfig(ctx, ownerDomain, serviceName, serviceImage, containerName, networkName)
2583	if err != nil {
2584		return err
2585	}
2586
2587	container, err := s.runtime.CreateContainer(ctx, config)
2588	if err != nil {
2589		return log.WrapErr(err, "failed to create attachment container")
2590	}
2591
2592	// Start container
2593	if err := s.runtime.StartContainer(ctx, container.ID); err != nil {
2594		s.runtime.RemoveContainer(ctx, container.ID, true)
2595		return log.WrapErr(err, "failed to start attachment container")
2596	}
2597
2598	// Wait for attachment to be ready before proceeding
2599	if err := s.waitForAttachmentReady(ctx, container.ID, config); err != nil {
2600		log.WrapErr(err, "attachment readiness check failed, cleaning up")
2601		if stopErr := s.runtime.StopContainer(ctx, container.ID); stopErr != nil {
2602			log.Warn().Err(stopErr).Str(zerowrap.FieldEntityID, container.ID).Msg("failed to stop unready attachment")
2603		}
2604		s.runtime.RemoveContainer(ctx, container.ID, true)
2605		return fmt.Errorf("attachment %q not ready: %w", serviceName, err)
2606	}
2607	// Track attachment
2608	s.mu.Lock()
2609	s.attachments[ownerDomain] = append(s.attachments[ownerDomain], container.ID)
2610	s.mu.Unlock()
2611
2612	// Start log collection for attachment
2613	s.startLogCollection(ctx, container.ID, containerName)
2614
2615	log.Info().Str(zerowrap.FieldEntityID, container.ID).Msg("attachment deployed successfully")
2616	return nil
2617}
2618
2619func (s *Service) buildAttachmentContainerConfig(ctx context.Context, ownerDomain, serviceName, serviceImage, containerName, networkName string) (*domain.ContainerConfig, error) {
2620	imageRef, err := s.buildValidatedImageRef(ctx, serviceImage)
2621	if err != nil {
2622		return nil, err
2623	}
2624	actualImageRef, err := s.ensureImage(ctx, imageRef)
2625	if err != nil {
2626		return nil, err
2627	}
2628
2629	exposedPorts := s.attachmentExposedPorts(ctx, actualImageRef)
2630	volumes, err := s.attachmentVolumes(ctx, containerName, actualImageRef)
2631	if err != nil {
2632		return nil, err
2633	}
2634	envVars, err := s.attachmentEnv(ctx, containerName, actualImageRef)
2635	if err != nil {
2636		return nil, err
2637	}
2638
2639	s.mu.RLock()
2640	cfg := s.config
2641	s.mu.RUnlock()
2642
2643	config := &domain.ContainerConfig{
2644		Image:       actualImageRef,
2645		Name:        containerName,
2646		Hostname:    serviceName,
2647		Aliases:     []string{serviceName},
2648		Ports:       exposedPorts,
2649		Env:         envVars,
2650		Volumes:     volumes,
2651		NetworkMode: networkName,
2652		Labels: map[string]string{
2653			domain.LabelManaged:    "true",
2654			domain.LabelAttachment: "true",
2655			domain.LabelAttachedTo: ownerDomain,
2656			domain.LabelEnvHash:    hashEnvironment(envVars),
2657			domain.LabelImage:      serviceImage,
2658		},
2659		MemoryLimit:   cfg.DefaultMemoryLimit,
2660		NanoCPUs:      cfg.DefaultNanoCPUs,
2661		PidsLimit:     cfg.DefaultPidsLimit,
2662		RestartPolicy: domain.RestartPolicyAlways,
2663	}
2664	applySecurityProfile(config, cfg)
2665	return config, nil
2666}
2667
2668func (s *Service) attachmentExposedPorts(ctx context.Context, imageRef string) []int {
2669	log := zerowrap.FromCtx(ctx)
2670	exposedPorts, err := s.runtime.GetImageExposedPorts(ctx, imageRef)
2671	if err != nil {
2672		log.WrapErr(err, "failed to get exposed ports for attachment, using defaults")
2673		return []int{}
2674	}
2675	return exposedPorts
2676}
2677
2678func (s *Service) attachmentVolumes(ctx context.Context, containerName, imageRef string) (map[string]string, error) {
2679	volumes, err := s.setupVolumes(ctx, containerName, imageRef)
2680	if err != nil {
2681		return nil, fmt.Errorf("failed to setup volumes for attachment %s with image %s: %w", containerName, imageRef, err)
2682	}
2683	return volumes, nil
2684}
2685
2686func (s *Service) attachmentEnv(ctx context.Context, containerName, imageRef string) ([]string, error) {
2687	envVars, err := s.loadEnvironment(ctx, nil, containerName, imageRef)
2688	if err != nil {
2689		return nil, fmt.Errorf("failed to load environment for attachment %s with image %s: %w", containerName, imageRef, err)
2690	}
2691	return envVars, nil
2692}
2693
2694func (s *Service) attachmentEnvDrifted(ctx context.Context, existing *domain.Container, containerName, serviceImage string) (bool, error) {
2695	imageRef, err := s.buildValidatedImageRef(ctx, serviceImage)
2696	if err != nil {
2697		return false, err
2698	}
2699	currentEnv, err := s.loadEnvironment(ctx, nil, containerName, imageRef)
2700	if err != nil {
2701		return false, err
2702	}
2703
2704	currentHash := hashEnvironment(currentEnv)
2705	existingHash, ok := existing.Labels[domain.LabelEnvHash]
2706	if ok && existingHash == currentHash {
2707		return false, nil
2708	}
2709
2710	return true, nil
2711}
2712
2713func (s *Service) handleRunningAttachment(ctx context.Context, existing *domain.Container, containerName, serviceImage string) (bool, error) {
2714	if existing.Status != string(domain.ContainerStatusRunning) {
2715		return false, nil
2716	}
2717
2718	log := zerowrap.FromCtx(ctx)
2719
2720	// Check image drift
2721	existingImage := existing.Labels[domain.LabelImage]
2722	imageDrifted := existingImage != "" && existingImage != serviceImage
2723
2724	// Check env drift
2725	envDrifted, err := s.attachmentEnvDrifted(ctx, existing, containerName, serviceImage)
2726	if err != nil {
2727		return false, log.WrapErr(err, "failed to check attachment env drift")
2728	}
2729
2730	if !envDrifted && !imageDrifted {
2731		log.Debug().Str("container_name", containerName).Msg("attachment already running with current env and image, skipping")
2732		return true, nil
2733	}
2734
2735	if imageDrifted {
2736		log.Info().Str("container_name", containerName).Str("existing_image", existingImage).Str("desired_image", serviceImage).Msg("attachment image changed, recreating")
2737	}
2738	if envDrifted {
2739		log.Info().Str("container_name", containerName).Msg("attachment env changed, recreating")
2740	}
2741
2742	if err := s.runtime.StopContainer(ctx, existing.ID); err != nil {
2743		return false, log.WrapErr(err, "failed to stop attachment for drift update")
2744	}
2745	if err := s.runtime.RemoveContainer(ctx, existing.ID, true); err != nil {
2746		return false, log.WrapErr(err, "failed to remove attachment for drift update")
2747	}
2748
2749	return false, nil
2750}
2751
2752func (s *Service) removeStoppedAttachment(ctx context.Context, existing *domain.Container, containerName string) error {
2753	if existing == nil {
2754		return nil
2755	}
2756
2757	log := zerowrap.FromCtx(ctx)
2758	log.Info().Str("container_name", containerName).Msg("removing stopped attachment container")
2759	if err := s.runtime.RemoveContainer(ctx, existing.ID, true); err != nil {
2760		return log.WrapErr(err, "failed to remove existing attachment container")
2761	}
2762
2763	return nil
2764}
2765
2766func (s *Service) resolveExistingAttachment(ctx context.Context, ownerDomain, serviceName string) (*domain.Container, error) {
2767	log := zerowrap.FromCtx(ctx)
2768	containerName := fmt.Sprintf("gordon-%s-%s", sanitizeName(ownerDomain), serviceName)
2769	existingContainer := s.findContainerByName(ctx, containerName)
2770	if existingContainer != nil {
2771		return existingContainer, nil
2772	}
2773
2774	containerNameLegacy := fmt.Sprintf("gordon-%s-%s", sanitizeNameLegacy(ownerDomain), serviceName)
2775	existingContainer = s.findContainerByName(ctx, containerNameLegacy)
2776	if existingContainer == nil {
2777		return nil, nil
2778	}
2779
2780	log.Info().Str("container_name", containerNameLegacy).Msg("found attachment with legacy naming, will be replaced")
2781	if err := s.runtime.StopContainer(ctx, existingContainer.ID); err != nil {
2782		return nil, log.WrapErr(err, "failed to stop legacy attachment container")
2783	}
2784	if err := s.runtime.RemoveContainer(ctx, existingContainer.ID, true); err != nil {
2785		return nil, log.WrapErr(err, "failed to remove legacy attachment container")
2786	}
2787
2788	return nil, nil
2789}
2790
2791// Utility functions
2792
2793func normalizeImageRef(image string) string {
2794	if isDigestRef(image) {
2795		return image
2796	}
2797
2798	// Extract repository from image reference, handling port numbers correctly.
2799	// Examples:
2800	//   nginx:latest -> docker.io/library/nginx
2801	//   user/repo:tag -> docker.io/user/repo
2802	//   localhost:5000/image:tag -> localhost:5000/image
2803	//   registry.example.com/image:tag -> registry.example.com/image
2804	//
2805	// Note: This function assumes valid Docker image references. Edge cases like
2806	// "registry.com:5000" (registry with port but no image name) are not valid
2807	// image references per Docker's naming conventions, which require at least
2808	// one path component after the registry (e.g., "registry.com:5000/image").
2809
2810	// Find the tag separator: the last colon that isn't part of a port number.
2811	// A colon is part of a port if there's no slash after it until the next colon.
2812	repo := image
2813	lastColon := strings.LastIndex(image, ":")
2814	if lastColon != -1 {
2815		afterColon := image[lastColon+1:]
2816		// If there's no slash after the colon, it's the tag separator
2817		if !strings.Contains(afterColon, "/") {
2818			repo = image[:lastColon]
2819		}
2820	}
2821
2822	if !strings.Contains(repo, "/") {
2823		return "docker.io/library/" + repo
2824	}
2825
2826	if strings.Count(repo, "/") == 1 && !strings.Contains(strings.Split(repo, "/")[0], ".") && !strings.Contains(strings.Split(repo, "/")[0], ":") {
2827		return "docker.io/" + repo
2828	}
2829
2830	return repo
2831}
2832
2833func generateVolumeName(prefix, domainName, volumePath string) string {
2834	return fmt.Sprintf("%s-%s-%s",
2835		prefix,
2836		strings.ReplaceAll(domainName, ".", "-"),
2837		strings.ReplaceAll(strings.Trim(volumePath, "/"), "/", "-"))
2838}
2839
2840func mergeEnvironmentVariables(dockerfileEnv, userEnv []string) []string {
2841	envMap := make(map[string]string)
2842
2843	for _, env := range dockerfileEnv {
2844		if k, v, ok := strings.Cut(env, "="); ok {
2845			envMap[k] = v
2846		}
2847	}
2848
2849	for _, env := range userEnv {
2850		if k, v, ok := strings.Cut(env, "="); ok {
2851			envMap[k] = v
2852		}
2853	}
2854
2855	keys := make([]string, 0, len(envMap))
2856	for k := range envMap {
2857		keys = append(keys, k)
2858	}
2859	slices.Sort(keys)
2860
2861	result := make([]string, 0, len(envMap))
2862	for _, k := range keys {
2863		result = append(result, k+"="+envMap[k])
2864	}
2865
2866	return result
2867}
2868
2869// extractServiceName gets service name from image reference.
2870// "my-postgres:latest" → "postgres", "redis:7" → "redis"
2871func extractServiceName(image string) string {
2872	// Remove tag
2873	parts := strings.Split(image, ":")
2874	name := parts[0]
2875
2876	// Remove registry prefix if present
2877	if strings.Contains(name, "/") {
2878		nameParts := strings.Split(name, "/")
2879		name = nameParts[len(nameParts)-1]
2880	}
2881
2882	// Remove common prefixes like "my-"
2883	name = strings.TrimPrefix(name, "my-")
2884
2885	return name
2886}
2887
2888// sanitizeName is a convenience wrapper around domain.SanitizeDomainForContainer.
2889func sanitizeName(d string) string {
2890	return domain.SanitizeDomainForContainer(d)
2891}
2892
2893// sanitizeNameLegacy is a convenience wrapper around domain.SanitizeDomainForContainerLegacy.
2894func sanitizeNameLegacy(d string) string {
2895	return domain.SanitizeDomainForContainerLegacy(d)
2896}
2897
2898func isContainerNotFoundError(err error) bool {
2899	if err == nil {
2900		return false
2901	}
2902	if errors.Is(err, domain.ErrContainerNotFound) {
2903		return true
2904	}
2905	msg := strings.ToLower(err.Error())
2906	return strings.Contains(msg, "no such container") ||
2907		strings.Contains(msg, "no container with name or id") ||
2908		strings.Contains(msg, "container not found")
2909}
2910
2911func isConnectionRefusedError(err error) bool {
2912	if err == nil {
2913		return false
2914	}
2915	return strings.Contains(strings.ToLower(err.Error()), "connection refused")
2916}
2917
2918// findContainerByName finds a container by its name.
2919func (s *Service) findContainerByName(ctx context.Context, name string) *domain.Container {
2920	containers, err := s.runtime.ListContainers(ctx, true)
2921	if err != nil {
2922		return nil
2923	}
2924
2925	for _, c := range containers {
2926		if c.Name == name {
2927			return c
2928		}
2929	}
2930	return nil
2931}
2932
2933func (s *Service) waitForReady(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig) error {
2934	if err := s.pollContainerRunning(ctx, containerID); err != nil {
2935		return err
2936	}
2937
2938	s.mu.RLock()
2939	cfg := s.config
2940	s.mu.RUnlock()
2941
2942	readinessMode := cfg.ReadinessMode
2943	if readinessMode == "" {
2944		readinessMode = "delay"
2945	}
2946
2947	switch readinessMode {
2948	case "delay":
2949		return s.waitForReadyByDelay(ctx, containerID)
2950	case "docker-health":
2951		_, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(ctx, containerID)
2952		if err != nil {
2953			return err
2954		}
2955		if !hasHealthcheck {
2956			return errors.New("no healthcheck detected")
2957		}
2958		return s.waitForHealthy(ctx, containerID, cfg.HealthTimeout)
2959	default: // auto
2960		return s.readinessCascade(ctx, containerID, containerConfig, cfg)
2961	}
2962}
2963
2964// readinessCascade auto-detects the strongest available readiness signal:
2965//  1. Docker healthcheck (if present) → wait for healthy status
2966//  2. HTTP probe (if gordon.health label set) → GET until 2xx/3xx
2967//  3. Default HTTP probe (GET / on exposed port) → wait for 2xx/3xx
2968//  4. TCP probe (if port info available) → connect until accepted
2969//  5. Delay fallback (last resort) → waitForReadyByDelay
2970func (s *Service) readinessCascade(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) error {
2971	log := zerowrap.FromCtx(ctx)
2972
2973	// 1. Docker healthcheck
2974	_, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(ctx, containerID)
2975	if err != nil {
2976		return err
2977	}
2978	if hasHealthcheck {
2979		log.Info().Msg("readiness cascade: using Docker healthcheck")
2980		return s.waitForHealthy(ctx, containerID, cfg.HealthTimeout)
2981	}
2982
2983	// 2. HTTP probe via gordon.health label
2984	if probed, probeErr := s.tryHTTPProbe(ctx, containerID, containerConfig, cfg); probed {
2985		return probeErr
2986	}
2987
2988	// 3. Default HTTP probe on root path (if port info available)
2989	if probed, probeErr := s.tryDefaultHTTPProbe(ctx, containerID, containerConfig, cfg); probed {
2990		return probeErr
2991	}
2992
2993	// 4. TCP probe (if port info available)
2994	if probed, probeErr := s.tryTCPProbe(ctx, containerID, containerConfig, cfg); probed {
2995		return probeErr
2996	}
2997
2998	// 5. Delay fallback
2999	log.Info().Msg("readiness cascade: using delay fallback")
3000	return s.waitForReadyByDelay(ctx, containerID)
3001}
3002
3003// waitForAttachmentReady runs a reduced readiness cascade for attachments:
3004// Docker healthcheck → TCP probe → delay fallback.
3005// HTTP probes are skipped — attachments are typically databases/caches.
3006func (s *Service) waitForAttachmentReady(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig) error {
3007	if err := s.pollContainerRunning(ctx, containerID); err != nil {
3008		return err
3009	}
3010
3011	log := zerowrap.FromCtx(ctx)
3012
3013	s.mu.RLock()
3014	cfg := s.config
3015	s.mu.RUnlock()
3016
3017	// 1. Docker healthcheck (if present)
3018	_, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(ctx, containerID)
3019	if err != nil {
3020		return err
3021	}
3022	if hasHealthcheck {
3023		log.Info().Msg("attachment readiness: using Docker healthcheck")
3024		timeout := cfg.HealthTimeout
3025		if timeout == 0 {
3026			timeout = 90 * time.Second
3027		}
3028		return s.waitForHealthy(ctx, containerID, timeout)
3029	}
3030
3031	// 2. TCP probe (if port info available)
3032	if containerConfig != nil && len(containerConfig.Ports) > 0 {
3033		ip, port, probeErr := s.resolveProbeEndpoint(ctx, containerID, containerConfig)
3034		if probeErr == nil && ip != "" && port > 0 {
3035			addr := fmt.Sprintf("%s:%d", ip, port)
3036			timeout := cfg.AttachmentReadinessTimeout
3037			if timeout == 0 {
3038				timeout = 30 * time.Second
3039			}
3040			log.Info().Str("addr", addr).Dur("timeout", timeout).Msg("attachment readiness: using TCP probe")
3041			return tcpProbe(ctx, addr, timeout)
3042		}
3043		log.Debug().Err(probeErr).Msg("attachment readiness: TCP probe skipped, could not resolve endpoint")
3044	}
3045
3046	// 3. Delay fallback
3047	log.Info().Msg("attachment readiness: using delay fallback")
3048	return s.waitForReadyByDelay(ctx, containerID)
3049}
3050
3051// tryHTTPProbe attempts an HTTP probe if the gordon.health label is set.
3052// Returns (true, err) if the probe was attempted, (false, nil) if skipped.
3053func (s *Service) tryHTTPProbe(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) (bool, error) {
3054	log := zerowrap.FromCtx(ctx)
3055	if containerConfig == nil {
3056		return false, nil
3057	}
3058	healthPath, ok := containerConfig.Labels[domain.LabelHealth]
3059	if !ok || healthPath == "" {
3060		return false, nil
3061	}
3062	ip, port, probeErr := s.resolveProbeEndpoint(ctx, containerID, containerConfig)
3063	if probeErr != nil || ip == "" || port <= 0 {
3064		log.Debug().Err(probeErr).Msg("readiness cascade: HTTP probe skipped, could not resolve container endpoint")
3065		return false, nil
3066	}
3067	url := fmt.Sprintf("http://%s:%d%s", ip, port, healthPath)
3068	timeout := cfg.HTTPProbeTimeout
3069	if timeout == 0 {
3070		timeout = 60 * time.Second
3071	}
3072	log.Info().Str("url", url).Dur("timeout", timeout).Msg("readiness cascade: using HTTP probe")
3073	return true, httpProbe(ctx, url, timeout)
3074}
3075
3076// tryDefaultHTTPProbe attempts an HTTP GET on "/" using the container's
3077// exposed port. This is the default readiness check for web containers —
3078// it verifies the app is actually serving HTTP, not just accepting TCP.
3079// Returns (true, err) if the probe was attempted, (false, nil) if skipped.
3080func (s *Service) tryDefaultHTTPProbe(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) (bool, error) {
3081	log := zerowrap.FromCtx(ctx)
3082	if containerConfig == nil || len(containerConfig.Ports) == 0 {
3083		return false, nil
3084	}
3085	ip, port, probeErr := s.resolveProbeEndpoint(ctx, containerID, containerConfig)
3086	if probeErr != nil || ip == "" || port <= 0 {
3087		log.Debug().Err(probeErr).Msg("readiness cascade: default HTTP probe skipped, could not resolve container endpoint")
3088		return false, nil
3089	}
3090	url := fmt.Sprintf("http://%s:%d/", ip, port)
3091	timeout := cfg.HTTPProbeTimeout
3092	if timeout == 0 {
3093		timeout = 60 * time.Second
3094	}
3095	log.Info().Str("url", url).Dur("timeout", timeout).Msg("readiness cascade: using default HTTP alive probe")
3096	return true, httpAliveProbe(ctx, url, timeout)
3097}
3098
3099// tryTCPProbe attempts a TCP probe if port info is available.
3100// Returns (true, err) if the probe was attempted, (false, nil) if skipped.
3101func (s *Service) tryTCPProbe(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) (bool, error) {
3102	log := zerowrap.FromCtx(ctx)
3103	if containerConfig == nil || len(containerConfig.Ports) == 0 {
3104		return false, nil
3105	}
3106	ip, port, probeErr := s.resolveProbeEndpoint(ctx, containerID, containerConfig)
3107	if probeErr != nil || ip == "" || port <= 0 {
3108		log.Debug().Err(probeErr).Msg("readiness cascade: TCP probe skipped, could not resolve container endpoint")
3109		return false, nil
3110	}
3111	addr := fmt.Sprintf("%s:%d", ip, port)
3112	timeout := cfg.TCPProbeTimeout
3113	if timeout == 0 {
3114		timeout = 30 * time.Second
3115	}
3116	log.Info().Str("addr", addr).Dur("timeout", timeout).Msg("readiness cascade: using TCP probe")
3117	return true, tcpProbe(ctx, addr, timeout)
3118}
3119
3120// resolveContainerEndpoint returns a host-reachable address for probing.
3121// In rootless podman/Docker setups, container internal IPs are not routable
3122// from the host. We use the host port binding (127.0.0.1:<mapped_port>)
3123// which is always reachable. Falls back to the container's internal IP
3124// only if no host port mapping exists (e.g. host-network mode).
3125func (s *Service) resolveProbeEndpoint(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig) (string, int, error) {
3126	log := zerowrap.FromCtx(ctx)
3127
3128	internalPort := 0
3129	if containerConfig != nil {
3130		if labels := containerConfig.Labels; labels != nil {
3131			for _, key := range []string{domain.LabelProxyPort, domain.LabelPort} {
3132				if portStr, ok := labels[key]; ok && portStr != "" {
3133					port, err := strconv.Atoi(portStr)
3134					if err == nil && port > 0 && port <= 65535 {
3135						internalPort = port
3136						break
3137					}
3138					log.Warn().Str("label", key).Str("port_value", portStr).Msg("invalid probe port label value")
3139				}
3140			}
3141		}
3142	}
3143
3144	if internalPort == 0 {
3145		_, resolvedPort, err := s.runtime.GetContainerNetworkInfo(ctx, containerID)
3146		if err != nil {
3147			return "", 0, err
3148		}
3149		internalPort = resolvedPort
3150	}
3151
3152	hostPort, hostErr := s.runtime.GetContainerPort(ctx, containerID, internalPort)
3153	if hostErr == nil && hostPort > 0 {
3154		log.Debug().
3155			Int("internal_port", internalPort).
3156			Int("host_port", hostPort).
3157			Msg("resolved container endpoint via host port binding")
3158		return "127.0.0.1", hostPort, nil
3159	}
3160
3161	ip, _, fallbackErr := s.runtime.GetContainerNetworkInfo(ctx, containerID)
3162	if fallbackErr != nil {
3163		return "", 0, fallbackErr
3164	}
3165	log.Debug().
3166		Str("ip", ip).
3167		Int("port", internalPort).
3168		Msg("resolved container endpoint via internal IP (no host port binding)")
3169	return ip, internalPort, nil
3170}
3171
3172// waitForReadyByDelay waits using the legacy running+delay strategy.
3173func (s *Service) waitForReadyByDelay(ctx context.Context, containerID string) error {
3174	log := zerowrap.FromCtx(ctx)
3175
3176	s.mu.RLock()
3177	cfg := s.config
3178	s.mu.RUnlock()
3179
3180	delay := cfg.ReadinessDelay
3181	if delay == 0 {
3182		delay = 5 * time.Second
3183	}
3184
3185	log.Debug().Dur("delay", delay).Msg("waiting for container readiness")
3186	select {
3187	case <-time.After(delay):
3188	case <-ctx.Done():
3189		return ctx.Err()
3190	}
3191
3192	// Verify still running after delay
3193	running, err := s.runtime.IsContainerRunning(ctx, containerID)
3194	if err != nil {
3195		return err
3196	}
3197	if running {
3198		return nil
3199	}
3200
3201	return s.waitForRecovery(ctx, containerID)
3202}
3203
3204func (s *Service) waitForHealthy(ctx context.Context, containerID string, timeout time.Duration) error {
3205	if timeout == 0 {
3206		timeout = 90 * time.Second
3207	}
3208	waitCtx, cancel := context.WithTimeout(ctx, timeout)
3209	defer cancel()
3210
3211	var lastStatus string
3212	for {
3213		status, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(waitCtx, containerID)
3214		if err != nil {
3215			return err
3216		}
3217		if !hasHealthcheck {
3218			return errors.New("no healthcheck detected")
3219		}
3220		// Treat empty health status as transitional startup. Some runtimes can
3221		// temporarily report an empty status before first probe results.
3222		if status == "" {
3223			status = "starting"
3224		}
3225		if status == "healthy" {
3226			return nil
3227		}
3228		if status == "unhealthy" {
3229			return errors.New("container reported unhealthy")
3230		}
3231		lastStatus = status
3232
3233		select {
3234		case <-time.After(time.Second):
3235		case <-waitCtx.Done():
3236			return fmt.Errorf("container healthcheck timeout after %s (last status: %s)", timeout, lastStatus)
3237		}
3238	}
3239}
3240
3241// pollContainerRunning polls until the container is running (max 30 seconds).
3242func (s *Service) pollContainerRunning(ctx context.Context, containerID string) error {
3243	for i := 0; i < 30; i++ {
3244		running, err := s.runtime.IsContainerRunning(ctx, containerID)
3245		if err != nil {
3246			return err
3247		}
3248		if running {
3249			return nil
3250		}
3251		// Check if the container has already exited (crash) rather than still starting up.
3252		if info, inspectErr := s.runtime.InspectContainer(ctx, containerID); inspectErr == nil && info != nil && info.Status == string(domain.ContainerStatusExited) {
3253			return fmt.Errorf("container exited immediately with code %d: %w", info.ExitCode, domain.ErrContainerExited)
3254		}
3255		if i == 29 {
3256			return fmt.Errorf("container did not start within 30 seconds")
3257		}
3258		select {
3259		case <-time.After(time.Second):
3260		case <-ctx.Done():
3261			return ctx.Err()
3262		}
3263	}
3264	return fmt.Errorf("container did not start within 30 seconds")
3265}
3266
3267// waitForRecovery waits for a container to recover within the recovery window.
3268func (s *Service) waitForRecovery(ctx context.Context, containerID string) error {
3269	log := zerowrap.FromCtx(ctx)
3270	log.Warn().
3271		Dur("recovery_window", readinessRecoveryWindow).
3272		Msg("container not running after readiness delay, waiting for recovery")
3273
3274	deadline := time.Now().Add(readinessRecoveryWindow)
3275	for time.Now().Before(deadline) {
3276		select {
3277		case <-time.After(time.Second):
3278		case <-ctx.Done():
3279			return ctx.Err()
3280		}
3281
3282		running, err := s.runtime.IsContainerRunning(ctx, containerID)
3283		if err != nil {
3284			return err
3285		}
3286		if running {
3287			log.Info().Msg("container recovered during readiness recovery window")
3288			return nil
3289		}
3290	}
3291
3292	return fmt.Errorf("container not running after readiness delay and recovery window")
3293}
3294
3295// publishContainerDeployed publishes a container.deployed event.
3296func (s *Service) publishContainerDeployed(ctx context.Context, domainName, containerID string) {
3297	payload := &domain.ContainerEventPayload{
3298		ContainerID: containerID,
3299		Domain:      domainName,
3300		Action:      "deployed",
3301	}
3302
3303	if err := s.eventBus.Publish(domain.EventContainerDeployed, payload); err != nil {
3304		log := zerowrap.FromCtx(ctx)
3305		log.Warn().Err(err).Msg("failed to publish container deployed event")
3306	}
3307}
3308
3309// rewriteToRegistryDomain rewrites an image reference to use the configured registry domain.
3310// e.g., "myapp:latest" -> "registry.example.com/myapp:latest"
3311func rewriteToRegistryDomain(imageRef, registryDomain string) string {
3312	if registryDomain == "" {
3313		return imageRef
3314	}
3315
3316	prefix := registryDomain + "/"
3317	if strings.HasPrefix(imageRef, prefix) {
3318		return imageRef
3319	}
3320
3321	return prefix + imageRef
3322}
3323
3324// rewriteToLocalRegistry rewrites Gordon-managed image references to use the
3325// local registry address. Current and legacy Gordon registry domains are
3326// stripped before prefixing localhost:<registryPort>/. Bare refs are prefixed.
3327// External registries are preserved unchanged.
3328func rewriteToLocalRegistry(imageRef, registryDomain string, legacyRegistryDomains []string, registryPort int) string {
3329	if imageRef == "" {
3330		return imageRef
3331	}
3332
3333	localRegistry := fmt.Sprintf("localhost:%d", registryPort)
3334	localPrefix := localRegistry + "/"
3335	if strings.HasPrefix(imageRef, localPrefix) {
3336		return imageRef
3337	}
3338
3339	if domain.IsGordonRegistryImageRef(imageRef, registryDomain, legacyRegistryDomains) {
3340		return localPrefix + domain.StripKnownGordonRegistry(imageRef, registryDomain, legacyRegistryDomains)
3341	}
3342	if hasExplicitRegistry(imageRef) {
3343		return imageRef
3344	}
3345
3346	return localPrefix + imageRef
3347}