service.go RawHistoryFolder

1// Package container implements the container management use case.
2package container
3
4import (
5	"bufio"
6	"context"
7	"encoding/binary"
8	"encoding/hex"
9	"errors"
10	"fmt"
11	"io"
12	"maps"
13	"net"
14	"slices"
15	"strconv"
16	"strings"
17	"sync"
18	"time"
19
20	"github.com/bnema/zerowrap"
21	"go.opentelemetry.io/otel"
22	"go.opentelemetry.io/otel/attribute"
23	"go.opentelemetry.io/otel/codes"
24	"go.opentelemetry.io/otel/metric"
25	"go.opentelemetry.io/otel/trace"
26
27	"github.com/bnema/gordon/internal/adapters/out/telemetry"
28	"github.com/bnema/gordon/internal/boundaries/out"
29	"github.com/bnema/gordon/internal/domain"
30)
31
32// Config holds configuration needed by the container service.
33type Config struct {
34	RegistryAuthEnabled        bool
35	RegistryDomain             string
36	LegacyRegistryDomains      []string
37	RegistryPort               int
38	ServiceTokenUsername       string
39	ServiceToken               string
40	InternalRegistryUsername   string
41	InternalRegistryPassword   string
42	PullPolicy                 string
43	VolumeAutoCreate           bool
44	VolumePrefix               string
45	VolumePreserve             bool
46	NetworkIsolation           bool
47	NetworkPrefix              string
48	NetworkGroups              map[string][]string
49	NetworkInternal            bool
50	Attachments                map[string][]string
51	AllowedRegistries          []string
52	RequireImageDigest         bool
53	SecurityProfile            string
54	ReadinessDelay             time.Duration // Delay after container starts before considering it ready
55	ReadinessMode              string        // Readiness strategy: auto, docker-health, delay
56	HealthTimeout              time.Duration // Max wait for health-based readiness
57	DrainDelay                 time.Duration // Grace period after cache invalidation before stopping old container
58	DrainDelayConfigured       bool          // True when deploy.drain_delay was explicitly configured
59	DrainMode                  string        // Drain strategy: auto, inflight, delay
60	DrainTimeout               time.Duration // Max wait for in-flight requests to drain
61	StabilizationDelay         time.Duration // Post-switch monitoring window (default 2s)
62	TCPProbeTimeout            time.Duration // TCP probe timeout (default 30s)
63	HTTPProbeTimeout           time.Duration // HTTP probe timeout (default 60s)
64	AttachmentReadinessTimeout time.Duration // Max wait for attachment readiness (default 30s)
65	DefaultMemoryLimit         int64         // Default memory limit in bytes for containers (0 = no limit)
66	DefaultNanoCPUs            int64         // Default CPU quota in nanoseconds for containers (0 = no limit)
67	DefaultPidsLimit           int64         // Default max PIDs for containers (0 = no limit)
68}
69
70var tracer = otel.Tracer("gordon.container")
71
72const (
73	PullPolicyAlways       = "always"
74	PullPolicyIfNotPresent = "if-not-present"
75	PullPolicyIfTagChanged = "if-tag-changed"
76
77	// readinessRecoveryWindow is an additional grace window used when a container
78	// is briefly not running at the end of readiness delay. This avoids false
79	// negatives during short startup flaps.
80	readinessRecoveryWindow = 30 * time.Second
81	internalPullMaxAttempts = 3
82	deployFailureLogLimit   = 20
83)
84
85// Service implements the ContainerService interface.
86type Service struct {
87	runtime          out.ContainerRuntime
88	envLoader        out.EnvLoader
89	eventBus         out.EventPublisher
90	logWriter        out.ContainerLogWriter
91	cacheInvalidator out.ProxyCacheInvalidator
92	drainWaiter      out.ProxyDrainWaiter
93	config           Config
94	configProvider   AttachmentConfigProvider // live config reads for attachments/networks (may be nil)
95	metrics          *telemetry.Metrics
96	containers       map[string]*domain.Container
97	attachments      map[string][]string // ownerDomain → []containerIDs
98	managedCount     int64               // tracks UpDownCounter value for delta computation
99	mu               sync.RWMutex
100	deployMu         sync.Map       // per-domain deploy locks (domain → *domainDeployLock)
101	cleanupWg        sync.WaitGroup // tracks background old-container cleanup goroutines
102	monitor          *Monitor
103}
104
105// domainDeployLock is a context-aware mutex using a buffered channel.
106// It allows Lock() to be interrupted by context cancellation.
107type domainDeployLock struct {
108	ch chan struct{}
109}
110
111// newDomainDeployLock creates a new lock that is initially unlocked.
112func newDomainDeployLock() *domainDeployLock {
113	l := &domainDeployLock{ch: make(chan struct{}, 1)}
114	l.ch <- struct{}{}
115	return l
116}
117
118// Lock acquires the lock or returns an error if the context is cancelled.
119func (l *domainDeployLock) Lock(ctx context.Context) error {
120	// Fast path: honor already-canceled contexts before blocking.
121	if err := ctx.Err(); err != nil {
122		return err
123	}
124	select {
125	case <-ctx.Done():
126		return ctx.Err()
127	case <-l.ch:
128		// Re-check context after acquisition: if canceled while we were
129		// in the select, release the token so the lock isn't consumed.
130		if err := ctx.Err(); err != nil {
131			l.ch <- struct{}{}
132			return err
133		}
134		return nil
135	}
136}
137
138// Unlock releases the lock.
139func (l *domainDeployLock) Unlock() {
140	select {
141	case l.ch <- struct{}{}:
142	default:
143		// This should never happen if Lock/Unlock are used correctly.
144		// Panic to detect double-unlocks or other misuse early.
145		panic("domainDeployLock: double unlock detected")
146	}
147}
148
149// NewService creates a new container service.
150func NewService(
151	runtime out.ContainerRuntime,
152	envLoader out.EnvLoader,
153	eventBus out.EventPublisher,
154	logWriter out.ContainerLogWriter,
155	config Config,
156	configProvider AttachmentConfigProvider,
157) *Service {
158	return &Service{
159		runtime:        runtime,
160		envLoader:      envLoader,
161		eventBus:       eventBus,
162		logWriter:      logWriter,
163		config:         config,
164		configProvider: configProvider,
165		containers:     make(map[string]*domain.Container),
166		attachments:    make(map[string][]string),
167	}
168}
169
170// SetMetrics sets the telemetry metrics for the container service.
171func (s *Service) SetMetrics(m *telemetry.Metrics) {
172	s.metrics = m
173}
174
175// SetProxyCacheInvalidator sets the proxy cache invalidator for synchronous
176// cache invalidation during zero-downtime deployments. This must be called
177// after construction because the proxy service is created after the container
178// service during application initialization.
179func (s *Service) SetProxyCacheInvalidator(inv out.ProxyCacheInvalidator) {
180	s.mu.Lock()
181	s.cacheInvalidator = inv
182	s.mu.Unlock()
183}
184
185// SetProxyDrainWaiter sets the proxy in-flight drain waiter used during
186// zero-downtime replacement before stopping the old container.
187func (s *Service) SetProxyDrainWaiter(waiter out.ProxyDrainWaiter) {
188	s.mu.Lock()
189	s.drainWaiter = waiter
190	s.mu.Unlock()
191}
192
193// domainDeployMu returns a per-domain lock for serializing deploy operations.
194func (s *Service) domainDeployMu(domain string) *domainDeployLock {
195	v, _ := s.deployMu.LoadOrStore(domain, newDomainDeployLock())
196	// Type assertion is safe: LoadOrStore always stores *domainDeployLock values,
197	// and any concurrent LoadOrStore calls will also store *domainDeployLock.
198	return v.(*domainDeployLock)
199}
200
201// acquireDomainDeployLock acquires the per-domain deploy lock with context cancellation support.
202// Returns an unlock function that must be called to release the lock.
203func (s *Service) acquireDomainDeployLock(ctx context.Context, domain string) (func(), error) {
204	lock := s.domainDeployMu(domain)
205	if err := lock.Lock(ctx); err != nil {
206		return nil, err
207	}
208	return func() { lock.Unlock() }, nil
209}
210
211// buildContainerConfig constructs the container configuration for deployment.
212func (s *Service) deploymentContainerName(containerDomain string, existing *domain.Container) string {
213	canonicalName := fmt.Sprintf("gordon-%s", containerDomain)
214	if existing == nil {
215		return canonicalName
216	}
217
218	// Alternate temporary names if the tracked container was left with a temp suffix.
219	// This prevents name collisions after interrupted zero-downtime deploys.
220	newName := canonicalName + "-new"
221	nextName := canonicalName + "-next"
222	switch existing.Name {
223	case newName:
224		return nextName
225	case nextName:
226		return newName
227	default:
228		return newName
229	}
230}
231
232// containerConfigInput groups the parameters for building a container configuration.
233type containerConfigInput struct {
234	Domain       string
235	Image        string
236	ImageRef     string
237	ExposedPorts []int
238	EnvVars      []string
239	EnvHash      string
240	Volumes      map[string]string
241	NetworkName  string
242	ImageLabels  map[string]string
243	Existing     *domain.Container
244}
245
246func (s *Service) buildContainerConfig(in containerConfigInput) *domain.ContainerConfig {
247	s.mu.RLock()
248	cfg := s.config
249	s.mu.RUnlock()
250
251	containerName := s.deploymentContainerName(in.Domain, in.Existing)
252	labels := map[string]string{
253		domain.LabelDomain:  in.Domain,
254		domain.LabelEnvHash: in.EnvHash,
255		domain.LabelImage:   in.Image,
256		domain.LabelManaged: "true",
257		domain.LabelRoute:   in.Domain,
258	}
259
260	// Propagate proxy/port labels from image so readiness probes can find them
261	for _, key := range []string{domain.LabelProxyPort, domain.LabelPort, domain.LabelHealth} {
262		if v, ok := in.ImageLabels[key]; ok && v != "" {
263			labels[key] = v
264		}
265	}
266
267	// Ensure the label-specified proxy port is included in exposed ports
268	// so Docker creates a host port binding for it.
269	ports := in.ExposedPorts
270	if portStr, ok := labels[domain.LabelProxyPort]; ok {
271		if port, err := strconv.Atoi(portStr); err == nil && port > 0 && port <= 65535 {
272			if !slices.Contains(ports, port) {
273				ports = append(ports, port)
274			}
275		}
276	}
277
278	containerConfig := &domain.ContainerConfig{
279		Image:         in.ImageRef,
280		Name:          containerName,
281		Ports:         ports,
282		Env:           in.EnvVars,
283		Volumes:       in.Volumes,
284		NetworkMode:   in.NetworkName,
285		Hostname:      in.Domain,
286		Labels:        labels,
287		AutoRemove:    false,
288		RestartPolicy: domain.RestartPolicyAlways,
289		MemoryLimit:   cfg.DefaultMemoryLimit,
290		NanoCPUs:      cfg.DefaultNanoCPUs,
291		PidsLimit:     cfg.DefaultPidsLimit,
292	}
293	applySecurityProfile(containerConfig, cfg)
294	return containerConfig
295}
296
297// Deploy creates and starts a container for the given route.
298// Implements zero-downtime deployment: new container starts before old one stops.
299func (s *Service) Deploy(ctx context.Context, route domain.Route) (*domain.Container, error) {
300	ctx, span := tracer.Start(ctx, "container.deploy",
301		trace.WithAttributes(
302			attribute.String("domain", route.Domain),
303			attribute.String("image", route.Image),
304		))
305	defer span.End()
306
307	deployStart := time.Now()
308
309	// Serialize deploys for the same domain to prevent race conditions
310	// (e.g. multiple image.pushed events + explicit deploy call from CLI).
311	unlock, err := s.acquireDomainDeployLock(ctx, route.Domain)
312	if err != nil {
313		return nil, err
314	}
315	defer unlock()
316
317	// Enrich context with use case fields for all downstream logs
318	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
319		zerowrap.FieldLayer:   "usecase",
320		zerowrap.FieldUseCase: "Deploy",
321		"domain":              route.Domain,
322	})
323	log := zerowrap.FromCtx(ctx)
324
325	// Record deploy metrics and trace status
326	defer s.recordDeployMetrics(ctx, span, route, deployStart, &err)
327
328	existing, hasExisting := s.resolveExistingContainer(ctx, route.Domain)
329
330	resources, err := s.prepareDeployResources(ctx, route, existing)
331	if err != nil {
332		if deployErr := s.wrapPullDeployFailure(route, existing, err); deployErr != nil {
333			err = deployErr
334			return nil, err
335		}
336		return nil, err
337	}
338
339	// Skip redundant deploy: if the existing container is already running
340	// the exact same image (by Docker image ID), return it immediately.
341	// This prevents the double-deploy caused by the event-based deploy
342	// (triggered by image.pushed) racing with the explicit CLI deploy call.
343	if hasExisting {
344		existingForSkip := existing
345		if existingForSkip.ImageID == "" && existingForSkip.Image != "" && normalizeImageRef(existingForSkip.Image) == normalizeImageRef(route.Image) {
346			existingForSkip = s.containerForRedundantCheck(ctx, existingForSkip)
347		}
348		if existingForSkip.ImageID != "" {
349			if skip, container := s.skipRedundantDeploy(ctx, existingForSkip, resources.actualImageRef, resources.envHash); skip {
350				return container, nil
351			}
352		}
353	}
354
355	newContainer, err := s.createStartedContainer(ctx, route, existing, resources)
356	if err != nil {
357		return nil, err
358	}
359
360	invalidated := s.activateDeployedContainer(ctx, route.Domain, newContainer)
361
362	// Post-switch stabilization: verify new container stays running
363	if hasExisting {
364		stable, stabilizeErr := s.stabilizeNewContainer(ctx, route.Domain, newContainer, existing)
365		if stabilizeErr != nil {
366			// Both old and new containers are dead; assign to named return so
367			// the deferred recordDeployMetrics and span see the failure.
368			err = stabilizeErr
369			return nil, err
370		}
371		if !stable {
372			// Rollback performed — old container is restored
373			return existing, nil
374		}
375	}
376
377	// Finalize old container in the background — the new container is already
378	// serving traffic, so there's no reason to block the deploy response while
379	// waiting for the old container to stop (which can take 20s if the app
380	// doesn't handle SIGTERM).
381	s.cleanupWg.Add(1)
382	go func() {
383		defer s.cleanupWg.Done()
384		s.finalizePreviousContainer(context.WithoutCancel(ctx), route.Domain, existing, hasExisting, invalidated, newContainer.ID)
385	}()
386
387	// Start container log collection (non-blocking, errors don't fail deployment)
388	s.startLogCollection(ctx, newContainer.ID, route.Domain)
389
390	log.Info().
391		Str("image", route.Image).
392		Str(zerowrap.FieldEntityID, newContainer.ID).
393		Ints("ports", newContainer.Ports).
394		Str("network", resources.networkName).
395		Bool("zero_downtime", hasExisting).
396		Msg("container deployed successfully")
397
398	return newContainer, nil
399}
400
401// recordDeployMetrics records span error status and deploy metrics at the end of a Deploy call.
402// It is called via defer and receives a pointer to the named return error so it can observe
403// the final error value after all deferred functions have run.
404func (s *Service) recordDeployMetrics(ctx context.Context, span trace.Span, route domain.Route, start time.Time, errPtr *error) {
405	err := *errPtr
406	if err != nil {
407		span.RecordError(err)
408		span.SetStatus(codes.Error, err.Error())
409	}
410	if s.metrics == nil {
411		return
412	}
413	attrs := metric.WithAttributes(
414		attribute.String("domain", route.Domain),
415		attribute.String("image", route.Image),
416	)
417	s.metrics.DeployTotal.Add(ctx, 1, attrs)
418	s.metrics.DeployDuration.Record(ctx, time.Since(start).Seconds(), attrs)
419	if err != nil {
420		s.metrics.DeployErrors.Add(ctx, 1, attrs)
421	}
422}
423
424// skipRedundantDeploy checks whether the existing container is already running
425// the same image (by Docker image ID) as the one we are about to deploy.
426// When a push triggers both an event-based deploy and an explicit CLI deploy,
427// the second one arrives after the first has already completed; this avoids
428// a full redundant create-start-readiness cycle.
429func (s *Service) skipRedundantDeploy(ctx context.Context, existing *domain.Container, actualImageRef, envHash string) (bool, *domain.Container) {
430	log := zerowrap.FromCtx(ctx)
431
432	newImageID, err := s.runtime.GetImageID(ctx, actualImageRef)
433	if err != nil {
434		log.Debug().Err(err).Msg("cannot resolve image ID for redundancy check, proceeding with deploy")
435		return false, nil
436	}
437
438	if existing.ImageID != newImageID {
439		return false, nil
440	}
441
442	// Verify the container is actually still running before skipping.
443	running, err := s.runtime.IsContainerRunning(ctx, existing.ID)
444	if err != nil || !running {
445		return false, nil
446	}
447
448	if existing.Labels != nil {
449		existingEnvHash, hasEnvHash := existing.Labels[domain.LabelEnvHash]
450		if !hasEnvHash || existingEnvHash != envHash {
451			log.Info().
452				Str("container_id", existing.ID).
453				Bool("has_env_hash", hasEnvHash).
454				Msg("env changed, proceeding with deploy despite same image")
455			return false, nil
456		}
457	} else {
458		return false, nil
459	}
460
461	log.Info().
462		Str("container_id", existing.ID).
463		Str("image_id", newImageID).
464		Msg("skipping redundant deploy: container already running this image")
465
466	return true, existing
467}
468
469func (s *Service) containerForRedundantCheck(ctx context.Context, existing *domain.Container) *domain.Container {
470	if existing == nil || existing.ImageID != "" || existing.ID == "" {
471		return existing
472	}
473
474	log := zerowrap.FromCtx(ctx)
475	inspected, err := s.runtime.InspectContainer(ctx, existing.ID)
476	if err != nil {
477		log.Debug().Err(err).Str("container_id", existing.ID).Msg("cannot inspect existing container for redundancy check")
478		return existing
479	}
480	if inspected == nil || inspected.ImageID == "" {
481		return existing
482	}
483
484	existingCopy := *existing
485	existingCopy.ImageID = inspected.ImageID
486	return &existingCopy
487}
488
489type deployResources struct {
490	networkName    string
491	actualImageRef string
492	exposedPorts   []int
493	imageLabels    map[string]string
494	envVars        []string
495	envHash        string
496	volumes        map[string]string
497}
498
499// resolveExistingContainer returns the currently running container for a domain.
500// It first checks the in-memory map (fast path). If not found, it queries the
501// runtime for a running container with matching name and managed label. This
502// handles cases where Gordon restarted and in-memory state is stale.
503//
504// The slow path checks canonical, -new, and -next names because a previous
505// deploy may have been interrupted (e.g. eventbus timeout, Gordon restart)
506// leaving the active container under a temp name. Canonical is preferred when
507// multiple matches exist.
508func (s *Service) resolveExistingContainer(ctx context.Context, domainName string) (*domain.Container, bool) {
509	// Fast path: check in-memory state
510	s.mu.RLock()
511	container, ok := s.containers[domainName]
512	s.mu.RUnlock()
513	if ok {
514		return container, true
515	}
516
517	// Slow path: query runtime for running containers
518	log := zerowrap.FromCtx(ctx)
519	canonicalName := fmt.Sprintf("gordon-%s", domainName)
520	candidateNames := map[string]bool{
521		canonicalName:           true,
522		canonicalName + "-new":  true,
523		canonicalName + "-next": true,
524	}
525
526	running, err := s.runtime.ListContainers(ctx, false)
527	if err != nil {
528		log.Warn().Err(err).Msg("failed to list running containers for existing container resolution")
529		return nil, false
530	}
531
532	// Prefer canonical name; fall back to any running managed temp container.
533	var best *domain.Container
534	for _, c := range running {
535		if !candidateNames[c.Name] || c.Labels[domain.LabelManaged] != "true" {
536			continue
537		}
538		if best == nil || c.Name == canonicalName {
539			best = c
540		}
541	}
542
543	if best != nil {
544		log.Info().
545			Str("container_id", best.ID).
546			Str("container_name", best.Name).
547			Msg("resolved existing container from runtime (in-memory state was stale)")
548
549		// Update in-memory state so subsequent lookups are fast
550		s.mu.Lock()
551		if _, alreadyTracked := s.containers[domainName]; !alreadyTracked {
552			s.managedCount++
553		}
554		s.containers[domainName] = best
555		s.mu.Unlock()
556
557		return best, true
558	}
559
560	return nil, false
561}
562
563func (s *Service) prepareDeployResources(ctx context.Context, route domain.Route, existing *domain.Container) (*deployResources, error) {
564	log := zerowrap.FromCtx(ctx)
565
566	existingID := ""
567	if existing != nil {
568		existingID = existing.ID
569	}
570	if err := s.cleanupOrphanedContainers(ctx, route.Domain, existingID); err != nil {
571		log.WrapErr(err, "failed to cleanup orphaned containers")
572	}
573
574	imageRef, err := s.buildValidatedImageRef(ctx, route.Image)
575	if err != nil {
576		return nil, err
577	}
578	actualImageRef, err := s.ensureImage(ctx, imageRef)
579	if err != nil {
580		return nil, err
581	}
582
583	networkName := s.getNetworkForApp(route.Domain)
584	if err := s.createNetworkIfNeeded(ctx, networkName); err != nil {
585		return nil, log.WrapErr(err, "failed to create network")
586	}
587	if err := s.deployAttachments(ctx, route.Domain, networkName); err != nil {
588		return nil, log.WrapErr(err, "failed to deploy attachments")
589	}
590
591	exposedPorts, err := s.runtime.GetImageExposedPorts(ctx, actualImageRef)
592	if err != nil {
593		log.WrapErr(err, "failed to get exposed ports, using defaults")
594		exposedPorts = []int{80, 8080, 3000}
595	}
596
597	imageLabels, err := s.runtime.GetImageLabels(ctx, actualImageRef)
598	if err != nil {
599		log.Warn().Err(err).Msg("failed to get image labels, skipping label propagation")
600		imageLabels = nil
601	}
602
603	envVars, err := s.loadEnvironment(ctx, route.Env, route.Domain, actualImageRef)
604	if err != nil {
605		return nil, err
606	}
607	envHash := hashEnvironment(envVars)
608
609	volumes, err := s.setupVolumes(ctx, route.Domain, actualImageRef)
610	if err != nil {
611		return nil, err
612	}
613
614	return &deployResources{
615		networkName:    networkName,
616		actualImageRef: actualImageRef,
617		exposedPorts:   exposedPorts,
618		imageLabels:    imageLabels,
619		envVars:        envVars,
620		envHash:        envHash,
621		volumes:        volumes,
622	}, nil
623}
624
625func (s *Service) createStartedContainer(ctx context.Context, route domain.Route, existing *domain.Container, resources *deployResources) (*domain.Container, error) {
626	ctx, span := tracer.Start(ctx, "container.create_and_start")
627	defer span.End()
628
629	log := zerowrap.FromCtx(ctx)
630
631	containerConfig := s.buildContainerConfig(containerConfigInput{
632		Domain:       route.Domain,
633		Image:        route.Image,
634		ImageRef:     resources.actualImageRef,
635		ExposedPorts: resources.exposedPorts,
636		EnvVars:      resources.envVars,
637		EnvHash:      resources.envHash,
638		Volumes:      resources.volumes,
639		NetworkName:  resources.networkName,
640		ImageLabels:  resources.imageLabels,
641		Existing:     existing,
642	})
643
644	newContainer, err := s.runtime.CreateContainer(ctx, containerConfig)
645	if err != nil {
646		return nil, log.WrapErr(err, "failed to create container")
647	}
648	if err := s.runtime.StartContainer(ctx, newContainer.ID); err != nil {
649		s.runtime.RemoveContainer(ctx, newContainer.ID, true)
650		return nil, log.WrapErr(err, "failed to start container")
651	}
652	if !domain.IsSkipReadiness(ctx) {
653		if err := s.waitForReady(ctx, newContainer.ID, containerConfig); err != nil {
654			deployErr := s.newDeployFailure(newContainer, "container failed readiness check", err, s.captureRecentContainerLogs(ctx, newContainer.ID))
655			s.cleanupFailedContainer(ctx, newContainer.ID)
656			return nil, deployErr
657		}
658	}
659
660	inspected, err := s.runtime.InspectContainer(ctx, newContainer.ID)
661	if err != nil {
662		s.cleanupFailedContainer(ctx, newContainer.ID)
663		return nil, log.WrapErr(err, "failed to inspect started container")
664	}
665
666	return inspected, nil
667}
668
669func (s *Service) newDeployFailure(container *domain.Container, cause string, err error, logs []string) error {
670	deployErr := &domain.DeployFailureError{
671		Summary: "failed to deploy",
672		Cause:   cause,
673		Hint:    inferDeployFailureHint(err),
674		Logs:    logs,
675		Err:     err,
676	}
677	if container != nil {
678		deployErr.ContainerName = container.Name
679		deployErr.ContainerID = container.ID
680	}
681
682	return deployErr
683}
684
685func (s *Service) wrapPullDeployFailure(route domain.Route, existing *domain.Container, err error) error {
686	if !isPullFailure(err) {
687		return nil
688	}
689
690	return s.newDeployFailure(&domain.Container{Name: s.deploymentContainerName(route.Domain, existing)}, "failed to pull image", err, nil)
691}
692
693func (s *Service) captureRecentContainerLogs(ctx context.Context, containerID string) []string {
694	if ctx.Err() != nil {
695		return nil
696	}
697
698	logStream, err := s.runtime.GetContainerLogs(ctx, containerID, false)
699	if err != nil {
700		return nil
701	}
702	defer func() {
703		_ = logStream.Close()
704	}()
705
706	logs, err := parseDockerLogLines(logStream, deployFailureLogLimit)
707	if err != nil {
708		return nil
709	}
710
711	return logs
712}
713
714func parseDockerLogLines(r io.Reader, maxLines int) ([]string, error) {
715	reader := bufio.NewReader(r)
716	collector := newLogLineCollector(maxLines)
717
718	header, err := reader.Peek(8)
719	if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, bufio.ErrBufferFull) {
720		return nil, err
721	}
722
723	if len(header) >= 8 && looksLikeDockerLogStream(header) {
724		return parseDockerFramedLogLines(reader, collector)
725	}
726
727	return parsePlainLogLines(reader, collector)
728}
729
730func parseDockerFramedLogLines(reader *bufio.Reader, collector *logLineCollector) ([]string, error) {
731	for {
732		frameHeader := make([]byte, 8)
733		if _, err := io.ReadFull(reader, frameHeader); err != nil {
734			if isLogStreamEOF(err) {
735				collector.flush()
736				return collector.lines, nil
737			}
738			return nil, err
739		}
740
741		size := int(binary.BigEndian.Uint32(frameHeader[4:8]))
742		if size <= 0 {
743			continue
744		}
745
746		payload, err := readLogPayload(reader, size)
747		if err != nil {
748			collector.addChunk(string(payload))
749			collector.flush()
750			if isLogStreamEOF(err) {
751				return collector.lines, nil
752			}
753			return nil, err
754		}
755
756		collector.addChunk(string(payload))
757	}
758}
759
760func parsePlainLogLines(reader *bufio.Reader, collector *logLineCollector) ([]string, error) {
761
762	scanner := bufio.NewScanner(reader)
763	for scanner.Scan() {
764		collector.addLine(scanner.Text())
765	}
766	if err := scanner.Err(); err != nil {
767		return nil, err
768	}
769
770	collector.flush()
771	return collector.lines, nil
772}
773
774func readLogPayload(reader *bufio.Reader, size int) ([]byte, error) {
775	payload := make([]byte, size)
776	_, err := io.ReadFull(reader, payload)
777	return payload, err
778}
779
780func isLogStreamEOF(err error) bool {
781	return errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF)
782}
783
784func looksLikeDockerLogStream(data []byte) bool {
785	return len(data) >= 8 && data[1] == 0 && data[2] == 0 && data[3] == 0
786}
787
788type logLineCollector struct {
789	lines    []string
790	pending  string
791	maxLines int
792}
793
794func newLogLineCollector(maxLines int) *logLineCollector {
795	return &logLineCollector{maxLines: maxLines}
796}
797
798func (c *logLineCollector) addChunk(chunk string) {
799	for _, part := range strings.SplitAfter(chunk, "\n") {
800		c.pending += part
801		if strings.HasSuffix(part, "\n") {
802			c.flush()
803		}
804	}
805}
806
807func (c *logLineCollector) addLine(line string) {
808	line = strings.TrimRight(line, "\r")
809	if line == "" {
810		return
811	}
812	c.lines = append(c.lines, line)
813	if c.maxLines > 0 && len(c.lines) > c.maxLines {
814		c.lines = c.lines[len(c.lines)-c.maxLines:]
815	}
816}
817
818func (c *logLineCollector) flush() {
819	if c.pending == "" {
820		return
821	}
822	c.addLine(strings.TrimRight(c.pending, "\r\n"))
823	c.pending = ""
824}
825
826func inferDeployFailureHint(err error) string {
827	if err == nil {
828		return ""
829	}
830
831	msg := strings.ToLower(err.Error())
832	switch {
833	case strings.Contains(msg, "healthcheck timeout"):
834		return "check startup logs and confirm the health endpoint becomes ready within the configured timeout"
835	case strings.Contains(msg, "unhealthy"):
836		return "inspect recent app logs and verify the healthcheck command reflects actual readiness"
837	case strings.Contains(msg, "no healthcheck detected"):
838		return "add a Docker healthcheck or switch readiness mode"
839	case isPullFailure(err), strings.Contains(msg, "access denied"), strings.Contains(msg, "unauthorized"):
840		return "verify registry auth and confirm the image exists at the requested tag"
841	default:
842		return ""
843	}
844}
845
846func isPullFailure(err error) bool {
847	if err == nil {
848		return false
849	}
850
851	if errors.Is(err, domain.ErrImagePullFailed) {
852		return true
853	}
854
855	msg := strings.ToLower(err.Error())
856	return strings.Contains(msg, "failed to pull image") ||
857		strings.Contains(msg, "pull access denied")
858}
859
860func (s *Service) activateDeployedContainer(ctx context.Context, domainName string, container *domain.Container) bool {
861	s.mu.Lock()
862	_, wasTracked := s.containers[domainName]
863	s.containers[domainName] = container
864	if !wasTracked {
865		s.managedCount++
866	}
867	s.mu.Unlock()
868
869	// Track managed container count (only increment for new domains, not replacements)
870	if !wasTracked && s.metrics != nil {
871		s.metrics.ManagedContainers.Add(ctx, 1)
872	}
873
874	s.publishContainerDeployed(ctx, domainName, container.ID)
875
876	s.mu.RLock()
877	inv := s.cacheInvalidator
878	s.mu.RUnlock()
879	if inv != nil {
880		inv.InvalidateTarget(ctx, domainName)
881		return true
882	}
883
884	return false
885}
886
887// stabilizeNewContainer monitors the new container briefly after traffic switch.
888// If it crashes during this window, rolls back to old container.
889// Returns true if stabilization succeeded, false if rollback was performed.
890// Returns an error only when both new and old containers are dead.
891func (s *Service) stabilizeNewContainer(ctx context.Context, domainName string, newContainer, oldContainer *domain.Container) (bool, error) {
892	log := zerowrap.FromCtx(ctx)
893
894	if oldContainer == nil {
895		return true, nil
896	}
897
898	s.mu.RLock()
899	delay := s.config.StabilizationDelay
900	s.mu.RUnlock()
901	if delay == 0 {
902		delay = 2 * time.Second
903	}
904
905	// Brief stabilization: verify new container is still running after delay
906	select {
907	case <-time.After(delay):
908	case <-ctx.Done():
909		return true, nil
910	}
911
912	running, err := s.runtime.IsContainerRunning(ctx, newContainer.ID)
913	if err != nil || !running {
914		log.Error().
915			Str("new_container", newContainer.ID).
916			Str("old_container", oldContainer.ID).
917			Msg("new container crashed during stabilization, rolling back to old")
918
919		// Verify old container is still running before restoring it.
920		// If both old and new are dead (e.g., OOM), returning a dead
921		// container as "existing" would leave the domain in a broken state.
922		oldRunning, oldErr := s.runtime.IsContainerRunning(ctx, oldContainer.ID)
923		if oldErr != nil || !oldRunning {
924			log.Error().
925				Str("old_container", oldContainer.ID).
926				Msg("old container is also not running, cannot rollback")
927
928			// Cleanup failed new container
929			if stopErr := s.runtime.StopContainer(ctx, newContainer.ID); stopErr != nil {
930				log.WrapErrWithFields(stopErr, "failed to stop failed new container during rollback", map[string]any{zerowrap.FieldEntityID: newContainer.ID})
931			}
932			if removeErr := s.runtime.RemoveContainer(ctx, newContainer.ID, true); removeErr != nil {
933				log.WrapErrWithFields(removeErr, "failed to remove failed new container during rollback", map[string]any{zerowrap.FieldEntityID: newContainer.ID})
934			}
935
936			return false, fmt.Errorf("stabilization failed: new container crashed and old container %s is also not running", oldContainer.ID)
937		}
938
939		// Rollback: restore old container as tracked
940		s.mu.Lock()
941		s.containers[domainName] = oldContainer
942		s.mu.Unlock()
943
944		// Re-invalidate proxy cache to point back to old
945		s.mu.RLock()
946		inv := s.cacheInvalidator
947		s.mu.RUnlock()
948		if inv != nil {
949			inv.InvalidateTarget(ctx, domainName)
950		}
951
952		// Cleanup failed new container
953		if stopErr := s.runtime.StopContainer(ctx, newContainer.ID); stopErr != nil {
954			log.WrapErrWithFields(stopErr, "failed to stop failed new container during rollback", map[string]any{zerowrap.FieldEntityID: newContainer.ID})
955		}
956		if removeErr := s.runtime.RemoveContainer(ctx, newContainer.ID, true); removeErr != nil {
957			log.WrapErrWithFields(removeErr, "failed to remove failed new container during rollback", map[string]any{zerowrap.FieldEntityID: newContainer.ID})
958		}
959
960		return false, nil
961	}
962
963	return true, nil
964}
965
966func (s *Service) finalizePreviousContainer(ctx context.Context, domainName string, existing *domain.Container, hasExisting, invalidated bool, newContainerID string) {
967	if !hasExisting {
968		return
969	}
970
971	if invalidated {
972		s.waitForDrain(ctx, existing.ID)
973	}
974
975	s.cleanupOldContainer(ctx, existing, newContainerID, domainName)
976}
977
978func (s *Service) waitForDrain(ctx context.Context, oldContainerID string) {
979	log := zerowrap.FromCtx(ctx)
980
981	s.mu.RLock()
982	cfg := s.config
983	waiter := s.drainWaiter
984	s.mu.RUnlock()
985
986	drainMode := cfg.DrainMode
987	if drainMode == "" {
988		drainMode = "delay"
989	}
990
991	shouldUseInFlight := (drainMode == "inflight" || drainMode == "auto") && waiter != nil
992	if shouldUseInFlight {
993		timeout := cfg.DrainTimeout
994		if timeout == 0 {
995			timeout = 30 * time.Second
996		}
997		drained := waiter.WaitForNoInFlight(ctx, oldContainerID, timeout)
998		if !drained {
999			log.Warn().
1000				Str("old_container_id", oldContainerID).
1001				Dur("drain_timeout", timeout).
1002				Msg("drain wait timed out; old container may still have in-flight traffic")
1003		}
1004		return
1005	}
1006
1007	drainDelay := 2 * time.Second
1008	if cfg.DrainDelayConfigured {
1009		drainDelay = cfg.DrainDelay
1010	}
1011	if drainDelay <= 0 {
1012		return
1013	}
1014	select {
1015	case <-time.After(drainDelay):
1016	case <-ctx.Done():
1017	}
1018}
1019
1020// restartContainerWithRecovery restarts a container, reconciling stale state if needed.
1021// It returns the (possibly refreshed) container and updated attachment IDs.
1022func (s *Service) restartContainerWithRecovery(ctx context.Context, domainName string, container *domain.Container, attachmentIDs []string) (*domain.Container, []string, error) {
1023	log := zerowrap.FromCtx(ctx)
1024
1025	if err := s.runtime.RestartContainer(ctx, container.ID); err != nil {
1026		if !isContainerNotFoundError(err) {
1027			return nil, nil, log.WrapErr(err, "failed to restart container")
1028		}
1029
1030		// In-memory container ID can become stale after external runtime changes.
1031		// Re-sync and retry once with the latest tracked container.
1032		log.Warn().
1033			Err(err).
1034			Str(zerowrap.FieldEntityID, container.ID).
1035			Msg("tracked container missing during restart, attempting state reconciliation")
1036
1037		if syncErr := s.SyncContainers(ctx); syncErr != nil {
1038			log.Warn().Err(syncErr).Msg("failed to sync container state during restart recovery")
1039		}
1040
1041		s.mu.RLock()
1042		refreshed, refreshedExists := s.containers[domainName]
1043		attachmentIDs = append([]string{}, s.attachments[domainName]...)
1044		s.mu.RUnlock()
1045
1046		if !refreshedExists || refreshed == nil {
1047			return nil, nil, domain.ErrContainerNotFound
1048		}
1049		if err := s.runtime.RestartContainer(ctx, refreshed.ID); err != nil {
1050			return nil, nil, log.WrapErr(err, "failed to restart container after state reconciliation")
1051		}
1052		return refreshed, attachmentIDs, nil
1053	}
1054
1055	return container, attachmentIDs, nil
1056}
1057
1058// Restart restarts a running container for the given domain.
1059func (s *Service) Restart(ctx context.Context, domainName string, withAttachments bool) error {
1060	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1061		zerowrap.FieldLayer:   "usecase",
1062		zerowrap.FieldUseCase: "Restart",
1063		"domain":              domainName,
1064	})
1065	log := zerowrap.FromCtx(ctx)
1066
1067	s.mu.RLock()
1068	container, exists := s.containers[domainName]
1069	attachments := s.attachments[domainName]
1070	var attachmentIDs []string
1071	if len(attachments) > 0 {
1072		attachmentIDs = make([]string, len(attachments))
1073		copy(attachmentIDs, attachments)
1074	}
1075	s.mu.RUnlock()
1076
1077	if !exists || container == nil {
1078		return domain.ErrContainerNotFound
1079	}
1080
1081	// When attachments are requested, sync runtime state first to get accurate
1082	// attachment tracking, then check if configured attachments are deployed.
1083	// If fewer attachment containers are tracked than configured, fail early with
1084	// guidance rather than restarting the main container into a broken state
1085	// (e.g., missing database hostname).
1086	if withAttachments {
1087		if syncErr := s.SyncContainers(ctx); syncErr != nil {
1088			log.Warn().Err(syncErr).Msg("failed to sync container state before attachment check")
1089		}
1090
1091		// Re-read attachment IDs after sync
1092		s.mu.RLock()
1093		if freshAttachments := s.attachments[domainName]; len(freshAttachments) > 0 {
1094			attachmentIDs = make([]string, len(freshAttachments))
1095			copy(attachmentIDs, freshAttachments)
1096		} else {
1097			attachmentIDs = nil
1098		}
1099		s.mu.RUnlock()
1100
1101		configuredAttachments := s.resolveAttachmentsForDomain(domainName)
1102		if len(configuredAttachments) > 0 && len(attachmentIDs) < len(configuredAttachments) {
1103			missing := len(configuredAttachments) - len(attachmentIDs)
1104			return fmt.Errorf("%w: domain %q has %d configured attachment(s) but only %d deployed (%d missing)",
1105				domain.ErrAttachmentNotDeployed, domainName, len(configuredAttachments), len(attachmentIDs), missing)
1106		}
1107	}
1108
1109	// Restart the main container (with stale-state recovery)
1110	container, attachmentIDs, err := s.restartContainerWithRecovery(ctx, domainName, container, attachmentIDs)
1111	if err != nil {
1112		return err
1113	}
1114	log.Info().Str(zerowrap.FieldEntityID, container.ID).Msg("container restarted")
1115
1116	// Record restart metric
1117	if s.metrics != nil {
1118		s.metrics.ContainerRestarts.Add(ctx, 1, metric.WithAttributes(
1119			attribute.String("domain", domainName),
1120			attribute.String("source", "api"),
1121		))
1122	}
1123
1124	// Restart attachments if requested
1125	if withAttachments && len(attachmentIDs) > 0 {
1126		for _, attachID := range attachmentIDs {
1127			if err := s.runtime.RestartContainer(ctx, attachID); err != nil {
1128				log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to restart attachment")
1129				continue // Don't fail the whole operation for one attachment
1130			}
1131			log.Info().Str("attachment_id", attachID).Msg("attachment restarted")
1132		}
1133	}
1134
1135	return nil
1136}
1137
1138// Stop stops a running container.
1139func (s *Service) Stop(ctx context.Context, containerID string) error {
1140	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1141		zerowrap.FieldLayer:    "usecase",
1142		zerowrap.FieldUseCase:  "Stop",
1143		zerowrap.FieldEntityID: containerID,
1144	})
1145	log := zerowrap.FromCtx(ctx)
1146
1147	// Stop log collection before stopping container
1148	if s.logWriter != nil {
1149		if err := s.logWriter.StopLogging(containerID); err != nil {
1150			log.Warn().Err(err).Msg("failed to stop container log collection")
1151		}
1152	}
1153
1154	if err := s.runtime.StopContainer(ctx, containerID); err != nil {
1155		return log.WrapErr(err, "failed to stop container")
1156	}
1157
1158	log.Info().Msg("container stopped")
1159	return nil
1160}
1161
1162// PreviewRemovedRouteCleanup reports retained runtime state for a route that is
1163// no longer configured, without mutating containers or in-memory tracking.
1164func (s *Service) PreviewRemovedRouteCleanup(ctx context.Context, domainName string) (*domain.CleanupReport, error) {
1165	canonicalDomain, ok := domain.CanonicalRouteDomain(domainName)
1166	if !ok {
1167		return &domain.CleanupReport{Domain: domainName}, domain.ErrRouteDomainInvalid
1168	}
1169	domainName = canonicalDomain
1170
1171	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1172		zerowrap.FieldLayer:   "usecase",
1173		zerowrap.FieldUseCase: "PreviewRemovedRouteCleanup",
1174		"domain":              domainName,
1175	})
1176	log := zerowrap.FromCtx(ctx)
1177
1178	report := &domain.CleanupReport{Domain: domainName}
1179
1180	unlock, err := s.acquireDomainDeployLock(ctx, domainName)
1181	if err != nil {
1182		return report, err
1183	}
1184	defer unlock()
1185
1186	allContainers, err := s.runtime.ListContainers(ctx, true)
1187	if err != nil {
1188		return report, log.WrapErr(err, "failed to list containers for removed route cleanup preview")
1189	}
1190
1191	routeContainers := routeContainersForDomain(allContainers, domainName)
1192	report.PreservedAttachments = preservedAttachmentsForDomain(allContainers, domainName)
1193	if len(report.PreservedAttachments) > 0 {
1194		report.Hints = append(report.Hints, "attachments preserved; remove or purge attachments explicitly if they are no longer needed")
1195	}
1196	for _, c := range routeContainers {
1197		report.OrphanedEntities = append(report.OrphanedEntities, domain.CleanupOrphanedEntity{
1198			Kind:   "route_container",
1199			ID:     c.ID,
1200			Name:   c.Name,
1201			Status: c.Status,
1202			Reason: "route runtime state still exists after route removal",
1203		})
1204	}
1205
1206	return report, nil
1207}
1208
1209// ReconcileRemovedRoute removes active runtime containers for a route that was
1210// removed from configuration while preserving stateful resources for explicit
1211// follow-up cleanup. It intentionally removes only main route containers, never
1212// attachment containers or volumes.
1213func (s *Service) ReconcileRemovedRoute(ctx context.Context, domainName string) (*domain.CleanupReport, error) {
1214	canonicalDomain, ok := domain.CanonicalRouteDomain(domainName)
1215	if !ok {
1216		return &domain.CleanupReport{Domain: domainName}, domain.ErrRouteDomainInvalid
1217	}
1218	domainName = canonicalDomain
1219
1220	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1221		zerowrap.FieldLayer:   "usecase",
1222		zerowrap.FieldUseCase: "ReconcileRemovedRoute",
1223		"domain":              domainName,
1224	})
1225	log := zerowrap.FromCtx(ctx)
1226
1227	report := &domain.CleanupReport{Domain: domainName}
1228
1229	unlock, err := s.acquireDomainDeployLock(ctx, domainName)
1230	if err != nil {
1231		return report, err
1232	}
1233	defer unlock()
1234
1235	allContainers, err := s.runtime.ListContainers(ctx, true)
1236	if err != nil {
1237		return report, log.WrapErr(err, "failed to list containers for removed route cleanup")
1238	}
1239
1240	routeContainers := routeContainersForDomain(allContainers, domainName)
1241
1242	// Clear in-memory state before stopping containers so the monitor cannot
1243	// interpret the deliberate removal as a crash that should be restarted.
1244	s.clearRouteContainerTracking(ctx, domainName)
1245	if inv := s.proxyCacheInvalidator(); inv != nil {
1246		inv.InvalidateTarget(ctx, domainName)
1247	}
1248
1249	report.PreservedAttachments = preservedAttachmentsForDomain(allContainers, domainName)
1250	if len(report.PreservedAttachments) > 0 {
1251		report.Hints = append(report.Hints, "attachments preserved; remove or purge attachments explicitly if they are no longer needed")
1252	}
1253
1254	if len(routeContainers) == 0 {
1255		report.Warnings = append(report.Warnings, "no active route container found for removed route")
1256		return report, nil
1257	}
1258
1259	s.removeRouteContainersForCleanup(ctx, log, routeContainers, report)
1260
1261	if len(report.PartialFailures) > 0 {
1262		report.Warnings = append(report.Warnings, "route cleanup completed with partial failures; manual follow-up may be required")
1263	}
1264
1265	return report, nil
1266}
1267
1268func routeContainersForDomain(containers []*domain.Container, domainName string) []*domain.Container {
1269	var routeContainers []*domain.Container
1270	for _, c := range containers {
1271		if isManagedRouteContainerForDomain(c, domainName) {
1272			routeContainers = append(routeContainers, c)
1273		}
1274	}
1275	return routeContainers
1276}
1277
1278func preservedAttachmentsForDomain(containers []*domain.Container, domainName string) []domain.CleanupAttachment {
1279	var attachments []domain.CleanupAttachment
1280	for _, c := range containers {
1281		if isPreservedAttachmentForDomain(c, domainName) {
1282			attachments = append(attachments, domain.CleanupAttachment{
1283				Name:        c.Name,
1284				Image:       c.Image,
1285				ContainerID: c.ID,
1286				Status:      c.Status,
1287				Owner:       c.Labels[domain.LabelAttachedTo],
1288				Reason:      "attachments are preserved by default; remove or purge them explicitly",
1289			})
1290		}
1291	}
1292	return attachments
1293}
1294
1295func (s *Service) removeRouteContainersForCleanup(ctx context.Context, log zerowrap.Logger, containers []*domain.Container, report *domain.CleanupReport) {
1296	for _, c := range containers {
1297		s.stopRouteLogCollectionForCleanup(log, c, report)
1298		s.stopRouteContainerForCleanup(ctx, log, c, report)
1299		if err := s.runtime.RemoveContainer(ctx, c.ID, true); err != nil {
1300			report.PartialFailures = append(report.PartialFailures, domain.CleanupFailure{Action: "remove", Kind: "container", ID: c.ID, Name: c.Name, Error: err.Error()})
1301			log.Warn().Err(err).Str(zerowrap.FieldEntityID, c.ID).Msg("failed to remove removed route container")
1302			continue
1303		}
1304		report.RemovedContainers = append(report.RemovedContainers, domain.CleanupContainer{ID: c.ID, Name: c.Name, Image: c.Image, Status: c.Status})
1305	}
1306}
1307
1308func (s *Service) stopRouteLogCollectionForCleanup(log zerowrap.Logger, c *domain.Container, report *domain.CleanupReport) {
1309	if s.logWriter == nil {
1310		return
1311	}
1312	if err := s.logWriter.StopLogging(c.ID); err != nil {
1313		report.PartialFailures = append(report.PartialFailures, domain.CleanupFailure{Action: "stop_logging", Kind: "container", ID: c.ID, Name: c.Name, Error: err.Error()})
1314		log.Warn().Err(err).Str(zerowrap.FieldEntityID, c.ID).Msg("failed to stop removed route log collection")
1315	}
1316}
1317
1318func (s *Service) stopRouteContainerForCleanup(ctx context.Context, log zerowrap.Logger, c *domain.Container, report *domain.CleanupReport) {
1319	if c.Status != string(domain.ContainerStatusRunning) {
1320		return
1321	}
1322	if err := s.runtime.StopContainer(ctx, c.ID); err != nil {
1323		report.PartialFailures = append(report.PartialFailures, domain.CleanupFailure{Action: "stop", Kind: "container", ID: c.ID, Name: c.Name, Error: err.Error()})
1324		log.Warn().Err(err).Str(zerowrap.FieldEntityID, c.ID).Msg("failed to stop removed route container")
1325	}
1326}
1327
1328func isManagedRouteContainerForDomain(c *domain.Container, domainName string) bool {
1329	if c == nil || c.Labels == nil || c.Labels[domain.LabelManaged] != "true" {
1330		return false
1331	}
1332	if c.Labels[domain.LabelAttachment] == "true" {
1333		return false
1334	}
1335	return c.Labels[domain.LabelRoute] == domainName || c.Labels[domain.LabelDomain] == domainName || c.Name == fmt.Sprintf("gordon-%s", domainName)
1336}
1337
1338func isPreservedAttachmentForDomain(c *domain.Container, domainName string) bool {
1339	if c == nil || c.Labels == nil || c.Labels[domain.LabelManaged] != "true" {
1340		return false
1341	}
1342	return c.Labels[domain.LabelAttachment] == "true" && c.Labels[domain.LabelAttachedTo] == domainName
1343}
1344
1345func (s *Service) proxyCacheInvalidator() out.ProxyCacheInvalidator {
1346	s.mu.RLock()
1347	defer s.mu.RUnlock()
1348	return s.cacheInvalidator
1349}
1350
1351func (s *Service) clearRouteContainerTracking(ctx context.Context, domainName string) {
1352	s.mu.Lock()
1353	removed := false
1354	if _, exists := s.containers[domainName]; exists {
1355		delete(s.containers, domainName)
1356		if s.managedCount > 0 {
1357			s.managedCount--
1358		}
1359		removed = true
1360	}
1361	s.mu.Unlock()
1362
1363	if removed && s.metrics != nil {
1364		s.metrics.ManagedContainers.Add(ctx, -1)
1365	}
1366}
1367
1368// Remove removes a container.
1369func (s *Service) Remove(ctx context.Context, containerID string, force bool) error {
1370	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1371		zerowrap.FieldLayer:    "usecase",
1372		zerowrap.FieldUseCase:  "Remove",
1373		zerowrap.FieldEntityID: containerID,
1374	})
1375	log := zerowrap.FromCtx(ctx)
1376
1377	// Stop log collection before removing container
1378	if s.logWriter != nil {
1379		if err := s.logWriter.StopLogging(containerID); err != nil {
1380			log.Warn().Err(err).Msg("failed to stop container log collection")
1381		}
1382	}
1383
1384	// Find domain and attachment IDs for cleanup
1385	containerDomain, attachmentIDs, cfg := s.findContainerContext(containerID)
1386
1387	if err := s.runtime.RemoveContainer(ctx, containerID, force); err != nil {
1388		return log.WrapErr(err, "failed to remove container")
1389	}
1390
1391	// Cleanup volumes
1392	if containerDomain != "" && !cfg.VolumePreserve {
1393		if err := s.cleanupVolumesForDomain(ctx, containerDomain); err != nil {
1394			log.WrapErrWithFields(err, "failed to cleanup volumes", map[string]any{"domain": containerDomain})
1395		}
1396	}
1397
1398	// Clean up attachment containers
1399	s.removeAttachments(ctx, attachmentIDs)
1400
1401	// Remove from tracking
1402	s.mu.Lock()
1403	var removedDomain string
1404	for d, c := range s.containers {
1405		if c.ID == containerID {
1406			delete(s.containers, d)
1407			removedDomain = d
1408			s.managedCount--
1409			log.Info().Str("domain", d).Msg("container removed")
1410			break
1411		}
1412	}
1413	if removedDomain != "" {
1414		delete(s.attachments, removedDomain)
1415	}
1416	s.mu.Unlock()
1417
1418	// Decrement managed container count
1419	if removedDomain != "" && s.metrics != nil {
1420		s.metrics.ManagedContainers.Add(ctx, -1)
1421	}
1422
1423	// Note: We intentionally do NOT clean up deployMu entries to avoid a race condition:
1424	// If Remove deletes the mutex entry while a concurrent Deploy is acquiring the lock,
1425	// the Deploy would create a fresh mutex, breaking serialization and allowing concurrent
1426	// deploys to the same domain. The memory footprint is acceptable (one small struct per
1427	// domain ever deployed), and this is the safer choice for correctness.
1428
1429	// Cleanup network
1430	if removedDomain != "" && cfg.NetworkIsolation {
1431		networkName := s.getNetworkForApp(removedDomain)
1432		if err := s.cleanupNetworkIfEmpty(ctx, networkName); err != nil {
1433			log.WrapErrWithFields(err, "failed to cleanup network", map[string]any{"network": networkName})
1434		}
1435	}
1436
1437	return nil
1438}
1439
1440// findContainerContext looks up the domain, attachment IDs, and config snapshot for a container.
1441func (s *Service) findContainerContext(containerID string) (string, []string, Config) {
1442	s.mu.RLock()
1443	defer s.mu.RUnlock()
1444	cfg := s.config
1445	var containerDomain string
1446	for d, c := range s.containers {
1447		if c.ID == containerID {
1448			containerDomain = d
1449			break
1450		}
1451	}
1452	var attachmentIDs []string
1453	if containerDomain != "" {
1454		attachmentIDs = append(attachmentIDs, s.attachments[containerDomain]...)
1455	}
1456	return containerDomain, attachmentIDs, cfg
1457}
1458
1459// removeAttachments stops and removes attachment containers best-effort.
1460func (s *Service) removeAttachments(ctx context.Context, attachmentIDs []string) {
1461	log := zerowrap.FromCtx(ctx)
1462	for _, attachID := range attachmentIDs {
1463		if s.logWriter != nil {
1464			if err := s.logWriter.StopLogging(attachID); err != nil {
1465				log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to stop attachment log collection")
1466			}
1467		}
1468		if err := s.runtime.StopContainer(ctx, attachID); err != nil {
1469			log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to stop attachment container")
1470		}
1471		if err := s.runtime.RemoveContainer(ctx, attachID, true); err != nil {
1472			log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to remove attachment container")
1473		}
1474	}
1475}
1476
1477// Get retrieves a container by domain.
1478func (s *Service) Get(_ context.Context, domainName string) (*domain.Container, bool) {
1479	s.mu.RLock()
1480	defer s.mu.RUnlock()
1481	container, exists := s.containers[domainName]
1482	return container, exists
1483}
1484
1485// List returns all managed containers.
1486func (s *Service) List(_ context.Context) map[string]*domain.Container {
1487	s.mu.RLock()
1488	defer s.mu.RUnlock()
1489
1490	result := make(map[string]*domain.Container, len(s.containers))
1491	maps.Copy(result, s.containers)
1492	return result
1493}
1494
1495// ListRoutesWithDetails returns routes with network and attachment info.
1496// Note: Container map is copied under lock, then external calls are made without lock.
1497// If containers are removed between copy and runtime calls, errors are handled gracefully
1498// (network becomes empty string). This trade-off avoids holding locks during I/O.
1499func (s *Service) ListRoutesWithDetails(ctx context.Context) []domain.RouteInfo {
1500	s.mu.RLock()
1501	containers := make(map[string]*domain.Container, len(s.containers))
1502	maps.Copy(containers, s.containers)
1503	s.mu.RUnlock()
1504
1505	// Fetch all attachments once to avoid N+1 queries
1506	attachmentsByDomain := s.getAllAttachments(ctx)
1507
1508	results := make([]domain.RouteInfo, 0, len(containers))
1509	for domainName, container := range containers {
1510		network := ""
1511		image := ""
1512		containerID := ""
1513		status := ""
1514		if container != nil {
1515			containerID = container.ID
1516			status = container.Status
1517			image = container.Image
1518			if container.Labels != nil {
1519				if labelImage, ok := container.Labels[domain.LabelImage]; ok && labelImage != "" {
1520					image = labelImage
1521				}
1522			}
1523			// Strip registry domain prefix from image for cleaner display
1524			image = s.stripRegistryPrefix(image)
1525			if networkName, err := s.runtime.GetContainerNetwork(ctx, container.ID); err == nil {
1526				network = networkName
1527			}
1528		}
1529
1530		results = append(results, domain.RouteInfo{
1531			Domain:          domainName,
1532			Image:           image,
1533			ContainerID:     containerID,
1534			ContainerStatus: status,
1535			Network:         network,
1536			Attachments:     attachmentsByDomain[domainName],
1537		})
1538	}
1539
1540	return results
1541}
1542
1543// stripRegistryPrefix removes the configured current or legacy Gordon registry
1544// domain prefix from an image reference. For example,
1545// "reg.example.com/myapp:latest" becomes "myapp:latest" when
1546// "reg.example.com" is a current or legacy Gordon registry domain.
1547func (s *Service) stripRegistryPrefix(image string) string {
1548	s.mu.RLock()
1549	cfg := s.config
1550	s.mu.RUnlock()
1551
1552	return domain.StripKnownGordonRegistry(image, cfg.RegistryDomain, cfg.LegacyRegistryDomains)
1553}
1554
1555// ListAttachments returns attachments for a domain.
1556func (s *Service) ListAttachments(ctx context.Context, domainName string) []domain.Attachment {
1557	return s.getAttachmentsForDomain(ctx, domainName)
1558}
1559
1560// ListOrphanedAttachments returns managed attachment containers no longer configured.
1561func (s *Service) ListOrphanedAttachments(ctx context.Context) ([]domain.CleanupAttachment, error) {
1562	containers, err := s.runtime.ListContainers(ctx, true)
1563	if err != nil {
1564		return nil, fmt.Errorf("failed to list containers with runtime for orphaned attachments: %w", err)
1565	}
1566	return s.orphanedAttachmentsFromContainers(containers), nil
1567}
1568
1569// CleanupOrphanedAttachments optionally stops/removes orphaned attachment containers.
1570// When owner is non-empty, cleanup is scoped to that attachment owner.
1571func (s *Service) CleanupOrphanedAttachments(ctx context.Context, owner string, stop bool) (*domain.CleanupReport, error) {
1572	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1573		zerowrap.FieldLayer:   "usecase",
1574		zerowrap.FieldUseCase: "CleanupOrphanedAttachments",
1575		"owner":               owner,
1576	})
1577	log := zerowrap.FromCtx(ctx)
1578	report := &domain.CleanupReport{Domain: owner}
1579
1580	containers, err := s.runtime.ListContainers(ctx, true)
1581	if err != nil {
1582		return report, fmt.Errorf("failed to list containers with runtime for orphaned attachment cleanup: %w", err)
1583	}
1584	orphans := filterAttachmentContainersByOwner(s.orphanedAttachmentContainers(containers), owner)
1585	if !stop {
1586		report.PreservedAttachments = cleanupAttachmentsFromContainers(orphans)
1587		if len(report.PreservedAttachments) > 0 {
1588			report.Hints = append(report.Hints, "orphaned attachments preserved; rerun with --stop to stop/remove containers while preserving volumes")
1589		}
1590		return report, nil
1591	}
1592
1593	for _, c := range orphans {
1594		s.stopRouteLogCollectionForCleanup(log, c, report)
1595		s.stopRouteContainerForCleanup(ctx, log, c, report)
1596		if err := s.runtime.RemoveContainer(ctx, c.ID, true); err != nil {
1597			report.PartialFailures = append(report.PartialFailures, domain.CleanupFailure{Action: "remove", Kind: "attachment", ID: c.ID, Name: c.Name, Error: err.Error()})
1598			log.Warn().Err(err).Str(zerowrap.FieldEntityID, c.ID).Msg("failed to remove orphaned attachment")
1599			continue
1600		}
1601		report.RemovedContainers = append(report.RemovedContainers, domain.CleanupContainer{ID: c.ID, Name: c.Name, Image: attachmentImage(c), Status: c.Status})
1602		s.untrackAttachmentContainer(c.Labels[domain.LabelAttachedTo], c.ID)
1603	}
1604	if len(report.RemovedContainers) > 0 {
1605		report.Hints = append(report.Hints, "attachment volumes and data were preserved")
1606	}
1607	if len(report.PartialFailures) > 0 {
1608		report.Warnings = append(report.Warnings, "attachment cleanup completed with partial failures; manual follow-up may be required")
1609	}
1610	return report, nil
1611}
1612
1613func (s *Service) orphanedAttachmentsFromContainers(containers []*domain.Container) []domain.CleanupAttachment {
1614	return cleanupAttachmentsFromContainers(s.orphanedAttachmentContainers(containers))
1615}
1616
1617func (s *Service) orphanedAttachmentContainers(containers []*domain.Container) []*domain.Container {
1618	var out []*domain.Container
1619	for _, c := range containers {
1620		if !isManagedAttachment(c) {
1621			continue
1622		}
1623		owner := c.Labels[domain.LabelAttachedTo]
1624		image := attachmentImage(c)
1625		if owner == "" || image == "" || !s.isAttachmentConfiguredForOwner(owner, image) {
1626			out = append(out, c)
1627		}
1628	}
1629	return out
1630}
1631
1632func filterAttachmentContainersByOwner(containers []*domain.Container, owner string) []*domain.Container {
1633	if owner == "" {
1634		return containers
1635	}
1636	var out []*domain.Container
1637	for _, c := range containers {
1638		if c != nil && c.Labels != nil && strings.EqualFold(c.Labels[domain.LabelAttachedTo], owner) {
1639			out = append(out, c)
1640		}
1641	}
1642	return out
1643}
1644
1645func cleanupAttachmentsFromContainers(containers []*domain.Container) []domain.CleanupAttachment {
1646	attachments := make([]domain.CleanupAttachment, 0, len(containers))
1647	for _, c := range containers {
1648		attachments = append(attachments, domain.CleanupAttachment{
1649			Name:        c.Name,
1650			Image:       attachmentImage(c),
1651			ContainerID: c.ID,
1652			Status:      c.Status,
1653			Owner:       c.Labels[domain.LabelAttachedTo],
1654			Reason:      "attachment container is no longer configured",
1655		})
1656	}
1657	return attachments
1658}
1659
1660func isManagedAttachment(c *domain.Container) bool {
1661	return c != nil && c.Labels != nil && c.Labels[domain.LabelManaged] == "true" && c.Labels[domain.LabelAttachment] == "true"
1662}
1663
1664func attachmentImage(c *domain.Container) string {
1665	if c == nil {
1666		return ""
1667	}
1668	if c.Labels != nil && c.Labels[domain.LabelImage] != "" {
1669		return c.Labels[domain.LabelImage]
1670	}
1671	return c.Image
1672}
1673
1674func (s *Service) isAttachmentConfiguredForOwner(owner, image string) bool {
1675	return slices.Contains(s.resolveAttachmentsForDomain(owner), image)
1676}
1677
1678func (s *Service) untrackAttachmentContainer(owner, containerID string) {
1679	s.mu.Lock()
1680	defer s.mu.Unlock()
1681	ids := s.attachments[owner]
1682	for i, id := range ids {
1683		if id == containerID {
1684			s.attachments[owner] = slices.Delete(ids, i, i+1)
1685			break
1686		}
1687	}
1688	if len(s.attachments[owner]) == 0 {
1689		delete(s.attachments, owner)
1690	}
1691}
1692
1693// ListNetworks returns Gordon-managed networks.
1694func (s *Service) ListNetworks(ctx context.Context) ([]*domain.NetworkInfo, error) {
1695	s.mu.RLock()
1696	cfg := s.config
1697	s.mu.RUnlock()
1698
1699	networks, err := s.runtime.ListNetworks(ctx)
1700	if err != nil {
1701		return nil, err
1702	}
1703
1704	var filtered []*domain.NetworkInfo
1705	for _, network := range networks {
1706		if strings.HasPrefix(network.Name, cfg.NetworkPrefix+"-") && network.Labels[domain.LabelManaged] == "true" {
1707			filtered = append(filtered, network)
1708		}
1709	}
1710
1711	return filtered, nil
1712}
1713
1714// HealthCheck performs health checks on all containers.
1715func (s *Service) HealthCheck(ctx context.Context) map[string]bool {
1716	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1717		zerowrap.FieldLayer:   "usecase",
1718		zerowrap.FieldUseCase: "HealthCheck",
1719	})
1720	log := zerowrap.FromCtx(ctx)
1721
1722	// Copy container IDs under lock, then release before making Docker API calls
1723	s.mu.RLock()
1724	snapshot := make(map[string]string, len(s.containers))
1725	for d, c := range s.containers {
1726		snapshot[d] = c.ID
1727	}
1728	s.mu.RUnlock()
1729
1730	health := make(map[string]bool, len(snapshot))
1731	for d, id := range snapshot {
1732		running, err := s.runtime.IsContainerRunning(ctx, id)
1733		if err != nil {
1734			log.WrapErrWithFields(err, "health check failed", map[string]any{"domain": d, zerowrap.FieldEntityID: id})
1735			health[d] = false
1736		} else {
1737			health[d] = running
1738		}
1739	}
1740	return health
1741}
1742
1743// SyncContainers synchronizes containers with runtime state.
1744func isTrackedManagedContainerStatus(status string) bool {
1745	return status == "" || status == string(domain.ContainerStatusRunning) || status == "restarting"
1746}
1747
1748func (s *Service) SyncContainers(ctx context.Context) error {
1749	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1750		zerowrap.FieldLayer:   "usecase",
1751		zerowrap.FieldUseCase: "SyncContainers",
1752	})
1753	log := zerowrap.FromCtx(ctx)
1754
1755	// List containers without holding lock to avoid blocking during the runtime call.
1756	// Use all=true so we can rebuild the running-container maps from a single
1757	// snapshot without a second list call.
1758	allContainers, err := s.runtime.ListContainers(ctx, true)
1759	if err != nil {
1760		return log.WrapErr(err, "failed to list containers")
1761	}
1762
1763	managed := make(map[string]*domain.Container)
1764	attachments := make(map[string][]string)
1765	for _, c := range allContainers {
1766		if !isTrackedManagedContainerStatus(c.Status) {
1767			continue
1768		}
1769		if c.Labels == nil || c.Labels[domain.LabelManaged] != "true" {
1770			continue
1771		}
1772		if c.Labels[domain.LabelAttachment] == "true" {
1773			owner := c.Labels[domain.LabelAttachedTo]
1774			if owner != "" {
1775				attachments[owner] = append(attachments[owner], c.ID)
1776			}
1777			continue
1778		}
1779		if d, ok := c.Labels[domain.LabelDomain]; ok {
1780			managed[d] = c
1781		}
1782	}
1783
1784	s.mu.Lock()
1785	s.containers = managed
1786	s.attachments = attachments
1787	newCount := int64(len(managed))
1788	delta := newCount - s.managedCount
1789	s.managedCount = newCount
1790	s.mu.Unlock()
1791
1792	// Report initial/delta managed container count to OTel UpDownCounter.
1793	if delta != 0 && s.metrics != nil {
1794		s.metrics.ManagedContainers.Add(ctx, delta)
1795	}
1796
1797	log.Info().Int(zerowrap.FieldCount, len(managed)).Msg("container state synchronized")
1798	return nil
1799}
1800
1801// AutoStart starts containers for the provided routes that aren't running.
1802// It skips readiness checks to avoid blocking boot; the background monitor
1803// handles crash recovery.
1804func (s *Service) AutoStart(ctx context.Context, routes []domain.Route) error {
1805	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1806		zerowrap.FieldLayer:   "usecase",
1807		zerowrap.FieldUseCase: "AutoStart",
1808	})
1809	log := zerowrap.FromCtx(ctx)
1810
1811	log.Info().Int("route_count", len(routes)).Msg("auto-starting containers for configured routes")
1812
1813	// Filter out routes that already have a running container.
1814	var pending []domain.Route
1815	skipped := 0
1816	for _, route := range routes {
1817		if _, exists := s.Get(ctx, route.Domain); exists {
1818			log.Debug().Str("domain", route.Domain).Msg("container already running, skipping")
1819			skipped++
1820		} else {
1821			pending = append(pending, route)
1822		}
1823	}
1824
1825	if len(pending) == 0 {
1826		log.Info().Int("skipped", skipped).Msg("auto-start completed, all containers already running")
1827		return nil
1828	}
1829
1830	// Deploy all pending routes concurrently with readiness checks skipped.
1831	deployCtx := domain.WithSkipReadiness(ctx)
1832
1833	type result struct {
1834		route domain.Route
1835		err   error
1836	}
1837	results := make(chan result, len(pending))
1838	sem := make(chan struct{}, 4) // limit concurrency
1839
1840	for _, route := range pending {
1841		sem <- struct{}{}
1842		go func(r domain.Route) {
1843			defer func() { <-sem }()
1844
1845			log.Info().Str("domain", r.Domain).Str("image", r.Image).Msg("auto-starting container for route")
1846
1847			if _, err := s.Deploy(deployCtx, r); err != nil {
1848				results <- result{route: r, err: err}
1849				return
1850			}
1851			results <- result{route: r}
1852		}(route)
1853	}
1854
1855	var started, errCount int
1856	for i := 0; i < len(pending); i++ {
1857		res := <-results
1858		if res.err != nil {
1859			log.Warn().Err(res.err).Str("domain", res.route.Domain).Msg("failed to auto-start container")
1860			errCount++
1861		} else {
1862			started++
1863		}
1864	}
1865
1866	log.Info().
1867		Int("started", started).
1868		Int("skipped", skipped).
1869		Int("errors", errCount).
1870		Msg("auto-start completed")
1871
1872	if errCount > 0 {
1873		return fmt.Errorf("auto-start completed with %d errors", errCount)
1874	}
1875	return nil
1876}
1877
1878// Shutdown gracefully shuts down all managed containers.
1879func (s *Service) Shutdown(ctx context.Context) error {
1880	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1881		zerowrap.FieldLayer:   "usecase",
1882		zerowrap.FieldUseCase: "Shutdown",
1883	})
1884	log := zerowrap.FromCtx(ctx)
1885	log.Info().Msg("shutting down container manager...")
1886
1887	// Wait for any in-flight background cleanup goroutines to finish
1888	s.cleanupWg.Wait()
1889
1890	// Containers are left running across Gordon restarts.
1891	// SyncContainers + AutoStart will pick them back up on next boot.
1892
1893	// Close log writer to stop all log collection
1894	if s.logWriter != nil {
1895		if err := s.logWriter.Close(); err != nil {
1896			log.Warn().Err(err).Msg("failed to close container log writer")
1897		}
1898	}
1899
1900	log.Info().Msg("container manager shutdown complete")
1901	return nil
1902}
1903
1904// WaitForCleanup blocks until all background cleanup goroutines complete.
1905// Intended for use in tests to avoid mock assertion races.
1906func (s *Service) WaitForCleanup() {
1907	s.cleanupWg.Wait()
1908}
1909
1910// StartMonitor begins background monitoring of tracked containers.
1911// The monitor restarts crashed containers and detects crash loops.
1912// Safe to call multiple times; subsequent calls are no-ops.
1913func (s *Service) StartMonitor(ctx context.Context) {
1914	s.mu.Lock()
1915	if s.monitor != nil {
1916		s.mu.Unlock()
1917		return
1918	}
1919	s.monitor = newMonitor(s)
1920	m := s.monitor
1921	m.Start(ctx)
1922	s.mu.Unlock()
1923}
1924
1925// StopMonitor stops the background container monitor.
1926// Safe to call multiple times; subsequent calls are no-ops.
1927func (s *Service) StopMonitor() {
1928	s.mu.Lock()
1929	m := s.monitor
1930	s.monitor = nil
1931	s.mu.Unlock()
1932	if m != nil {
1933		m.Stop()
1934	}
1935}
1936
1937// UpdateConfig updates the service configuration.
1938func (s *Service) UpdateConfig(config Config) {
1939	s.mu.Lock()
1940	s.config = config
1941	s.mu.Unlock()
1942}
1943
1944// UpdateAttachments updates only the attachment configuration in the service.
1945// This is called after a config reload to propagate attachment changes without restart.
1946// The incoming map is deep-copied so external callers cannot mutate service state.
1947func (s *Service) UpdateAttachments(attachments map[string][]string) {
1948	var copied map[string][]string
1949	if attachments != nil {
1950		copied = make(map[string][]string, len(attachments))
1951		for k, v := range attachments {
1952			sl := make([]string, len(v))
1953			copy(sl, v)
1954			copied[k] = sl
1955		}
1956	}
1957	s.mu.Lock()
1958	s.config.Attachments = copied
1959	s.mu.Unlock()
1960}
1961
1962// Helper methods
1963
1964// cleanupFailedContainer stops and removes a container that failed to start properly.
1965func (s *Service) cleanupFailedContainer(ctx context.Context, containerID string) {
1966	log := zerowrap.FromCtx(ctx)
1967	if err := s.runtime.StopContainer(ctx, containerID); err != nil {
1968		log.Warn().Err(err).Str(zerowrap.FieldEntityID, containerID).Msg("failed to stop container after failure")
1969	}
1970	if err := s.runtime.RemoveContainer(ctx, containerID, true); err != nil {
1971		log.Warn().Err(err).Str(zerowrap.FieldEntityID, containerID).Msg("failed to remove container after failure")
1972	}
1973}
1974
1975// cleanupOldContainer stops and removes an old container after zero-downtime switch.
1976// It also renames the new container to the canonical name.
1977func (s *Service) cleanupOldContainer(ctx context.Context, old *domain.Container, newContainerID, domainName string) {
1978	log := zerowrap.FromCtx(ctx)
1979	log.Info().Str(zerowrap.FieldEntityID, old.ID).Msg("stopping old container after zero-downtime switch")
1980
1981	if s.logWriter != nil {
1982		if err := s.logWriter.StopLogging(old.ID); err != nil {
1983			log.Warn().Err(err).Str(zerowrap.FieldEntityID, old.ID).Msg("failed to stop logging for old container")
1984		}
1985	}
1986
1987	if err := s.runtime.StopContainer(ctx, old.ID); err != nil {
1988		log.Warn().Err(err).Str(zerowrap.FieldEntityID, old.ID).Msg("failed to stop old container")
1989	}
1990
1991	if err := s.runtime.RemoveContainer(ctx, old.ID, true); err != nil {
1992		log.Warn().Err(err).Str(zerowrap.FieldEntityID, old.ID).Msg("failed to remove old container")
1993	}
1994
1995	// Rename new container to canonical name
1996	canonicalName := fmt.Sprintf("gordon-%s", domainName)
1997	if err := s.runtime.RenameContainer(ctx, newContainerID, canonicalName); err != nil {
1998		log.Warn().Err(err).Str("canonical_name", canonicalName).Msg("failed to rename container to canonical name")
1999	}
2000}
2001
2002// startLogCollection starts log collection for a container in the background.
2003// Errors are logged but don't fail the calling operation.
2004func (s *Service) startLogCollection(ctx context.Context, containerID, domainName string) {
2005	if s.logWriter == nil {
2006		return
2007	}
2008
2009	log := zerowrap.FromCtx(ctx)
2010
2011	// Use context.WithoutCancel so the log stream outlives the HTTP request.
2012	// Without this, the Docker log stream closes when the deploy response completes.
2013	bgCtx := context.WithoutCancel(ctx)
2014
2015	logStream, err := s.runtime.GetContainerLogs(bgCtx, containerID, true)
2016	if err != nil {
2017		log.Warn().Err(err).Msg("failed to get container logs for collection")
2018		return
2019	}
2020
2021	if err := s.logWriter.StartLogging(bgCtx, containerID, domainName, logStream); err != nil {
2022		log.Warn().Err(err).Msg("failed to start container log collection")
2023		logStream.Close()
2024	}
2025}
2026
2027func (s *Service) buildValidatedImageRef(ctx context.Context, image string) (string, error) {
2028	imageRef := s.buildImageRef(image)
2029	if err := s.validateImageRef(ctx, imageRef); err != nil {
2030		return "", err
2031	}
2032	return imageRef, nil
2033}
2034
2035func (s *Service) buildImageRef(image string) string {
2036	s.mu.RLock()
2037	cfg := s.config
2038	s.mu.RUnlock()
2039
2040	if !cfg.RegistryAuthEnabled || cfg.RegistryDomain == "" {
2041		return image
2042	}
2043
2044	// Normalize registry domain by trimming any trailing slash
2045	reg := strings.TrimSuffix(cfg.RegistryDomain, "/")
2046
2047	// Check if image already has the registry domain prefix
2048	prefix := reg + "/"
2049	if strings.HasPrefix(image, prefix) {
2050		return image
2051	}
2052
2053	// Check if image already has an explicit registry (host:port/ or host.domain/).
2054	// We need to detect patterns like:
2055	// - "docker.io/library/nginx:latest" (has dot and slash)
2056	// - "localhost:5001/myapp:latest" (has host:port/ pattern)
2057	// - "myregistry:5000/app:v1" (has host:port/ pattern)
2058	// - "[fd00::1]:5000/app:v1" (IPv6 with port)
2059	if hasExplicitRegistry(image) {
2060		return image
2061	}
2062
2063	return fmt.Sprintf("%s/%s", reg, image)
2064}
2065
2066// hasExplicitRegistry checks if an image reference already includes an explicit registry.
2067// Returns true for patterns like "docker.io/image", "localhost:5000/image", "localhost/image",
2068// "registry:8080/image", "[fd00::1]/image", "[fd00::1]:5000/image".
2069func hasExplicitRegistry(image string) bool {
2070	// Find the first slash which separates registry from image name
2071	slashIdx := strings.Index(image, "/")
2072	if slashIdx == -1 {
2073		return false // No slash means no registry prefix (e.g., "myapp:latest")
2074	}
2075
2076	// Extract the part before the first slash (potential registry)
2077	registryPart := image[:slashIdx]
2078
2079	// Check for bracketed IPv6 address (e.g., "[fd00::1]" or "[fd00::1]:5000")
2080	if strings.HasPrefix(registryPart, "[") {
2081		// Look for closing bracket
2082		if closeBracket := strings.Index(registryPart, "]"); closeBracket != -1 {
2083			// Valid bracketed IPv6: either ends at ] or has ]:port
2084			return true
2085		}
2086	}
2087
2088	// Check for localhost (with or without port)
2089	if registryPart == "localhost" {
2090		return true
2091	}
2092
2093	// Check for host:port pattern (e.g., "localhost:5000", "registry:8080")
2094	if colonIdx := strings.LastIndex(registryPart, ":"); colonIdx != -1 {
2095		port := registryPart[colonIdx+1:]
2096		// If everything after colon is digits, it's a port
2097		if len(port) > 0 && isNumeric(port) {
2098			return true
2099		}
2100	}
2101
2102	// Check for domain pattern (has a dot, e.g., "docker.io", "gcr.io")
2103	if strings.Contains(registryPart, ".") {
2104		return true
2105	}
2106
2107	return false
2108}
2109
2110func (s *Service) validateImageRef(ctx context.Context, imageRef string) error {
2111	if !hasExplicitRegistry(imageRef) {
2112		return nil
2113	}
2114	return s.validateExternalImageRef(ctx, imageRef, imageRegistry(imageRef))
2115}
2116
2117func (s *Service) validateImagePullRef(ctx context.Context, imageRef string) error {
2118	return s.validateExternalImageRef(ctx, imageRef, imageRegistryForPolicy(imageRef))
2119}
2120
2121func (s *Service) validateExternalImageRef(ctx context.Context, imageRef, registry string) error {
2122	s.mu.RLock()
2123	cfg := s.config
2124	s.mu.RUnlock()
2125
2126	if registry == "" {
2127		return fmt.Errorf("image %q rejected: invalid image reference", imageRef)
2128	}
2129
2130	if domain.IsGordonRegistryImageRef(imageRef, cfg.RegistryDomain, cfg.LegacyRegistryDomains) || sameRegistry(registry, cfg.RegistryDomain) {
2131		return nil
2132	}
2133	dangerous, err := isDangerousRegistryHost(ctx, imageRegistryHost(registry))
2134	if err != nil {
2135		return fmt.Errorf("image %q rejected: registry %q could not be verified: %w", imageRef, registry, err)
2136	}
2137	if dangerous {
2138		return fmt.Errorf("image %q rejected: registry %q is not allowed", imageRef, registry)
2139	}
2140	if len(cfg.AllowedRegistries) == 0 {
2141		return fmt.Errorf("image %q rejected: registry %q is not in images.allowed_registries", imageRef, registry)
2142	}
2143	for _, allowed := range cfg.AllowedRegistries {
2144		if sameRegistry(registry, allowed) {
2145			if cfg.RequireImageDigest && !isValidSha256DigestRef(imageRef) {
2146				return fmt.Errorf("image %q rejected: digest reference required", imageRef)
2147			}
2148			return nil
2149		}
2150	}
2151	return fmt.Errorf("image %q rejected: registry %q is not in images.allowed_registries", imageRef, registry)
2152}
2153
2154func imageRegistry(imageRef string) string {
2155	slashIdx := strings.Index(imageRef, "/")
2156	if slashIdx == -1 {
2157		return ""
2158	}
2159	return normalizeRegistry(imageRef[:slashIdx])
2160}
2161
2162func imageRegistryForPolicy(imageRef string) string {
2163	if hasExplicitRegistry(imageRef) {
2164		return imageRegistry(imageRef)
2165	}
2166	return "docker.io"
2167}
2168
2169// normalizeRegistry normalizes a registry string for comparison purposes only.
2170// This function is used by sameRegistry to determine if two registry strings
2171// refer to the same registry. The output intentionally strips IPv6 brackets
2172// (e.g., "[fd00::1]:5000" -> "fd00::1:5000") which is acceptable for equality
2173// checks but the result MUST NOT be used to construct or emit valid registry URLs.
2174func normalizeRegistry(registry string) string {
2175	registry = strings.ToLower(strings.TrimSuffix(strings.TrimSpace(registry), "."))
2176	registry = strings.TrimSuffix(registry, "/")
2177	if strings.HasPrefix(registry, "[") {
2178		if end := strings.Index(registry, "]"); end != -1 {
2179			host := strings.Trim(registry[:end+1], "[]")
2180			port := registry[end+1:]
2181			return host + port
2182		}
2183	}
2184	return registry
2185}
2186
2187func imageRegistryHost(registry string) string {
2188	if ip := net.ParseIP(registry); ip != nil {
2189		return registry
2190	}
2191	if h, _, err := net.SplitHostPort(registry); err == nil {
2192		return strings.Trim(h, "[]")
2193	}
2194	if strings.HasPrefix(registry, "[") && strings.Contains(registry, "]") {
2195		return strings.TrimPrefix(strings.Split(registry, "]")[0], "[")
2196	}
2197	if colon := strings.LastIndex(registry, ":"); colon != -1 && isNumeric(registry[colon+1:]) {
2198		return registry[:colon]
2199	}
2200	return registry
2201}
2202
2203func sameRegistry(a, b string) bool {
2204	return normalizeRegistry(a) == normalizeRegistry(b)
2205}
2206
2207func isDangerousRegistryHost(ctx context.Context, host string) (bool, error) {
2208	host = strings.ToLower(strings.TrimSuffix(host, "."))
2209	if host == "localhost" || host == "metadata.google.internal" {
2210		return true, nil
2211	}
2212	if ip := net.ParseIP(host); ip != nil {
2213		return isDangerousRegistryIP(ip), nil
2214	}
2215	addrs, err := net.DefaultResolver.LookupIPAddr(ctx, host)
2216	if err != nil {
2217		return false, err
2218	}
2219	for _, addr := range addrs {
2220		if isDangerousRegistryIP(addr.IP) {
2221			return true, nil
2222		}
2223	}
2224	return false, nil
2225}
2226
2227func isDangerousRegistryIP(ip net.IP) bool {
2228	return ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsUnspecified()
2229}
2230
2231func applySecurityProfile(config *domain.ContainerConfig, cfg Config) {
2232	if strings.EqualFold(cfg.SecurityProfile, "strict") {
2233		config.ReadOnlyRootFS = true
2234		config.CapDrop = []string{"ALL"}
2235		config.CapAdd = []string{"NET_BIND_SERVICE"}
2236	}
2237}
2238
2239// isNumeric checks if a string contains only digits.
2240func isNumeric(s string) bool {
2241	for _, c := range s {
2242		if c < '0' || c > '9' {
2243			return false
2244		}
2245	}
2246	return true
2247}
2248
2249func normalizePullPolicy(policy string) string {
2250	switch strings.ToLower(strings.TrimSpace(policy)) {
2251	case PullPolicyAlways:
2252		return PullPolicyAlways
2253	case PullPolicyIfTagChanged:
2254		return PullPolicyIfTagChanged
2255	case PullPolicyIfNotPresent:
2256		return PullPolicyIfNotPresent
2257	default:
2258		return PullPolicyIfNotPresent
2259	}
2260}
2261
2262func isDigestRef(imageRef string) bool {
2263	return isValidSha256DigestRef(imageRef)
2264}
2265
2266func isValidSha256DigestRef(imageRef string) bool {
2267	_, digest, ok := strings.Cut(imageRef, "@sha256:")
2268	if !ok || len(digest) != 64 {
2269		return false
2270	}
2271	_, err := hex.DecodeString(digest)
2272	return err == nil
2273}
2274
2275func (s *Service) pullRefForDeploy(ctx context.Context, imageRef string) (string, bool) {
2276	if !domain.IsInternalDeploy(ctx) {
2277		return imageRef, false
2278	}
2279	s.mu.RLock()
2280	cfg := s.config
2281	s.mu.RUnlock()
2282	return rewriteToLocalRegistry(imageRef, cfg.RegistryDomain, cfg.LegacyRegistryDomains, cfg.RegistryPort), true
2283}
2284
2285// ensureImage ensures the image is available locally, pulling if needed.
2286// Returns the image reference to use for container operations:
2287//   - For digest references (@sha256:...), returns the pullRef since Docker can't tag digests
2288//   - For tagged images, returns the original imageRef after tagging the pulled image
2289func (s *Service) ensureImage(ctx context.Context, imageRef string) (string, error) {
2290	ctx, span := tracer.Start(ctx, "container.ensure_image",
2291		trace.WithAttributes(attribute.String("image", imageRef)))
2292	defer span.End()
2293
2294	ctx = zerowrap.CtxWithField(ctx, "image", imageRef)
2295	log := zerowrap.FromCtx(ctx)
2296
2297	// Determine if this is an internal deploy and what reference to use for pulls.
2298	pullRef, isInternal := s.pullRefForDeploy(ctx, imageRef)
2299	if err := s.validateImagePullRef(ctx, imageRef); err != nil {
2300		return "", err
2301	}
2302	if pullRef != imageRef {
2303		log.Info().
2304			Str("original_ref", imageRef).
2305			Str("pull_ref", pullRef).
2306			Msg("internal deploy: using localhost registry for pull")
2307	}
2308
2309	found, err := s.ensureLocalImage(ctx, imageRef, pullRef)
2310	if err != nil {
2311		return "", err
2312	}
2313	if found {
2314		return imageRef, nil
2315	}
2316
2317	// Pull image
2318	log.Info().Msg("pulling image from registry")
2319
2320	if err := s.pullImage(ctx, pullRef, isInternal); err != nil {
2321		return "", err
2322	}
2323
2324	// For digest references, we can't create a tag (Docker doesn't allow it).
2325	// In this case, use the pullRef directly since the image is available under that reference.
2326	if strings.Contains(imageRef, "@sha256:") {
2327		log.Info().
2328			Str("pull_ref", pullRef).
2329			Msg("digest reference: using pull reference for container operations")
2330		return pullRef, nil
2331	}
2332
2333	if err := s.tagImageIfNeeded(ctx, pullRef, imageRef); err != nil {
2334		return "", err
2335	}
2336
2337	// Clean up the temporary pull reference tag to avoid duplicate entries
2338	if pullRef != imageRef {
2339		if err := s.runtime.UntagImage(ctx, pullRef); err != nil {
2340			// Log but don't fail - the canonical tag is already applied
2341			log.Debug().Err(err).Str("pull_ref", pullRef).Msg("failed to remove temporary pull tag")
2342		}
2343	}
2344
2345	log.Info().Msg("image pulled successfully")
2346	return imageRef, nil
2347}
2348
2349func (s *Service) ensureLocalImage(ctx context.Context, imageRef, pullRef string) (bool, error) {
2350	log := zerowrap.FromCtx(ctx)
2351
2352	// For internal deploys (image push events), always pull fresh image
2353	// because the same tag may reference new content (e.g., latest tag updated)
2354	if domain.IsInternalDeploy(ctx) {
2355		log.Info().Msg("internal deploy detected, forcing image pull to ensure latest content")
2356		return false, nil
2357	}
2358
2359	s.mu.RLock()
2360	cfg := s.config
2361	s.mu.RUnlock()
2362
2363	pullPolicy := normalizePullPolicy(cfg.PullPolicy)
2364	switch pullPolicy {
2365	case PullPolicyAlways:
2366		log.Info().Str("pull_policy", pullPolicy).Msg("pull policy forces image pull")
2367		return false, nil
2368	case PullPolicyIfTagChanged:
2369		if !isDigestRef(imageRef) {
2370			log.Info().Str("pull_policy", pullPolicy).Msg("tag reference detected, pulling to check for updates")
2371			return false, nil
2372		}
2373	}
2374
2375	localImages, err := s.runtime.ListImages(ctx)
2376	if err != nil {
2377		log.WrapErr(err, "failed to list local images, will attempt pull")
2378		return false, nil
2379	}
2380
2381	normalizedRef := normalizeImageRef(imageRef)
2382	normalizedPullRef := normalizeImageRef(pullRef)
2383	for _, img := range localImages {
2384		normalizedImage := normalizeImageRef(img)
2385		if normalizedImage == normalizedRef {
2386			log.Info().Msg("image found locally, skipping pull")
2387			return true, nil
2388		}
2389		if normalizedImage == normalizedPullRef {
2390			if err := s.tagImageIfNeeded(ctx, pullRef, imageRef); err != nil {
2391				return false, err
2392			}
2393			// Clean up the temporary pull reference tag
2394			if pullRef != imageRef {
2395				if err := s.runtime.UntagImage(ctx, pullRef); err != nil {
2396					log.Debug().Err(err).Str("pull_ref", pullRef).Msg("failed to remove temporary pull tag")
2397				}
2398			}
2399			log.Info().Msg("image found locally, skipping pull")
2400			return true, nil
2401		}
2402	}
2403
2404	return false, nil
2405}
2406
2407func (s *Service) pullImage(ctx context.Context, pullRef string, isInternal bool) error {
2408	s.mu.RLock()
2409	cfg := s.config
2410	s.mu.RUnlock()
2411
2412	log := zerowrap.FromCtx(ctx)
2413
2414	switch {
2415	case isInternal && cfg.RegistryAuthEnabled:
2416		if cfg.InternalRegistryUsername == "" || cfg.InternalRegistryPassword == "" {
2417			return log.WrapErr(fmt.Errorf("internal registry auth not configured"), "failed to pull image for internal deploy")
2418		}
2419		if err := s.pullImageWithRetry(ctx, func(ctx context.Context) error {
2420			return s.runtime.PullImageWithAuth(ctx, pullRef, cfg.InternalRegistryUsername, cfg.InternalRegistryPassword)
2421		}, isInternal); err != nil {
2422			return log.WrapErr(fmt.Errorf("%w: %w", domain.ErrImagePullFailed, err), "failed to pull image with internal auth")
2423		}
2424	case isInternal:
2425		if err := s.pullImageWithRetry(ctx, func(ctx context.Context) error {
2426			return s.runtime.PullImage(ctx, pullRef)
2427		}, isInternal); err != nil {
2428			return log.WrapErr(fmt.Errorf("%w: %w", domain.ErrImagePullFailed, err), "failed to pull image")
2429		}
2430	case cfg.RegistryAuthEnabled:
2431		if cfg.ServiceTokenUsername == "" || cfg.ServiceToken == "" {
2432			return log.WrapErr(fmt.Errorf("registry service token not configured"), "failed to pull image for registry auth")
2433		}
2434		if err := s.runtime.PullImageWithAuth(ctx, pullRef, cfg.ServiceTokenUsername, cfg.ServiceToken); err != nil {
2435			return log.WrapErr(fmt.Errorf("%w: %w", domain.ErrImagePullFailed, err), "failed to pull image with auth")
2436		}
2437	default:
2438		if err := s.runtime.PullImage(ctx, pullRef); err != nil {
2439			return log.WrapErr(fmt.Errorf("%w: %w", domain.ErrImagePullFailed, err), "failed to pull image")
2440		}
2441	}
2442
2443	return nil
2444}
2445
2446func (s *Service) pullImageWithRetry(ctx context.Context, pullFn func(context.Context) error, isInternal bool) error {
2447	log := zerowrap.FromCtx(ctx)
2448
2449	for attempt := 1; attempt <= internalPullMaxAttempts; attempt++ {
2450		err := pullFn(ctx)
2451		if err == nil {
2452			return nil
2453		}
2454		if !isInternal || !isConnectionRefusedError(err) || attempt == internalPullMaxAttempts {
2455			return fmt.Errorf("internal image pull failed after %d attempts: %w", attempt, err)
2456		}
2457
2458		backoff := time.Duration(attempt) * time.Second
2459		log.Warn().
2460			Err(err).
2461			Int("attempt", attempt).
2462			Dur("backoff", backoff).
2463			Msg("internal image pull failed with connection refused, retrying")
2464
2465		select {
2466		case <-time.After(backoff):
2467		case <-ctx.Done():
2468			return ctx.Err()
2469		}
2470	}
2471	return nil
2472}
2473
2474func (s *Service) tagImageIfNeeded(ctx context.Context, sourceRef, targetRef string) error {
2475	if sourceRef == targetRef {
2476		return nil
2477	}
2478
2479	// Cannot create a tag with a digest reference - Docker/Podman doesn't allow it.
2480	// When using digest references (image@sha256:...), skip tagging as the image
2481	// is already available by its digest.
2482	if strings.Contains(targetRef, "@sha256:") {
2483		return nil
2484	}
2485
2486	log := zerowrap.FromCtx(ctx)
2487	if err := s.runtime.TagImage(ctx, sourceRef, targetRef); err != nil {
2488		return log.WrapErr(err, "failed to tag image from pull reference")
2489	}
2490	return nil
2491}
2492
2493func (s *Service) loadEnvironment(ctx context.Context, preResolved []string, domainName, imageRef string) ([]string, error) {
2494	log := zerowrap.FromCtx(ctx)
2495
2496	var userEnvVars []string
2497	if len(preResolved) > 0 {
2498		userEnvVars = preResolved
2499	} else {
2500		var err error
2501		userEnvVars, err = s.envLoader.LoadEnv(ctx, domainName)
2502		if err != nil {
2503			return nil, log.WrapErr(err, "failed to load environment variables")
2504		}
2505	}
2506
2507	dockerfileEnvVars, err := s.runtime.InspectImageEnv(ctx, imageRef)
2508	if err != nil {
2509		log.WrapErr(err, "failed to inspect image environment")
2510		dockerfileEnvVars = []string{}
2511	}
2512
2513	return mergeEnvironmentVariables(dockerfileEnvVars, userEnvVars), nil
2514}
2515
2516func (s *Service) setupVolumes(ctx context.Context, domainName, imageRef string) (map[string]string, error) {
2517	s.mu.RLock()
2518	cfg := s.config
2519	s.mu.RUnlock()
2520
2521	log := zerowrap.FromCtx(ctx)
2522	volumes := make(map[string]string)
2523
2524	if !cfg.VolumeAutoCreate {
2525		return volumes, nil
2526	}
2527
2528	volumePaths, err := s.runtime.InspectImageVolumes(ctx, imageRef)
2529	if err != nil {
2530		log.WrapErr(err, "failed to inspect image volumes")
2531		return volumes, nil
2532	}
2533
2534	for _, path := range volumePaths {
2535		name := generateVolumeName(cfg.VolumePrefix, domainName, path)
2536
2537		exists, err := s.runtime.VolumeExists(ctx, name)
2538		if err != nil {
2539			log.WrapErrWithFields(err, "failed to check volume", map[string]any{"volume": name})
2540			continue
2541		}
2542
2543		if !exists {
2544			if err := s.runtime.CreateVolume(ctx, name); err != nil {
2545				log.WrapErrWithFields(err, "failed to create volume", map[string]any{"volume": name})
2546				continue
2547			}
2548			log.Info().Str("volume", name).Str(zerowrap.FieldPath, path).Msg("created volume")
2549		}
2550
2551		volumes[path] = name
2552	}
2553
2554	return volumes, nil
2555}
2556
2557func (s *Service) getNetworkForApp(domainName string) string {
2558	var networkIsolation bool
2559	var networkGroups map[string][]string
2560
2561	if s.configProvider != nil {
2562		snap := s.configProvider.GetAttachmentConfig()
2563		networkGroups = snap.NetworkGroups
2564		s.mu.RLock()
2565		networkIsolation = s.config.NetworkIsolation
2566		s.mu.RUnlock()
2567	} else {
2568		s.mu.RLock()
2569		networkIsolation = s.config.NetworkIsolation
2570		networkGroups = s.config.NetworkGroups
2571		s.mu.RUnlock()
2572	}
2573
2574	if !networkIsolation {
2575		return "bridge"
2576	}
2577
2578	for groupName, domains := range networkGroups {
2579		if slices.Contains(domains, domainName) {
2580			return s.generateNetworkName(groupName)
2581		}
2582	}
2583
2584	return s.generateNetworkName(domainName)
2585}
2586
2587func (s *Service) generateNetworkName(identifier string) string {
2588	s.mu.RLock()
2589	cfg := s.config
2590	s.mu.RUnlock()
2591
2592	return fmt.Sprintf("%s-%s", cfg.NetworkPrefix, strings.ReplaceAll(identifier, ".", "-"))
2593}
2594
2595func (s *Service) createNetworkIfNeeded(ctx context.Context, networkName string) error {
2596	if networkName == "bridge" || networkName == "default" {
2597		return nil
2598	}
2599
2600	ctx = zerowrap.CtxWithField(ctx, "network", networkName)
2601	log := zerowrap.FromCtx(ctx)
2602
2603	exists, err := s.runtime.NetworkExists(ctx, networkName)
2604	if err != nil {
2605		return log.WrapErr(err, "failed to check network existence")
2606	}
2607
2608	if !exists {
2609		s.mu.RLock()
2610		internal := s.config.NetworkInternal
2611		s.mu.RUnlock()
2612		if err := s.runtime.CreateNetwork(ctx, networkName, domain.NetworkConfig{Driver: "bridge", Internal: internal}); err != nil {
2613			return log.WrapErr(err, "failed to create network")
2614		}
2615		log.Info().Msg("created network for app isolation")
2616	}
2617
2618	return nil
2619}
2620
2621func (s *Service) cleanupOrphanedContainers(ctx context.Context, domainName string, skipContainerID string) error {
2622	log := zerowrap.FromCtx(ctx)
2623	expectedName := fmt.Sprintf("gordon-%s", domainName)
2624	expectedNewName := expectedName + "-new"
2625	expectedNextName := expectedName + "-next"
2626
2627	allContainers, err := s.runtime.ListContainers(ctx, true)
2628	if err != nil {
2629		return err
2630	}
2631
2632	for _, c := range allContainers {
2633		if (c.Name == expectedName || c.Name == expectedNewName || c.Name == expectedNextName) && c.ID != skipContainerID {
2634			// Skip any container that is still active in the runtime, including
2635			// restart backoff, regardless of name — a temp container may be
2636			// serving traffic while the new container stabilizes, or may be from
2637			// a concurrent deploy.
2638			if c.Status == "running" || c.Status == "restarting" {
2639				log.Debug().
2640					Str(zerowrap.FieldEntityID, c.ID).
2641					Str("container_name", c.Name).
2642					Str(zerowrap.FieldStatus, c.Status).
2643					Msg("skipping active container during orphan cleanup")
2644				continue
2645			}
2646
2647			log.Info().Str(zerowrap.FieldEntityID, c.ID).Str("container_name", c.Name).Str(zerowrap.FieldStatus, c.Status).Msg("found orphaned container, removing")
2648
2649			if err := s.runtime.StopContainer(ctx, c.ID); err != nil {
2650				log.WrapErrWithFields(err, "failed to stop orphaned container", map[string]any{zerowrap.FieldEntityID: c.ID})
2651			}
2652
2653			if err := s.runtime.RemoveContainer(ctx, c.ID, true); err != nil {
2654				log.WrapErrWithFields(err, "failed to remove orphaned container", map[string]any{zerowrap.FieldEntityID: c.ID})
2655			}
2656		}
2657	}
2658
2659	return nil
2660}
2661
2662func (s *Service) cleanupVolumesForDomain(_ context.Context, _ string) error {
2663	return nil
2664}
2665
2666func (s *Service) cleanupNetworkIfEmpty(ctx context.Context, networkName string) error {
2667	if networkName == "bridge" || networkName == "default" {
2668		return nil
2669	}
2670
2671	ctx = zerowrap.CtxWithField(ctx, "network", networkName)
2672	log := zerowrap.FromCtx(ctx)
2673
2674	networks, err := s.runtime.ListNetworks(ctx)
2675	if err != nil {
2676		return log.WrapErr(err, "failed to list networks")
2677	}
2678
2679	for _, n := range networks {
2680		if n.Name == networkName && len(n.Containers) == 0 {
2681			if err := s.runtime.RemoveNetwork(ctx, networkName); err != nil {
2682				return err
2683			}
2684			log.Info().Msg("cleaned up empty network")
2685			break
2686		}
2687	}
2688
2689	return nil
2690}
2691
2692func (s *Service) deployAttachments(ctx context.Context, domainName, networkName string) error {
2693	// Collect attachments from both domain-specific and network group configs
2694	attachments := s.resolveAttachmentsForDomain(domainName)
2695	if len(attachments) == 0 {
2696		return nil
2697	}
2698
2699	log := zerowrap.FromCtx(ctx)
2700	var deployed []string // track successfully deployed attachment IDs for rollback
2701	for _, svc := range attachments {
2702		if err := s.deployAttachedService(ctx, domainName, svc, networkName); err != nil {
2703			log.WrapErrWithFields(err, "failed to deploy attachment", map[string]any{zerowrap.FieldService: svc, "domain": domainName})
2704
2705			// Rollback: clean up already-deployed attachments
2706			for _, id := range deployed {
2707				if stopErr := s.runtime.StopContainer(ctx, id); stopErr != nil {
2708					log.Warn().Err(stopErr).Str("attachment_id", id).Msg("rollback: failed to stop attachment")
2709				}
2710				if rmErr := s.runtime.RemoveContainer(ctx, id, true); rmErr != nil {
2711					log.Warn().Err(rmErr).Str("attachment_id", id).Msg("rollback: failed to remove attachment")
2712				}
2713			}
2714			// Deregister rolled-back attachments
2715			if len(deployed) > 0 {
2716				s.mu.Lock()
2717				remaining := s.attachments[domainName]
2718				filtered := remaining[:0]
2719				rollbackSet := make(map[string]bool, len(deployed))
2720				for _, id := range deployed {
2721					rollbackSet[id] = true
2722				}
2723				for _, id := range remaining {
2724					if !rollbackSet[id] {
2725						filtered = append(filtered, id)
2726					}
2727				}
2728				s.attachments[domainName] = filtered
2729				s.mu.Unlock()
2730			}
2731
2732			return fmt.Errorf("failed to deploy attachment %q (rolled back %d already-deployed)", svc, len(deployed))
2733		}
2734		// Collect IDs of attachments tracked under this domain after successful deploy
2735		s.mu.RLock()
2736		ids := s.attachments[domainName]
2737		if len(ids) > 0 {
2738			latest := ids[len(ids)-1]
2739			deployed = append(deployed, latest)
2740		}
2741		s.mu.RUnlock()
2742	}
2743	return nil
2744}
2745
2746// resolveAttachmentsForDomain returns attachments for a domain by checking:
2747// 1. Direct domain attachments (attachments[domain])
2748// 2. Network group attachments (attachments[group] where domain is in network_groups[group])
2749func (s *Service) resolveAttachmentsForDomain(domainName string) []string {
2750	var attachments map[string][]string
2751	var networkGroups map[string][]string
2752
2753	if s.configProvider != nil {
2754		snap := s.configProvider.GetAttachmentConfig()
2755		attachments = snap.Attachments
2756		networkGroups = snap.NetworkGroups
2757	} else {
2758		s.mu.RLock()
2759		attachments = s.config.Attachments
2760		networkGroups = s.config.NetworkGroups
2761		s.mu.RUnlock()
2762	}
2763
2764	seen := make(map[string]bool)
2765	var result []string
2766
2767	// First, add domain-specific attachments
2768	if domainAttachments, ok := attachments[domainName]; ok {
2769		for _, img := range domainAttachments {
2770			if !seen[img] {
2771				seen[img] = true
2772				result = append(result, img)
2773			}
2774		}
2775	}
2776
2777	// Then, find which network group this domain belongs to and add group attachments
2778	for groupName, domains := range networkGroups {
2779		if slices.Contains(domains, domainName) {
2780			if groupAttachments, ok := attachments[groupName]; ok {
2781				for _, img := range groupAttachments {
2782					if !seen[img] {
2783						seen[img] = true
2784						result = append(result, img)
2785					}
2786				}
2787			}
2788			break // Domain can only be in one network group
2789		}
2790	}
2791
2792	return result
2793}
2794
2795func (s *Service) getAttachmentsForDomain(ctx context.Context, domainName string) []domain.Attachment {
2796	containers, err := s.runtime.ListContainers(ctx, true)
2797	if err != nil {
2798		return nil
2799	}
2800
2801	return s.filterAttachments(ctx, containers, domainName)
2802}
2803
2804// getAllAttachments fetches all containers once and returns attachments grouped by domain.
2805// This avoids N+1 queries when listing multiple routes.
2806func (s *Service) getAllAttachments(ctx context.Context) map[string][]domain.Attachment {
2807	containers, err := s.runtime.ListContainers(ctx, true)
2808	if err != nil {
2809		return nil
2810	}
2811
2812	result := make(map[string][]domain.Attachment)
2813	for _, container := range containers {
2814		if container.Labels == nil {
2815			continue
2816		}
2817		if container.Labels[domain.LabelAttachment] != "true" {
2818			continue
2819		}
2820		ownerDomain := container.Labels[domain.LabelAttachedTo]
2821		if ownerDomain == "" {
2822			continue
2823		}
2824		image := container.Image
2825		serviceName := container.Name
2826		if labelImage, ok := container.Labels[domain.LabelImage]; ok && labelImage != "" {
2827			image = labelImage
2828			serviceName = extractServiceName(labelImage)
2829		}
2830		// Get the container's network for display in the routes table
2831		network := ""
2832		if networkName, err := s.runtime.GetContainerNetwork(ctx, container.ID); err == nil {
2833			network = networkName
2834		}
2835		attachment := domain.Attachment{
2836			Name:        serviceName,
2837			Image:       s.stripRegistryPrefix(image),
2838			ContainerID: container.ID,
2839			Status:      container.Status,
2840			Network:     network,
2841			Ports:       append([]int(nil), container.Ports...),
2842		}
2843		result[ownerDomain] = append(result[ownerDomain], attachment)
2844	}
2845
2846	return result
2847}
2848
2849// filterAttachments extracts attachments for a specific domain from a container list.
2850func (s *Service) filterAttachments(ctx context.Context, containers []*domain.Container, domainName string) []domain.Attachment {
2851	attachments := make([]domain.Attachment, 0)
2852	for _, container := range containers {
2853		if container.Labels == nil {
2854			continue
2855		}
2856		if container.Labels[domain.LabelAttachment] != "true" {
2857			continue
2858		}
2859		if container.Labels[domain.LabelAttachedTo] != domainName {
2860			continue
2861		}
2862		image := container.Image
2863		serviceName := container.Name
2864		if labelImage, ok := container.Labels[domain.LabelImage]; ok && labelImage != "" {
2865			image = labelImage
2866			serviceName = extractServiceName(labelImage)
2867		}
2868		// Get the container's network for display in the routes table
2869		network := ""
2870		if networkName, err := s.runtime.GetContainerNetwork(ctx, container.ID); err == nil {
2871			network = networkName
2872		}
2873		attachment := domain.Attachment{
2874			Name:        serviceName,
2875			Image:       s.stripRegistryPrefix(image),
2876			ContainerID: container.ID,
2877			Status:      container.Status,
2878			Network:     network,
2879			Ports:       append([]int(nil), container.Ports...),
2880		}
2881		attachments = append(attachments, attachment)
2882	}
2883
2884	return attachments
2885}
2886
2887func (s *Service) deployAttachedService(ctx context.Context, ownerDomain, serviceImage, networkName string) error {
2888	log := zerowrap.FromCtx(ctx)
2889
2890	// Parse service name from image (e.g., "my-postgres:latest" → "postgres")
2891	serviceName := extractServiceName(serviceImage)
2892	containerName := fmt.Sprintf("gordon-%s-%s", sanitizeName(ownerDomain), serviceName)
2893
2894	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
2895		"attachment":     serviceName,
2896		"container_name": containerName,
2897		"owner_domain":   ownerDomain,
2898	})
2899
2900	// Check if already running (idempotent)
2901	// Try new name first, then fall back to legacy name for backwards compatibility
2902	existingContainer, err := s.resolveExistingAttachment(ctx, ownerDomain, serviceName)
2903	if err != nil {
2904		return err
2905	}
2906
2907	if existingContainer != nil {
2908		shouldSkip, err := s.handleRunningAttachment(ctx, existingContainer, containerName, serviceImage)
2909		if err != nil {
2910			return err
2911		}
2912		if shouldSkip {
2913			return nil
2914		}
2915		if existingContainer.Status == string(domain.ContainerStatusRunning) {
2916			existingContainer = nil
2917		}
2918	}
2919
2920	if err := s.removeStoppedAttachment(ctx, existingContainer, containerName); err != nil {
2921		return err
2922	}
2923
2924	log.Info().Str(zerowrap.FieldService, serviceImage).Msg("deploying attached service")
2925
2926	config, err := s.buildAttachmentContainerConfig(ctx, ownerDomain, serviceName, serviceImage, containerName, networkName)
2927	if err != nil {
2928		return err
2929	}
2930
2931	container, err := s.runtime.CreateContainer(ctx, config)
2932	if err != nil {
2933		return log.WrapErr(err, "failed to create attachment container")
2934	}
2935
2936	// Start container
2937	if err := s.runtime.StartContainer(ctx, container.ID); err != nil {
2938		s.runtime.RemoveContainer(ctx, container.ID, true)
2939		return log.WrapErr(err, "failed to start attachment container")
2940	}
2941
2942	// Wait for attachment to be ready before proceeding
2943	if err := s.waitForAttachmentReady(ctx, container.ID, config); err != nil {
2944		log.WrapErr(err, "attachment readiness check failed, cleaning up")
2945		if stopErr := s.runtime.StopContainer(ctx, container.ID); stopErr != nil {
2946			log.Warn().Err(stopErr).Str(zerowrap.FieldEntityID, container.ID).Msg("failed to stop unready attachment")
2947		}
2948		s.runtime.RemoveContainer(ctx, container.ID, true)
2949		return fmt.Errorf("attachment %q not ready: %w", serviceName, err)
2950	}
2951	// Track attachment
2952	s.mu.Lock()
2953	s.attachments[ownerDomain] = append(s.attachments[ownerDomain], container.ID)
2954	s.mu.Unlock()
2955
2956	// Start log collection for attachment
2957	s.startLogCollection(ctx, container.ID, containerName)
2958
2959	log.Info().Str(zerowrap.FieldEntityID, container.ID).Msg("attachment deployed successfully")
2960	return nil
2961}
2962
2963func (s *Service) buildAttachmentContainerConfig(ctx context.Context, ownerDomain, serviceName, serviceImage, containerName, networkName string) (*domain.ContainerConfig, error) {
2964	imageRef, err := s.buildValidatedImageRef(ctx, serviceImage)
2965	if err != nil {
2966		return nil, err
2967	}
2968	actualImageRef, err := s.ensureImage(ctx, imageRef)
2969	if err != nil {
2970		return nil, err
2971	}
2972
2973	exposedPorts := s.attachmentExposedPorts(ctx, actualImageRef)
2974	volumes, err := s.attachmentVolumes(ctx, containerName, actualImageRef)
2975	if err != nil {
2976		return nil, err
2977	}
2978	envVars, err := s.attachmentEnv(ctx, containerName, actualImageRef)
2979	if err != nil {
2980		return nil, err
2981	}
2982
2983	s.mu.RLock()
2984	cfg := s.config
2985	s.mu.RUnlock()
2986
2987	config := &domain.ContainerConfig{
2988		Image:       actualImageRef,
2989		Name:        containerName,
2990		Hostname:    serviceName,
2991		Aliases:     []string{serviceName},
2992		Ports:       exposedPorts,
2993		Env:         envVars,
2994		Volumes:     volumes,
2995		NetworkMode: networkName,
2996		Labels: map[string]string{
2997			domain.LabelManaged:    "true",
2998			domain.LabelAttachment: "true",
2999			domain.LabelAttachedTo: ownerDomain,
3000			domain.LabelEnvHash:    hashEnvironment(envVars),
3001			domain.LabelImage:      serviceImage,
3002		},
3003		MemoryLimit:   cfg.DefaultMemoryLimit,
3004		NanoCPUs:      cfg.DefaultNanoCPUs,
3005		PidsLimit:     cfg.DefaultPidsLimit,
3006		RestartPolicy: domain.RestartPolicyAlways,
3007	}
3008	applySecurityProfile(config, cfg)
3009	return config, nil
3010}
3011
3012func (s *Service) attachmentExposedPorts(ctx context.Context, imageRef string) []int {
3013	log := zerowrap.FromCtx(ctx)
3014	exposedPorts, err := s.runtime.GetImageExposedPorts(ctx, imageRef)
3015	if err != nil {
3016		log.WrapErr(err, "failed to get exposed ports for attachment, using defaults")
3017		return []int{}
3018	}
3019	return exposedPorts
3020}
3021
3022func (s *Service) attachmentVolumes(ctx context.Context, containerName, imageRef string) (map[string]string, error) {
3023	volumes, err := s.setupVolumes(ctx, containerName, imageRef)
3024	if err != nil {
3025		return nil, fmt.Errorf("failed to setup volumes for attachment %s with image %s: %w", containerName, imageRef, err)
3026	}
3027	return volumes, nil
3028}
3029
3030func (s *Service) attachmentEnv(ctx context.Context, containerName, imageRef string) ([]string, error) {
3031	envVars, err := s.loadEnvironment(ctx, nil, containerName, imageRef)
3032	if err != nil {
3033		return nil, fmt.Errorf("failed to load environment for attachment %s with image %s: %w", containerName, imageRef, err)
3034	}
3035	return envVars, nil
3036}
3037
3038func (s *Service) attachmentEnvDrifted(ctx context.Context, existing *domain.Container, containerName, serviceImage string) (bool, error) {
3039	imageRef, err := s.buildValidatedImageRef(ctx, serviceImage)
3040	if err != nil {
3041		return false, err
3042	}
3043	currentEnv, err := s.loadEnvironment(ctx, nil, containerName, imageRef)
3044	if err != nil {
3045		return false, err
3046	}
3047
3048	currentHash := hashEnvironment(currentEnv)
3049	existingHash, ok := existing.Labels[domain.LabelEnvHash]
3050	if ok && existingHash == currentHash {
3051		return false, nil
3052	}
3053
3054	return true, nil
3055}
3056
3057func (s *Service) handleRunningAttachment(ctx context.Context, existing *domain.Container, containerName, serviceImage string) (bool, error) {
3058	if existing.Status != string(domain.ContainerStatusRunning) {
3059		return false, nil
3060	}
3061
3062	log := zerowrap.FromCtx(ctx)
3063
3064	// Check image drift
3065	existingImage := existing.Labels[domain.LabelImage]
3066	imageDrifted := existingImage != "" && existingImage != serviceImage
3067
3068	// Check env drift
3069	envDrifted, err := s.attachmentEnvDrifted(ctx, existing, containerName, serviceImage)
3070	if err != nil {
3071		return false, log.WrapErr(err, "failed to check attachment env drift")
3072	}
3073
3074	if !envDrifted && !imageDrifted {
3075		log.Debug().Str("container_name", containerName).Msg("attachment already running with current env and image, skipping")
3076		return true, nil
3077	}
3078
3079	if imageDrifted {
3080		log.Info().Str("container_name", containerName).Str("existing_image", existingImage).Str("desired_image", serviceImage).Msg("attachment image changed, recreating")
3081	}
3082	if envDrifted {
3083		log.Info().Str("container_name", containerName).Msg("attachment env changed, recreating")
3084	}
3085
3086	if err := s.runtime.StopContainer(ctx, existing.ID); err != nil {
3087		return false, log.WrapErr(err, "failed to stop attachment for drift update")
3088	}
3089	if err := s.runtime.RemoveContainer(ctx, existing.ID, true); err != nil {
3090		return false, log.WrapErr(err, "failed to remove attachment for drift update")
3091	}
3092
3093	return false, nil
3094}
3095
3096func (s *Service) removeStoppedAttachment(ctx context.Context, existing *domain.Container, containerName string) error {
3097	if existing == nil {
3098		return nil
3099	}
3100
3101	log := zerowrap.FromCtx(ctx)
3102	log.Info().Str("container_name", containerName).Msg("removing stopped attachment container")
3103	if err := s.runtime.RemoveContainer(ctx, existing.ID, true); err != nil {
3104		return log.WrapErr(err, "failed to remove existing attachment container")
3105	}
3106
3107	return nil
3108}
3109
3110func (s *Service) resolveExistingAttachment(ctx context.Context, ownerDomain, serviceName string) (*domain.Container, error) {
3111	log := zerowrap.FromCtx(ctx)
3112	containerName := fmt.Sprintf("gordon-%s-%s", sanitizeName(ownerDomain), serviceName)
3113	existingContainer := s.findContainerByName(ctx, containerName)
3114	if existingContainer != nil {
3115		return existingContainer, nil
3116	}
3117
3118	containerNameLegacy := fmt.Sprintf("gordon-%s-%s", sanitizeNameLegacy(ownerDomain), serviceName)
3119	existingContainer = s.findContainerByName(ctx, containerNameLegacy)
3120	if existingContainer == nil {
3121		return nil, nil
3122	}
3123
3124	log.Info().Str("container_name", containerNameLegacy).Msg("found attachment with legacy naming, will be replaced")
3125	if err := s.runtime.StopContainer(ctx, existingContainer.ID); err != nil {
3126		return nil, log.WrapErr(err, "failed to stop legacy attachment container")
3127	}
3128	if err := s.runtime.RemoveContainer(ctx, existingContainer.ID, true); err != nil {
3129		return nil, log.WrapErr(err, "failed to remove legacy attachment container")
3130	}
3131
3132	return nil, nil
3133}
3134
3135// Utility functions
3136
3137func normalizeImageRef(image string) string {
3138	if isDigestRef(image) {
3139		return image
3140	}
3141
3142	// Extract repository from image reference, handling port numbers correctly.
3143	// Examples:
3144	//   nginx:latest -> docker.io/library/nginx
3145	//   user/repo:tag -> docker.io/user/repo
3146	//   localhost:5000/image:tag -> localhost:5000/image
3147	//   registry.example.com/image:tag -> registry.example.com/image
3148	//
3149	// Note: This function assumes valid Docker image references. Edge cases like
3150	// "registry.com:5000" (registry with port but no image name) are not valid
3151	// image references per Docker's naming conventions, which require at least
3152	// one path component after the registry (e.g., "registry.com:5000/image").
3153
3154	// Find the tag separator: the last colon that isn't part of a port number.
3155	// A colon is part of a port if there's no slash after it until the next colon.
3156	repo := image
3157	lastColon := strings.LastIndex(image, ":")
3158	if lastColon != -1 {
3159		afterColon := image[lastColon+1:]
3160		// If there's no slash after the colon, it's the tag separator
3161		if !strings.Contains(afterColon, "/") {
3162			repo = image[:lastColon]
3163		}
3164	}
3165
3166	if !strings.Contains(repo, "/") {
3167		return "docker.io/library/" + repo
3168	}
3169
3170	if strings.Count(repo, "/") == 1 && !strings.Contains(strings.Split(repo, "/")[0], ".") && !strings.Contains(strings.Split(repo, "/")[0], ":") {
3171		return "docker.io/" + repo
3172	}
3173
3174	return repo
3175}
3176
3177func generateVolumeName(prefix, domainName, volumePath string) string {
3178	return fmt.Sprintf("%s-%s-%s",
3179		prefix,
3180		strings.ReplaceAll(domainName, ".", "-"),
3181		strings.ReplaceAll(strings.Trim(volumePath, "/"), "/", "-"))
3182}
3183
3184func mergeEnvironmentVariables(dockerfileEnv, userEnv []string) []string {
3185	envMap := make(map[string]string)
3186
3187	for _, env := range dockerfileEnv {
3188		if k, v, ok := strings.Cut(env, "="); ok {
3189			envMap[k] = v
3190		}
3191	}
3192
3193	for _, env := range userEnv {
3194		if k, v, ok := strings.Cut(env, "="); ok {
3195			envMap[k] = v
3196		}
3197	}
3198
3199	keys := make([]string, 0, len(envMap))
3200	for k := range envMap {
3201		keys = append(keys, k)
3202	}
3203	slices.Sort(keys)
3204
3205	result := make([]string, 0, len(envMap))
3206	for _, k := range keys {
3207		result = append(result, k+"="+envMap[k])
3208	}
3209
3210	return result
3211}
3212
3213// extractServiceName gets service name from image reference.
3214// "my-postgres:latest" → "postgres", "redis:7" → "redis"
3215func extractServiceName(image string) string {
3216	// Remove tag
3217	parts := strings.Split(image, ":")
3218	name := parts[0]
3219
3220	// Remove registry prefix if present
3221	if strings.Contains(name, "/") {
3222		nameParts := strings.Split(name, "/")
3223		name = nameParts[len(nameParts)-1]
3224	}
3225
3226	// Remove common prefixes like "my-"
3227	name = strings.TrimPrefix(name, "my-")
3228
3229	return name
3230}
3231
3232// sanitizeName is a convenience wrapper around domain.SanitizeDomainForContainer.
3233func sanitizeName(d string) string {
3234	return domain.SanitizeDomainForContainer(d)
3235}
3236
3237// sanitizeNameLegacy is a convenience wrapper around domain.SanitizeDomainForContainerLegacy.
3238func sanitizeNameLegacy(d string) string {
3239	return domain.SanitizeDomainForContainerLegacy(d)
3240}
3241
3242func isContainerNotFoundError(err error) bool {
3243	if err == nil {
3244		return false
3245	}
3246	if errors.Is(err, domain.ErrContainerNotFound) {
3247		return true
3248	}
3249	msg := strings.ToLower(err.Error())
3250	return strings.Contains(msg, "no such container") ||
3251		strings.Contains(msg, "no container with name or id") ||
3252		strings.Contains(msg, "container not found")
3253}
3254
3255func isConnectionRefusedError(err error) bool {
3256	if err == nil {
3257		return false
3258	}
3259	return strings.Contains(strings.ToLower(err.Error()), "connection refused")
3260}
3261
3262// findContainerByName finds a container by its name.
3263func (s *Service) findContainerByName(ctx context.Context, name string) *domain.Container {
3264	containers, err := s.runtime.ListContainers(ctx, true)
3265	if err != nil {
3266		return nil
3267	}
3268
3269	for _, c := range containers {
3270		if c.Name == name {
3271			return c
3272		}
3273	}
3274	return nil
3275}
3276
3277func (s *Service) waitForReady(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig) error {
3278	if err := s.pollContainerRunning(ctx, containerID); err != nil {
3279		return err
3280	}
3281
3282	s.mu.RLock()
3283	cfg := s.config
3284	s.mu.RUnlock()
3285
3286	readinessMode := cfg.ReadinessMode
3287	if readinessMode == "" {
3288		readinessMode = "delay"
3289	}
3290
3291	switch readinessMode {
3292	case "delay":
3293		return s.waitForReadyByDelay(ctx, containerID)
3294	case "docker-health":
3295		_, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(ctx, containerID)
3296		if err != nil {
3297			return err
3298		}
3299		if !hasHealthcheck {
3300			return errors.New("no healthcheck detected")
3301		}
3302		return s.waitForHealthy(ctx, containerID, cfg.HealthTimeout)
3303	default: // auto
3304		return s.readinessCascade(ctx, containerID, containerConfig, cfg)
3305	}
3306}
3307
3308// readinessCascade auto-detects the strongest available readiness signal:
3309//  1. Docker healthcheck (if present) → wait for healthy status
3310//  2. HTTP probe (if gordon.health label set) → GET until 2xx/3xx
3311//  3. Default HTTP probe (GET / on exposed port) → wait for 2xx/3xx
3312//  4. TCP probe (if port info available) → connect until accepted
3313//  5. Delay fallback (last resort) → waitForReadyByDelay
3314func (s *Service) readinessCascade(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) error {
3315	log := zerowrap.FromCtx(ctx)
3316
3317	// 1. Docker healthcheck
3318	_, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(ctx, containerID)
3319	if err != nil {
3320		return err
3321	}
3322	if hasHealthcheck {
3323		log.Info().Msg("readiness cascade: using Docker healthcheck")
3324		return s.waitForHealthy(ctx, containerID, cfg.HealthTimeout)
3325	}
3326
3327	// 2. HTTP probe via gordon.health label
3328	if probed, probeErr := s.tryHTTPProbe(ctx, containerID, containerConfig, cfg); probed {
3329		return probeErr
3330	}
3331
3332	// 3. Default HTTP probe on root path (if port info available)
3333	if probed, probeErr := s.tryDefaultHTTPProbe(ctx, containerID, containerConfig, cfg); probed {
3334		return probeErr
3335	}
3336
3337	// 4. TCP probe (if port info available)
3338	if probed, probeErr := s.tryTCPProbe(ctx, containerID, containerConfig, cfg); probed {
3339		return probeErr
3340	}
3341
3342	// 5. Delay fallback
3343	log.Info().Msg("readiness cascade: using delay fallback")
3344	return s.waitForReadyByDelay(ctx, containerID)
3345}
3346
3347// waitForAttachmentReady runs a reduced readiness cascade for attachments:
3348// Docker healthcheck → TCP probe → delay fallback.
3349// HTTP probes are skipped — attachments are typically databases/caches.
3350func (s *Service) waitForAttachmentReady(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig) error {
3351	if err := s.pollContainerRunning(ctx, containerID); err != nil {
3352		return err
3353	}
3354
3355	log := zerowrap.FromCtx(ctx)
3356
3357	s.mu.RLock()
3358	cfg := s.config
3359	s.mu.RUnlock()
3360
3361	// 1. Docker healthcheck (if present)
3362	_, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(ctx, containerID)
3363	if err != nil {
3364		return err
3365	}
3366	if hasHealthcheck {
3367		log.Info().Msg("attachment readiness: using Docker healthcheck")
3368		timeout := cfg.HealthTimeout
3369		if timeout == 0 {
3370			timeout = 90 * time.Second
3371		}
3372		return s.waitForHealthy(ctx, containerID, timeout)
3373	}
3374
3375	// 2. TCP probe (if port info available)
3376	if containerConfig != nil && len(containerConfig.Ports) > 0 {
3377		ip, port, probeErr := s.resolveProbeEndpoint(ctx, containerID, containerConfig)
3378		if probeErr == nil && ip != "" && port > 0 {
3379			addr := fmt.Sprintf("%s:%d", ip, port)
3380			timeout := cfg.AttachmentReadinessTimeout
3381			if timeout == 0 {
3382				timeout = 30 * time.Second
3383			}
3384			log.Info().Str("addr", addr).Dur("timeout", timeout).Msg("attachment readiness: using TCP probe")
3385			return tcpProbe(ctx, addr, timeout)
3386		}
3387		log.Debug().Err(probeErr).Msg("attachment readiness: TCP probe skipped, could not resolve endpoint")
3388	}
3389
3390	// 3. Delay fallback
3391	log.Info().Msg("attachment readiness: using delay fallback")
3392	return s.waitForReadyByDelay(ctx, containerID)
3393}
3394
3395// tryHTTPProbe attempts an HTTP probe if the gordon.health label is set.
3396// Returns (true, err) if the probe was attempted, (false, nil) if skipped.
3397func (s *Service) tryHTTPProbe(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) (bool, error) {
3398	log := zerowrap.FromCtx(ctx)
3399	if containerConfig == nil {
3400		return false, nil
3401	}
3402	healthPath, ok := containerConfig.Labels[domain.LabelHealth]
3403	if !ok || healthPath == "" {
3404		return false, nil
3405	}
3406	ip, port, probeErr := s.resolveProbeEndpoint(ctx, containerID, containerConfig)
3407	if probeErr != nil || ip == "" || port <= 0 {
3408		log.Debug().Err(probeErr).Msg("readiness cascade: HTTP probe skipped, could not resolve container endpoint")
3409		return false, nil
3410	}
3411	url := fmt.Sprintf("http://%s:%d%s", ip, port, healthPath)
3412	timeout := cfg.HTTPProbeTimeout
3413	if timeout == 0 {
3414		timeout = 60 * time.Second
3415	}
3416	log.Info().Str("url", url).Dur("timeout", timeout).Msg("readiness cascade: using HTTP probe")
3417	return true, httpProbe(ctx, url, timeout)
3418}
3419
3420// tryDefaultHTTPProbe attempts an HTTP GET on "/" using the container's
3421// exposed port. This is the default readiness check for web containers —
3422// it verifies the app is actually serving HTTP, not just accepting TCP.
3423// Returns (true, err) if the probe was attempted, (false, nil) if skipped.
3424func (s *Service) tryDefaultHTTPProbe(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) (bool, error) {
3425	log := zerowrap.FromCtx(ctx)
3426	if containerConfig == nil || len(containerConfig.Ports) == 0 {
3427		return false, nil
3428	}
3429	ip, port, probeErr := s.resolveProbeEndpoint(ctx, containerID, containerConfig)
3430	if probeErr != nil || ip == "" || port <= 0 {
3431		log.Debug().Err(probeErr).Msg("readiness cascade: default HTTP probe skipped, could not resolve container endpoint")
3432		return false, nil
3433	}
3434	url := fmt.Sprintf("http://%s:%d/", ip, port)
3435	timeout := cfg.HTTPProbeTimeout
3436	if timeout == 0 {
3437		timeout = 60 * time.Second
3438	}
3439	log.Info().Str("url", url).Dur("timeout", timeout).Msg("readiness cascade: using default HTTP alive probe")
3440	return true, httpAliveProbe(ctx, url, timeout)
3441}
3442
3443// tryTCPProbe attempts a TCP probe if port info is available.
3444// Returns (true, err) if the probe was attempted, (false, nil) if skipped.
3445func (s *Service) tryTCPProbe(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) (bool, error) {
3446	log := zerowrap.FromCtx(ctx)
3447	if containerConfig == nil || len(containerConfig.Ports) == 0 {
3448		return false, nil
3449	}
3450	ip, port, probeErr := s.resolveProbeEndpoint(ctx, containerID, containerConfig)
3451	if probeErr != nil || ip == "" || port <= 0 {
3452		log.Debug().Err(probeErr).Msg("readiness cascade: TCP probe skipped, could not resolve container endpoint")
3453		return false, nil
3454	}
3455	addr := fmt.Sprintf("%s:%d", ip, port)
3456	timeout := cfg.TCPProbeTimeout
3457	if timeout == 0 {
3458		timeout = 30 * time.Second
3459	}
3460	log.Info().Str("addr", addr).Dur("timeout", timeout).Msg("readiness cascade: using TCP probe")
3461	return true, tcpProbe(ctx, addr, timeout)
3462}
3463
3464// resolveContainerEndpoint returns a host-reachable address for probing.
3465// In rootless podman/Docker setups, container internal IPs are not routable
3466// from the host. We use the host port binding (127.0.0.1:<mapped_port>)
3467// which is always reachable. Falls back to the container's internal IP
3468// only if no host port mapping exists (e.g. host-network mode).
3469func (s *Service) resolveProbeEndpoint(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig) (string, int, error) {
3470	log := zerowrap.FromCtx(ctx)
3471
3472	internalPort := 0
3473	if containerConfig != nil {
3474		if labels := containerConfig.Labels; labels != nil {
3475			for _, key := range []string{domain.LabelProxyPort, domain.LabelPort} {
3476				if portStr, ok := labels[key]; ok && portStr != "" {
3477					port, err := strconv.Atoi(portStr)
3478					if err == nil && port > 0 && port <= 65535 {
3479						internalPort = port
3480						break
3481					}
3482					log.Warn().Str("label", key).Str("port_value", portStr).Msg("invalid probe port label value")
3483				}
3484			}
3485		}
3486	}
3487
3488	if internalPort == 0 {
3489		_, resolvedPort, err := s.runtime.GetContainerNetworkInfo(ctx, containerID)
3490		if err != nil {
3491			return "", 0, err
3492		}
3493		internalPort = resolvedPort
3494	}
3495
3496	hostPort, hostErr := s.runtime.GetContainerPort(ctx, containerID, internalPort)
3497	if hostErr == nil && hostPort > 0 {
3498		log.Debug().
3499			Int("internal_port", internalPort).
3500			Int("host_port", hostPort).
3501			Msg("resolved container endpoint via host port binding")
3502		return "127.0.0.1", hostPort, nil
3503	}
3504
3505	ip, _, fallbackErr := s.runtime.GetContainerNetworkInfo(ctx, containerID)
3506	if fallbackErr != nil {
3507		return "", 0, fallbackErr
3508	}
3509	log.Debug().
3510		Str("ip", ip).
3511		Int("port", internalPort).
3512		Msg("resolved container endpoint via internal IP (no host port binding)")
3513	return ip, internalPort, nil
3514}
3515
3516// waitForReadyByDelay waits using the legacy running+delay strategy.
3517func (s *Service) waitForReadyByDelay(ctx context.Context, containerID string) error {
3518	log := zerowrap.FromCtx(ctx)
3519
3520	s.mu.RLock()
3521	cfg := s.config
3522	s.mu.RUnlock()
3523
3524	delay := cfg.ReadinessDelay
3525	if delay == 0 {
3526		delay = 5 * time.Second
3527	}
3528
3529	log.Debug().Dur("delay", delay).Msg("waiting for container readiness")
3530	select {
3531	case <-time.After(delay):
3532	case <-ctx.Done():
3533		return ctx.Err()
3534	}
3535
3536	// Verify still running after delay
3537	running, err := s.runtime.IsContainerRunning(ctx, containerID)
3538	if err != nil {
3539		return err
3540	}
3541	if running {
3542		return nil
3543	}
3544
3545	return s.waitForRecovery(ctx, containerID)
3546}
3547
3548func (s *Service) waitForHealthy(ctx context.Context, containerID string, timeout time.Duration) error {
3549	if timeout == 0 {
3550		timeout = 90 * time.Second
3551	}
3552	waitCtx, cancel := context.WithTimeout(ctx, timeout)
3553	defer cancel()
3554
3555	var lastStatus string
3556	for {
3557		status, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(waitCtx, containerID)
3558		if err != nil {
3559			return err
3560		}
3561		if !hasHealthcheck {
3562			return errors.New("no healthcheck detected")
3563		}
3564		// Treat empty health status as transitional startup. Some runtimes can
3565		// temporarily report an empty status before first probe results.
3566		if status == "" {
3567			status = "starting"
3568		}
3569		if status == "healthy" {
3570			return nil
3571		}
3572		if status == "unhealthy" {
3573			return errors.New("container reported unhealthy")
3574		}
3575		lastStatus = status
3576
3577		select {
3578		case <-time.After(time.Second):
3579		case <-waitCtx.Done():
3580			return fmt.Errorf("container healthcheck timeout after %s (last status: %s)", timeout, lastStatus)
3581		}
3582	}
3583}
3584
3585// pollContainerRunning polls until the container is running (max 30 seconds).
3586func (s *Service) pollContainerRunning(ctx context.Context, containerID string) error {
3587	for i := 0; i < 30; i++ {
3588		running, err := s.runtime.IsContainerRunning(ctx, containerID)
3589		if err != nil {
3590			return err
3591		}
3592		if running {
3593			return nil
3594		}
3595		// Check if the container has already exited (crash) rather than still starting up.
3596		if info, inspectErr := s.runtime.InspectContainer(ctx, containerID); inspectErr == nil && info != nil && info.Status == string(domain.ContainerStatusExited) {
3597			return fmt.Errorf("container exited immediately with code %d: %w", info.ExitCode, domain.ErrContainerExited)
3598		}
3599		if i == 29 {
3600			return fmt.Errorf("container did not start within 30 seconds")
3601		}
3602		select {
3603		case <-time.After(time.Second):
3604		case <-ctx.Done():
3605			return ctx.Err()
3606		}
3607	}
3608	return fmt.Errorf("container did not start within 30 seconds")
3609}
3610
3611// waitForRecovery waits for a container to recover within the recovery window.
3612func (s *Service) waitForRecovery(ctx context.Context, containerID string) error {
3613	log := zerowrap.FromCtx(ctx)
3614	log.Warn().
3615		Dur("recovery_window", readinessRecoveryWindow).
3616		Msg("container not running after readiness delay, waiting for recovery")
3617
3618	deadline := time.Now().Add(readinessRecoveryWindow)
3619	for time.Now().Before(deadline) {
3620		select {
3621		case <-time.After(time.Second):
3622		case <-ctx.Done():
3623			return ctx.Err()
3624		}
3625
3626		running, err := s.runtime.IsContainerRunning(ctx, containerID)
3627		if err != nil {
3628			return err
3629		}
3630		if running {
3631			log.Info().Msg("container recovered during readiness recovery window")
3632			return nil
3633		}
3634	}
3635
3636	return fmt.Errorf("container not running after readiness delay and recovery window")
3637}
3638
3639// publishContainerDeployed publishes a container.deployed event.
3640func (s *Service) publishContainerDeployed(ctx context.Context, domainName, containerID string) {
3641	payload := &domain.ContainerEventPayload{
3642		ContainerID: containerID,
3643		Domain:      domainName,
3644		Action:      "deployed",
3645	}
3646
3647	if err := s.eventBus.Publish(domain.EventContainerDeployed, payload); err != nil {
3648		log := zerowrap.FromCtx(ctx)
3649		log.Warn().Err(err).Msg("failed to publish container deployed event")
3650	}
3651}
3652
3653// rewriteToRegistryDomain rewrites an image reference to use the configured registry domain.
3654// e.g., "myapp:latest" -> "registry.example.com/myapp:latest"
3655func rewriteToRegistryDomain(imageRef, registryDomain string) string {
3656	if registryDomain == "" {
3657		return imageRef
3658	}
3659
3660	prefix := registryDomain + "/"
3661	if strings.HasPrefix(imageRef, prefix) {
3662		return imageRef
3663	}
3664
3665	return prefix + imageRef
3666}
3667
3668// rewriteToLocalRegistry rewrites Gordon-managed image references to use the
3669// local registry address. Current and legacy Gordon registry domains are
3670// stripped before prefixing localhost:<registryPort>/. Bare refs are prefixed.
3671// External registries are preserved unchanged.
3672func rewriteToLocalRegistry(imageRef, registryDomain string, legacyRegistryDomains []string, registryPort int) string {
3673	if imageRef == "" {
3674		return imageRef
3675	}
3676
3677	localRegistry := fmt.Sprintf("localhost:%d", registryPort)
3678	localPrefix := localRegistry + "/"
3679	if strings.HasPrefix(imageRef, localPrefix) {
3680		return imageRef
3681	}
3682
3683	if domain.IsGordonRegistryImageRef(imageRef, registryDomain, legacyRegistryDomains) {
3684		return localPrefix + domain.StripKnownGordonRegistry(imageRef, registryDomain, legacyRegistryDomains)
3685	}
3686	if hasExplicitRegistry(imageRef) {
3687		return imageRef
3688	}
3689
3690	return localPrefix + imageRef
3691}