service.go RawHistoryFolder

1// Package container implements the container management use case.
2package container
3
4import (
5	"context"
6	"errors"
7	"fmt"
8	"maps"
9	"slices"
10	"strconv"
11	"strings"
12	"sync"
13	"time"
14
15	"github.com/bnema/zerowrap"
16	"go.opentelemetry.io/otel"
17	"go.opentelemetry.io/otel/attribute"
18	"go.opentelemetry.io/otel/codes"
19	"go.opentelemetry.io/otel/metric"
20	"go.opentelemetry.io/otel/trace"
21
22	"github.com/bnema/gordon/internal/adapters/out/telemetry"
23	"github.com/bnema/gordon/internal/boundaries/out"
24	"github.com/bnema/gordon/internal/domain"
25)
26
27// Config holds configuration needed by the container service.
28type Config struct {
29	RegistryAuthEnabled      bool
30	RegistryDomain           string
31	RegistryPort             int
32	ServiceTokenUsername     string
33	ServiceToken             string
34	InternalRegistryUsername string
35	InternalRegistryPassword string
36	PullPolicy               string
37	VolumeAutoCreate         bool
38	VolumePrefix             string
39	VolumePreserve           bool
40	NetworkIsolation         bool
41	NetworkPrefix            string
42	NetworkGroups            map[string][]string
43	Attachments              map[string][]string
44	ReadinessDelay           time.Duration // Delay after container starts before considering it ready
45	ReadinessMode            string        // Readiness strategy: auto, docker-health, delay
46	HealthTimeout            time.Duration // Max wait for health-based readiness
47	DrainDelay               time.Duration // Grace period after cache invalidation before stopping old container
48	DrainDelayConfigured     bool          // True when deploy.drain_delay was explicitly configured
49	DrainMode                string        // Drain strategy: auto, inflight, delay
50	DrainTimeout             time.Duration // Max wait for in-flight requests to drain
51	StabilizationDelay       time.Duration // Post-switch monitoring window (default 2s)
52	TCPProbeTimeout          time.Duration // TCP probe timeout (default 30s)
53	HTTPProbeTimeout         time.Duration // HTTP probe timeout (default 60s)
54}
55
56var tracer = otel.Tracer("gordon.container")
57
58const (
59	PullPolicyAlways       = "always"
60	PullPolicyIfNotPresent = "if-not-present"
61	PullPolicyIfTagChanged = "if-tag-changed"
62
63	// readinessRecoveryWindow is an additional grace window used when a container
64	// is briefly not running at the end of readiness delay. This avoids false
65	// negatives during short startup flaps.
66	readinessRecoveryWindow = 30 * time.Second
67	internalPullMaxAttempts = 3
68)
69
70// Service implements the ContainerService interface.
71type Service struct {
72	runtime          out.ContainerRuntime
73	envLoader        out.EnvLoader
74	eventBus         out.EventPublisher
75	logWriter        out.ContainerLogWriter
76	cacheInvalidator out.ProxyCacheInvalidator
77	drainWaiter      out.ProxyDrainWaiter
78	config           Config
79	configProvider   AttachmentConfigProvider // live config reads for attachments/networks (may be nil)
80	metrics          *telemetry.Metrics
81	containers       map[string]*domain.Container
82	attachments      map[string][]string // ownerDomain → []containerIDs
83	managedCount     int64               // tracks UpDownCounter value for delta computation
84	mu               sync.RWMutex
85	deployMu         sync.Map       // per-domain deploy locks (domain → *domainDeployLock)
86	cleanupWg        sync.WaitGroup // tracks background old-container cleanup goroutines
87	monitor          *Monitor
88}
89
90// domainDeployLock is a context-aware mutex using a buffered channel.
91// It allows Lock() to be interrupted by context cancellation.
92type domainDeployLock struct {
93	ch chan struct{}
94}
95
96// newDomainDeployLock creates a new lock that is initially unlocked.
97func newDomainDeployLock() *domainDeployLock {
98	l := &domainDeployLock{ch: make(chan struct{}, 1)}
99	l.ch <- struct{}{}
100	return l
101}
102
103// Lock acquires the lock or returns an error if the context is cancelled.
104func (l *domainDeployLock) Lock(ctx context.Context) error {
105	// Fast path: honor already-canceled contexts before blocking.
106	if err := ctx.Err(); err != nil {
107		return err
108	}
109	select {
110	case <-ctx.Done():
111		return ctx.Err()
112	case <-l.ch:
113		// Re-check context after acquisition: if canceled while we were
114		// in the select, release the token so the lock isn't consumed.
115		if err := ctx.Err(); err != nil {
116			l.ch <- struct{}{}
117			return err
118		}
119		return nil
120	}
121}
122
123// Unlock releases the lock.
124func (l *domainDeployLock) Unlock() {
125	select {
126	case l.ch <- struct{}{}:
127	default:
128		// This should never happen if Lock/Unlock are used correctly.
129		// Panic to detect double-unlocks or other misuse early.
130		panic("domainDeployLock: double unlock detected")
131	}
132}
133
134// NewService creates a new container service.
135func NewService(
136	runtime out.ContainerRuntime,
137	envLoader out.EnvLoader,
138	eventBus out.EventPublisher,
139	logWriter out.ContainerLogWriter,
140	config Config,
141	configProvider AttachmentConfigProvider,
142) *Service {
143	return &Service{
144		runtime:        runtime,
145		envLoader:      envLoader,
146		eventBus:       eventBus,
147		logWriter:      logWriter,
148		config:         config,
149		configProvider: configProvider,
150		containers:     make(map[string]*domain.Container),
151		attachments:    make(map[string][]string),
152	}
153}
154
155// SetMetrics sets the telemetry metrics for the container service.
156func (s *Service) SetMetrics(m *telemetry.Metrics) {
157	s.metrics = m
158}
159
160// SetProxyCacheInvalidator sets the proxy cache invalidator for synchronous
161// cache invalidation during zero-downtime deployments. This must be called
162// after construction because the proxy service is created after the container
163// service during application initialization.
164func (s *Service) SetProxyCacheInvalidator(inv out.ProxyCacheInvalidator) {
165	s.mu.Lock()
166	s.cacheInvalidator = inv
167	s.mu.Unlock()
168}
169
170// SetProxyDrainWaiter sets the proxy in-flight drain waiter used during
171// zero-downtime replacement before stopping the old container.
172func (s *Service) SetProxyDrainWaiter(waiter out.ProxyDrainWaiter) {
173	s.mu.Lock()
174	s.drainWaiter = waiter
175	s.mu.Unlock()
176}
177
178// domainDeployMu returns a per-domain lock for serializing deploy operations.
179func (s *Service) domainDeployMu(domain string) *domainDeployLock {
180	v, _ := s.deployMu.LoadOrStore(domain, newDomainDeployLock())
181	// Type assertion is safe: LoadOrStore always stores *domainDeployLock values,
182	// and any concurrent LoadOrStore calls will also store *domainDeployLock.
183	return v.(*domainDeployLock)
184}
185
186// acquireDomainDeployLock acquires the per-domain deploy lock with context cancellation support.
187// Returns an unlock function that must be called to release the lock.
188func (s *Service) acquireDomainDeployLock(ctx context.Context, domain string) (func(), error) {
189	lock := s.domainDeployMu(domain)
190	if err := lock.Lock(ctx); err != nil {
191		return nil, err
192	}
193	return func() { lock.Unlock() }, nil
194}
195
196// buildContainerConfig constructs the container configuration for deployment.
197func (s *Service) deploymentContainerName(containerDomain string, existing *domain.Container) string {
198	canonicalName := fmt.Sprintf("gordon-%s", containerDomain)
199	if existing == nil {
200		return canonicalName
201	}
202
203	// Alternate temporary names if the tracked container was left with a temp suffix.
204	// This prevents name collisions after interrupted zero-downtime deploys.
205	newName := canonicalName + "-new"
206	nextName := canonicalName + "-next"
207	switch existing.Name {
208	case newName:
209		return nextName
210	case nextName:
211		return newName
212	default:
213		return newName
214	}
215}
216
217// containerConfigInput groups the parameters for building a container configuration.
218type containerConfigInput struct {
219	Domain       string
220	Image        string
221	ImageRef     string
222	ExposedPorts []int
223	EnvVars      []string
224	EnvHash      string
225	Volumes      map[string]string
226	NetworkName  string
227	ImageLabels  map[string]string
228	Existing     *domain.Container
229}
230
231func (s *Service) buildContainerConfig(in containerConfigInput) *domain.ContainerConfig {
232	containerName := s.deploymentContainerName(in.Domain, in.Existing)
233	labels := map[string]string{
234		domain.LabelDomain:  in.Domain,
235		domain.LabelEnvHash: in.EnvHash,
236		domain.LabelImage:   in.Image,
237		domain.LabelManaged: "true",
238		domain.LabelRoute:   in.Domain,
239	}
240
241	// Propagate proxy/port labels from image so readiness probes can find them
242	for _, key := range []string{domain.LabelProxyPort, domain.LabelPort, domain.LabelHealth} {
243		if v, ok := in.ImageLabels[key]; ok && v != "" {
244			labels[key] = v
245		}
246	}
247
248	// Ensure the label-specified proxy port is included in exposed ports
249	// so Docker creates a host port binding for it.
250	ports := in.ExposedPorts
251	if portStr, ok := labels[domain.LabelProxyPort]; ok {
252		if port, err := strconv.Atoi(portStr); err == nil && port > 0 && port <= 65535 {
253			if !slices.Contains(ports, port) {
254				ports = append(ports, port)
255			}
256		}
257	}
258
259	return &domain.ContainerConfig{
260		Image:       in.ImageRef,
261		Name:        containerName,
262		Ports:       ports,
263		Env:         in.EnvVars,
264		Volumes:     in.Volumes,
265		NetworkMode: in.NetworkName,
266		Hostname:    in.Domain,
267		Labels:      labels,
268		AutoRemove:  false,
269	}
270}
271
272// Deploy creates and starts a container for the given route.
273// Implements zero-downtime deployment: new container starts before old one stops.
274func (s *Service) Deploy(ctx context.Context, route domain.Route) (*domain.Container, error) {
275	ctx, span := tracer.Start(ctx, "container.deploy",
276		trace.WithAttributes(
277			attribute.String("domain", route.Domain),
278			attribute.String("image", route.Image),
279		))
280	defer span.End()
281
282	deployStart := time.Now()
283
284	// Serialize deploys for the same domain to prevent race conditions
285	// (e.g. multiple image.pushed events + explicit deploy call from CLI).
286	unlock, err := s.acquireDomainDeployLock(ctx, route.Domain)
287	if err != nil {
288		return nil, err
289	}
290	defer unlock()
291
292	// Enrich context with use case fields for all downstream logs
293	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
294		zerowrap.FieldLayer:   "usecase",
295		zerowrap.FieldUseCase: "Deploy",
296		"domain":              route.Domain,
297	})
298	log := zerowrap.FromCtx(ctx)
299
300	// Record deploy metrics and trace status
301	defer s.recordDeployMetrics(ctx, span, route, deployStart, &err)
302
303	existing, hasExisting := s.resolveExistingContainer(ctx, route.Domain)
304
305	resources, err := s.prepareDeployResources(ctx, route, existing)
306	if err != nil {
307		return nil, err
308	}
309
310	// Skip redundant deploy: if the existing container is already running
311	// the exact same image (by Docker image ID), return it immediately.
312	// This prevents the double-deploy caused by the event-based deploy
313	// (triggered by image.pushed) racing with the explicit CLI deploy call.
314	if hasExisting {
315		existingForSkip := existing
316		if existingForSkip.ImageID == "" && existingForSkip.Image != "" && normalizeImageRef(existingForSkip.Image) == normalizeImageRef(route.Image) {
317			existingForSkip = s.containerForRedundantCheck(ctx, existingForSkip)
318		}
319		if existingForSkip.ImageID != "" {
320			if skip, container := s.skipRedundantDeploy(ctx, existingForSkip, resources.actualImageRef, resources.envHash); skip {
321				return container, nil
322			}
323		}
324	}
325
326	newContainer, err := s.createStartedContainer(ctx, route, existing, resources)
327	if err != nil {
328		return nil, err
329	}
330
331	invalidated := s.activateDeployedContainer(ctx, route.Domain, newContainer)
332
333	// Post-switch stabilization: verify new container stays running
334	if hasExisting {
335		stable, stabilizeErr := s.stabilizeNewContainer(ctx, route.Domain, newContainer, existing)
336		if stabilizeErr != nil {
337			// Both old and new containers are dead; assign to named return so
338			// the deferred recordDeployMetrics and span see the failure.
339			err = stabilizeErr
340			return nil, err
341		}
342		if !stable {
343			// Rollback performed — old container is restored
344			return existing, nil
345		}
346	}
347
348	// Finalize old container in the background — the new container is already
349	// serving traffic, so there's no reason to block the deploy response while
350	// waiting for the old container to stop (which can take 20s if the app
351	// doesn't handle SIGTERM).
352	s.cleanupWg.Add(1)
353	go func() {
354		defer s.cleanupWg.Done()
355		s.finalizePreviousContainer(context.WithoutCancel(ctx), route.Domain, existing, hasExisting, invalidated, newContainer.ID)
356	}()
357
358	// Start container log collection (non-blocking, errors don't fail deployment)
359	s.startLogCollection(ctx, newContainer.ID, route.Domain)
360
361	log.Info().
362		Str("image", route.Image).
363		Str(zerowrap.FieldEntityID, newContainer.ID).
364		Ints("ports", newContainer.Ports).
365		Str("network", resources.networkName).
366		Bool("zero_downtime", hasExisting).
367		Msg("container deployed successfully")
368
369	return newContainer, nil
370}
371
372// recordDeployMetrics records span error status and deploy metrics at the end of a Deploy call.
373// It is called via defer and receives a pointer to the named return error so it can observe
374// the final error value after all deferred functions have run.
375func (s *Service) recordDeployMetrics(ctx context.Context, span trace.Span, route domain.Route, start time.Time, errPtr *error) {
376	err := *errPtr
377	if err != nil {
378		span.RecordError(err)
379		span.SetStatus(codes.Error, err.Error())
380	}
381	if s.metrics == nil {
382		return
383	}
384	attrs := metric.WithAttributes(
385		attribute.String("domain", route.Domain),
386		attribute.String("image", route.Image),
387	)
388	s.metrics.DeployTotal.Add(ctx, 1, attrs)
389	s.metrics.DeployDuration.Record(ctx, time.Since(start).Seconds(), attrs)
390	if err != nil {
391		s.metrics.DeployErrors.Add(ctx, 1, attrs)
392	}
393}
394
395// skipRedundantDeploy checks whether the existing container is already running
396// the same image (by Docker image ID) as the one we are about to deploy.
397// When a push triggers both an event-based deploy and an explicit CLI deploy,
398// the second one arrives after the first has already completed; this avoids
399// a full redundant create-start-readiness cycle.
400func (s *Service) skipRedundantDeploy(ctx context.Context, existing *domain.Container, actualImageRef, envHash string) (bool, *domain.Container) {
401	log := zerowrap.FromCtx(ctx)
402
403	newImageID, err := s.runtime.GetImageID(ctx, actualImageRef)
404	if err != nil {
405		log.Debug().Err(err).Msg("cannot resolve image ID for redundancy check, proceeding with deploy")
406		return false, nil
407	}
408
409	if existing.ImageID != newImageID {
410		return false, nil
411	}
412
413	// Verify the container is actually still running before skipping.
414	running, err := s.runtime.IsContainerRunning(ctx, existing.ID)
415	if err != nil || !running {
416		return false, nil
417	}
418
419	if existing.Labels != nil {
420		existingEnvHash, hasEnvHash := existing.Labels[domain.LabelEnvHash]
421		if !hasEnvHash || existingEnvHash != envHash {
422			log.Info().
423				Str("container_id", existing.ID).
424				Bool("has_env_hash", hasEnvHash).
425				Msg("env changed, proceeding with deploy despite same image")
426			return false, nil
427		}
428	} else {
429		return false, nil
430	}
431
432	log.Info().
433		Str("container_id", existing.ID).
434		Str("image_id", newImageID).
435		Msg("skipping redundant deploy: container already running this image")
436
437	return true, existing
438}
439
440func (s *Service) containerForRedundantCheck(ctx context.Context, existing *domain.Container) *domain.Container {
441	if existing == nil || existing.ImageID != "" || existing.ID == "" {
442		return existing
443	}
444
445	log := zerowrap.FromCtx(ctx)
446	inspected, err := s.runtime.InspectContainer(ctx, existing.ID)
447	if err != nil {
448		log.Debug().Err(err).Str("container_id", existing.ID).Msg("cannot inspect existing container for redundancy check")
449		return existing
450	}
451	if inspected == nil || inspected.ImageID == "" {
452		return existing
453	}
454
455	existingCopy := *existing
456	existingCopy.ImageID = inspected.ImageID
457	return &existingCopy
458}
459
460type deployResources struct {
461	networkName    string
462	actualImageRef string
463	exposedPorts   []int
464	imageLabels    map[string]string
465	envVars        []string
466	envHash        string
467	volumes        map[string]string
468}
469
470// resolveExistingContainer returns the currently running container for a domain.
471// It first checks the in-memory map (fast path). If not found, it queries the
472// runtime for a running container with matching name and managed label. This
473// handles cases where Gordon restarted and in-memory state is stale.
474//
475// The slow path checks canonical, -new, and -next names because a previous
476// deploy may have been interrupted (e.g. eventbus timeout, Gordon restart)
477// leaving the active container under a temp name. Canonical is preferred when
478// multiple matches exist.
479func (s *Service) resolveExistingContainer(ctx context.Context, domainName string) (*domain.Container, bool) {
480	// Fast path: check in-memory state
481	s.mu.RLock()
482	container, ok := s.containers[domainName]
483	s.mu.RUnlock()
484	if ok {
485		return container, true
486	}
487
488	// Slow path: query runtime for running containers
489	log := zerowrap.FromCtx(ctx)
490	canonicalName := fmt.Sprintf("gordon-%s", domainName)
491	candidateNames := map[string]bool{
492		canonicalName:           true,
493		canonicalName + "-new":  true,
494		canonicalName + "-next": true,
495	}
496
497	running, err := s.runtime.ListContainers(ctx, false)
498	if err != nil {
499		log.Warn().Err(err).Msg("failed to list running containers for existing container resolution")
500		return nil, false
501	}
502
503	// Prefer canonical name; fall back to any running managed temp container.
504	var best *domain.Container
505	for _, c := range running {
506		if !candidateNames[c.Name] || c.Labels[domain.LabelManaged] != "true" {
507			continue
508		}
509		if best == nil || c.Name == canonicalName {
510			best = c
511		}
512	}
513
514	if best != nil {
515		log.Info().
516			Str("container_id", best.ID).
517			Str("container_name", best.Name).
518			Msg("resolved existing container from runtime (in-memory state was stale)")
519
520		// Update in-memory state so subsequent lookups are fast
521		s.mu.Lock()
522		if _, alreadyTracked := s.containers[domainName]; !alreadyTracked {
523			s.managedCount++
524		}
525		s.containers[domainName] = best
526		s.mu.Unlock()
527
528		return best, true
529	}
530
531	return nil, false
532}
533
534func (s *Service) prepareDeployResources(ctx context.Context, route domain.Route, existing *domain.Container) (*deployResources, error) {
535	log := zerowrap.FromCtx(ctx)
536
537	existingID := ""
538	if existing != nil {
539		existingID = existing.ID
540	}
541	if err := s.cleanupOrphanedContainers(ctx, route.Domain, existingID); err != nil {
542		log.WrapErr(err, "failed to cleanup orphaned containers")
543	}
544
545	imageRef := s.buildImageRef(route.Image)
546	actualImageRef, err := s.ensureImage(ctx, imageRef)
547	if err != nil {
548		return nil, err
549	}
550
551	networkName := s.getNetworkForApp(route.Domain)
552	if err := s.createNetworkIfNeeded(ctx, networkName); err != nil {
553		return nil, log.WrapErr(err, "failed to create network")
554	}
555	if err := s.deployAttachments(ctx, route.Domain, networkName); err != nil {
556		return nil, log.WrapErr(err, "failed to deploy attachments")
557	}
558
559	exposedPorts, err := s.runtime.GetImageExposedPorts(ctx, actualImageRef)
560	if err != nil {
561		log.WrapErr(err, "failed to get exposed ports, using defaults")
562		exposedPorts = []int{80, 8080, 3000}
563	}
564
565	imageLabels, err := s.runtime.GetImageLabels(ctx, actualImageRef)
566	if err != nil {
567		log.Warn().Err(err).Msg("failed to get image labels, skipping label propagation")
568		imageLabels = nil
569	}
570
571	envVars, err := s.loadEnvironment(ctx, route.Env, route.Domain, actualImageRef)
572	if err != nil {
573		return nil, err
574	}
575	envHash := hashEnvironment(envVars)
576
577	volumes, err := s.setupVolumes(ctx, route.Domain, actualImageRef)
578	if err != nil {
579		return nil, err
580	}
581
582	return &deployResources{
583		networkName:    networkName,
584		actualImageRef: actualImageRef,
585		exposedPorts:   exposedPorts,
586		imageLabels:    imageLabels,
587		envVars:        envVars,
588		envHash:        envHash,
589		volumes:        volumes,
590	}, nil
591}
592
593func (s *Service) createStartedContainer(ctx context.Context, route domain.Route, existing *domain.Container, resources *deployResources) (*domain.Container, error) {
594	ctx, span := tracer.Start(ctx, "container.create_and_start")
595	defer span.End()
596
597	log := zerowrap.FromCtx(ctx)
598
599	containerConfig := s.buildContainerConfig(containerConfigInput{
600		Domain:       route.Domain,
601		Image:        route.Image,
602		ImageRef:     resources.actualImageRef,
603		ExposedPorts: resources.exposedPorts,
604		EnvVars:      resources.envVars,
605		EnvHash:      resources.envHash,
606		Volumes:      resources.volumes,
607		NetworkName:  resources.networkName,
608		ImageLabels:  resources.imageLabels,
609		Existing:     existing,
610	})
611
612	newContainer, err := s.runtime.CreateContainer(ctx, containerConfig)
613	if err != nil {
614		return nil, log.WrapErr(err, "failed to create container")
615	}
616	if err := s.runtime.StartContainer(ctx, newContainer.ID); err != nil {
617		s.runtime.RemoveContainer(ctx, newContainer.ID, true)
618		return nil, log.WrapErr(err, "failed to start container")
619	}
620	if !domain.IsSkipReadiness(ctx) {
621		if err := s.waitForReady(ctx, newContainer.ID, containerConfig); err != nil {
622			s.cleanupFailedContainer(ctx, newContainer.ID)
623			return nil, log.WrapErr(err, "container failed readiness check")
624		}
625	}
626
627	inspected, err := s.runtime.InspectContainer(ctx, newContainer.ID)
628	if err != nil {
629		s.cleanupFailedContainer(ctx, newContainer.ID)
630		return nil, log.WrapErr(err, "failed to inspect started container")
631	}
632
633	return inspected, nil
634}
635
636func (s *Service) activateDeployedContainer(ctx context.Context, domainName string, container *domain.Container) bool {
637	s.mu.Lock()
638	_, wasTracked := s.containers[domainName]
639	s.containers[domainName] = container
640	if !wasTracked {
641		s.managedCount++
642	}
643	s.mu.Unlock()
644
645	// Track managed container count (only increment for new domains, not replacements)
646	if !wasTracked && s.metrics != nil {
647		s.metrics.ManagedContainers.Add(ctx, 1)
648	}
649
650	s.publishContainerDeployed(ctx, domainName, container.ID)
651
652	s.mu.RLock()
653	inv := s.cacheInvalidator
654	s.mu.RUnlock()
655	if inv != nil {
656		inv.InvalidateTarget(ctx, domainName)
657		return true
658	}
659
660	return false
661}
662
663// stabilizeNewContainer monitors the new container briefly after traffic switch.
664// If it crashes during this window, rolls back to old container.
665// Returns true if stabilization succeeded, false if rollback was performed.
666// Returns an error only when both new and old containers are dead.
667func (s *Service) stabilizeNewContainer(ctx context.Context, domainName string, newContainer, oldContainer *domain.Container) (bool, error) {
668	log := zerowrap.FromCtx(ctx)
669
670	if oldContainer == nil {
671		return true, nil
672	}
673
674	s.mu.RLock()
675	delay := s.config.StabilizationDelay
676	s.mu.RUnlock()
677	if delay == 0 {
678		delay = 2 * time.Second
679	}
680
681	// Brief stabilization: verify new container is still running after delay
682	select {
683	case <-time.After(delay):
684	case <-ctx.Done():
685		return true, nil
686	}
687
688	running, err := s.runtime.IsContainerRunning(ctx, newContainer.ID)
689	if err != nil || !running {
690		log.Error().
691			Str("new_container", newContainer.ID).
692			Str("old_container", oldContainer.ID).
693			Msg("new container crashed during stabilization, rolling back to old")
694
695		// Verify old container is still running before restoring it.
696		// If both old and new are dead (e.g., OOM), returning a dead
697		// container as "existing" would leave the domain in a broken state.
698		oldRunning, oldErr := s.runtime.IsContainerRunning(ctx, oldContainer.ID)
699		if oldErr != nil || !oldRunning {
700			log.Error().
701				Str("old_container", oldContainer.ID).
702				Msg("old container is also not running, cannot rollback")
703
704			// Cleanup failed new container
705			if stopErr := s.runtime.StopContainer(ctx, newContainer.ID); stopErr != nil {
706				log.WrapErrWithFields(stopErr, "failed to stop failed new container during rollback", map[string]any{zerowrap.FieldEntityID: newContainer.ID})
707			}
708			if removeErr := s.runtime.RemoveContainer(ctx, newContainer.ID, true); removeErr != nil {
709				log.WrapErrWithFields(removeErr, "failed to remove failed new container during rollback", map[string]any{zerowrap.FieldEntityID: newContainer.ID})
710			}
711
712			return false, fmt.Errorf("stabilization failed: new container crashed and old container %s is also not running", oldContainer.ID)
713		}
714
715		// Rollback: restore old container as tracked
716		s.mu.Lock()
717		s.containers[domainName] = oldContainer
718		s.mu.Unlock()
719
720		// Re-invalidate proxy cache to point back to old
721		s.mu.RLock()
722		inv := s.cacheInvalidator
723		s.mu.RUnlock()
724		if inv != nil {
725			inv.InvalidateTarget(ctx, domainName)
726		}
727
728		// Cleanup failed new container
729		if stopErr := s.runtime.StopContainer(ctx, newContainer.ID); stopErr != nil {
730			log.WrapErrWithFields(stopErr, "failed to stop failed new container during rollback", map[string]any{zerowrap.FieldEntityID: newContainer.ID})
731		}
732		if removeErr := s.runtime.RemoveContainer(ctx, newContainer.ID, true); removeErr != nil {
733			log.WrapErrWithFields(removeErr, "failed to remove failed new container during rollback", map[string]any{zerowrap.FieldEntityID: newContainer.ID})
734		}
735
736		return false, nil
737	}
738
739	return true, nil
740}
741
742func (s *Service) finalizePreviousContainer(ctx context.Context, domainName string, existing *domain.Container, hasExisting, invalidated bool, newContainerID string) {
743	if !hasExisting {
744		return
745	}
746
747	if invalidated {
748		s.waitForDrain(ctx, existing.ID)
749	}
750
751	s.cleanupOldContainer(ctx, existing, newContainerID, domainName)
752}
753
754func (s *Service) waitForDrain(ctx context.Context, oldContainerID string) {
755	log := zerowrap.FromCtx(ctx)
756
757	s.mu.RLock()
758	cfg := s.config
759	waiter := s.drainWaiter
760	s.mu.RUnlock()
761
762	drainMode := cfg.DrainMode
763	if drainMode == "" {
764		drainMode = "delay"
765	}
766
767	shouldUseInFlight := (drainMode == "inflight" || drainMode == "auto") && waiter != nil
768	if shouldUseInFlight {
769		timeout := cfg.DrainTimeout
770		if timeout == 0 {
771			timeout = 30 * time.Second
772		}
773		drained := waiter.WaitForNoInFlight(ctx, oldContainerID, timeout)
774		if !drained {
775			log.Warn().
776				Str("old_container_id", oldContainerID).
777				Dur("drain_timeout", timeout).
778				Msg("drain wait timed out; old container may still have in-flight traffic")
779		}
780		return
781	}
782
783	drainDelay := 2 * time.Second
784	if cfg.DrainDelayConfigured {
785		drainDelay = cfg.DrainDelay
786	}
787	if drainDelay <= 0 {
788		return
789	}
790	select {
791	case <-time.After(drainDelay):
792	case <-ctx.Done():
793	}
794}
795
796// restartContainerWithRecovery restarts a container, reconciling stale state if needed.
797// It returns the (possibly refreshed) container and updated attachment IDs.
798func (s *Service) restartContainerWithRecovery(ctx context.Context, domainName string, container *domain.Container, attachmentIDs []string) (*domain.Container, []string, error) {
799	log := zerowrap.FromCtx(ctx)
800
801	if err := s.runtime.RestartContainer(ctx, container.ID); err != nil {
802		if !isContainerNotFoundError(err) {
803			return nil, nil, log.WrapErr(err, "failed to restart container")
804		}
805
806		// In-memory container ID can become stale after external runtime changes.
807		// Re-sync and retry once with the latest tracked container.
808		log.Warn().
809			Err(err).
810			Str(zerowrap.FieldEntityID, container.ID).
811			Msg("tracked container missing during restart, attempting state reconciliation")
812
813		if syncErr := s.SyncContainers(ctx); syncErr != nil {
814			log.Warn().Err(syncErr).Msg("failed to sync container state during restart recovery")
815		}
816
817		s.mu.RLock()
818		refreshed, refreshedExists := s.containers[domainName]
819		attachmentIDs = append([]string{}, s.attachments[domainName]...)
820		s.mu.RUnlock()
821
822		if !refreshedExists || refreshed == nil {
823			return nil, nil, domain.ErrContainerNotFound
824		}
825		if err := s.runtime.RestartContainer(ctx, refreshed.ID); err != nil {
826			return nil, nil, log.WrapErr(err, "failed to restart container after state reconciliation")
827		}
828		return refreshed, attachmentIDs, nil
829	}
830
831	return container, attachmentIDs, nil
832}
833
834// Restart restarts a running container for the given domain.
835func (s *Service) Restart(ctx context.Context, domainName string, withAttachments bool) error {
836	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
837		zerowrap.FieldLayer:   "usecase",
838		zerowrap.FieldUseCase: "Restart",
839		"domain":              domainName,
840	})
841	log := zerowrap.FromCtx(ctx)
842
843	s.mu.RLock()
844	container, exists := s.containers[domainName]
845	attachments := s.attachments[domainName]
846	var attachmentIDs []string
847	if len(attachments) > 0 {
848		attachmentIDs = make([]string, len(attachments))
849		copy(attachmentIDs, attachments)
850	}
851	s.mu.RUnlock()
852
853	if !exists || container == nil {
854		return domain.ErrContainerNotFound
855	}
856
857	// When attachments are requested, sync runtime state first to get accurate
858	// attachment tracking, then check if configured attachments are deployed.
859	// If fewer attachment containers are tracked than configured, fail early with
860	// guidance rather than restarting the main container into a broken state
861	// (e.g., missing database hostname).
862	if withAttachments {
863		if syncErr := s.SyncContainers(ctx); syncErr != nil {
864			log.Warn().Err(syncErr).Msg("failed to sync container state before attachment check")
865		}
866
867		// Re-read attachment IDs after sync
868		s.mu.RLock()
869		if freshAttachments := s.attachments[domainName]; len(freshAttachments) > 0 {
870			attachmentIDs = make([]string, len(freshAttachments))
871			copy(attachmentIDs, freshAttachments)
872		} else {
873			attachmentIDs = nil
874		}
875		s.mu.RUnlock()
876
877		configuredAttachments := s.resolveAttachmentsForDomain(domainName)
878		if len(configuredAttachments) > 0 && len(attachmentIDs) < len(configuredAttachments) {
879			missing := len(configuredAttachments) - len(attachmentIDs)
880			return fmt.Errorf("%w: domain %q has %d configured attachment(s) but only %d deployed (%d missing)",
881				domain.ErrAttachmentNotDeployed, domainName, len(configuredAttachments), len(attachmentIDs), missing)
882		}
883	}
884
885	// Restart the main container (with stale-state recovery)
886	container, attachmentIDs, err := s.restartContainerWithRecovery(ctx, domainName, container, attachmentIDs)
887	if err != nil {
888		return err
889	}
890	log.Info().Str(zerowrap.FieldEntityID, container.ID).Msg("container restarted")
891
892	// Record restart metric
893	if s.metrics != nil {
894		s.metrics.ContainerRestarts.Add(ctx, 1, metric.WithAttributes(
895			attribute.String("domain", domainName),
896			attribute.String("source", "api"),
897		))
898	}
899
900	// Restart attachments if requested
901	if withAttachments && len(attachmentIDs) > 0 {
902		for _, attachID := range attachmentIDs {
903			if err := s.runtime.RestartContainer(ctx, attachID); err != nil {
904				log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to restart attachment")
905				continue // Don't fail the whole operation for one attachment
906			}
907			log.Info().Str("attachment_id", attachID).Msg("attachment restarted")
908		}
909	}
910
911	return nil
912}
913
914// Stop stops a running container.
915func (s *Service) Stop(ctx context.Context, containerID string) error {
916	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
917		zerowrap.FieldLayer:    "usecase",
918		zerowrap.FieldUseCase:  "Stop",
919		zerowrap.FieldEntityID: containerID,
920	})
921	log := zerowrap.FromCtx(ctx)
922
923	// Stop log collection before stopping container
924	if s.logWriter != nil {
925		if err := s.logWriter.StopLogging(containerID); err != nil {
926			log.Warn().Err(err).Msg("failed to stop container log collection")
927		}
928	}
929
930	if err := s.runtime.StopContainer(ctx, containerID); err != nil {
931		return log.WrapErr(err, "failed to stop container")
932	}
933
934	log.Info().Msg("container stopped")
935	return nil
936}
937
938// Remove removes a container.
939func (s *Service) Remove(ctx context.Context, containerID string, force bool) error {
940	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
941		zerowrap.FieldLayer:    "usecase",
942		zerowrap.FieldUseCase:  "Remove",
943		zerowrap.FieldEntityID: containerID,
944	})
945	log := zerowrap.FromCtx(ctx)
946
947	// Stop log collection before removing container
948	if s.logWriter != nil {
949		if err := s.logWriter.StopLogging(containerID); err != nil {
950			log.Warn().Err(err).Msg("failed to stop container log collection")
951		}
952	}
953
954	// Find domain and attachment IDs for cleanup
955	containerDomain, attachmentIDs, cfg := s.findContainerContext(containerID)
956
957	if err := s.runtime.RemoveContainer(ctx, containerID, force); err != nil {
958		return log.WrapErr(err, "failed to remove container")
959	}
960
961	// Cleanup volumes
962	if containerDomain != "" && !cfg.VolumePreserve {
963		if err := s.cleanupVolumesForDomain(ctx, containerDomain); err != nil {
964			log.WrapErrWithFields(err, "failed to cleanup volumes", map[string]any{"domain": containerDomain})
965		}
966	}
967
968	// Clean up attachment containers
969	s.removeAttachments(ctx, attachmentIDs)
970
971	// Remove from tracking
972	s.mu.Lock()
973	var removedDomain string
974	for d, c := range s.containers {
975		if c.ID == containerID {
976			delete(s.containers, d)
977			removedDomain = d
978			s.managedCount--
979			log.Info().Str("domain", d).Msg("container removed")
980			break
981		}
982	}
983	if removedDomain != "" {
984		delete(s.attachments, removedDomain)
985	}
986	s.mu.Unlock()
987
988	// Decrement managed container count
989	if removedDomain != "" && s.metrics != nil {
990		s.metrics.ManagedContainers.Add(ctx, -1)
991	}
992
993	// Note: We intentionally do NOT clean up deployMu entries to avoid a race condition:
994	// If Remove deletes the mutex entry while a concurrent Deploy is acquiring the lock,
995	// the Deploy would create a fresh mutex, breaking serialization and allowing concurrent
996	// deploys to the same domain. The memory footprint is acceptable (one small struct per
997	// domain ever deployed), and this is the safer choice for correctness.
998
999	// Cleanup network
1000	if removedDomain != "" && cfg.NetworkIsolation {
1001		networkName := s.getNetworkForApp(removedDomain)
1002		if err := s.cleanupNetworkIfEmpty(ctx, networkName); err != nil {
1003			log.WrapErrWithFields(err, "failed to cleanup network", map[string]any{"network": networkName})
1004		}
1005	}
1006
1007	return nil
1008}
1009
1010// findContainerContext looks up the domain, attachment IDs, and config snapshot for a container.
1011func (s *Service) findContainerContext(containerID string) (string, []string, Config) {
1012	s.mu.RLock()
1013	defer s.mu.RUnlock()
1014	cfg := s.config
1015	var containerDomain string
1016	for d, c := range s.containers {
1017		if c.ID == containerID {
1018			containerDomain = d
1019			break
1020		}
1021	}
1022	var attachmentIDs []string
1023	if containerDomain != "" {
1024		attachmentIDs = append(attachmentIDs, s.attachments[containerDomain]...)
1025	}
1026	return containerDomain, attachmentIDs, cfg
1027}
1028
1029// removeAttachments stops and removes attachment containers best-effort.
1030func (s *Service) removeAttachments(ctx context.Context, attachmentIDs []string) {
1031	log := zerowrap.FromCtx(ctx)
1032	for _, attachID := range attachmentIDs {
1033		if s.logWriter != nil {
1034			if err := s.logWriter.StopLogging(attachID); err != nil {
1035				log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to stop attachment log collection")
1036			}
1037		}
1038		if err := s.runtime.StopContainer(ctx, attachID); err != nil {
1039			log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to stop attachment container")
1040		}
1041		if err := s.runtime.RemoveContainer(ctx, attachID, true); err != nil {
1042			log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to remove attachment container")
1043		}
1044	}
1045}
1046
1047// Get retrieves a container by domain.
1048func (s *Service) Get(_ context.Context, domainName string) (*domain.Container, bool) {
1049	s.mu.RLock()
1050	defer s.mu.RUnlock()
1051	container, exists := s.containers[domainName]
1052	return container, exists
1053}
1054
1055// List returns all managed containers.
1056func (s *Service) List(_ context.Context) map[string]*domain.Container {
1057	s.mu.RLock()
1058	defer s.mu.RUnlock()
1059
1060	result := make(map[string]*domain.Container, len(s.containers))
1061	maps.Copy(result, s.containers)
1062	return result
1063}
1064
1065// ListRoutesWithDetails returns routes with network and attachment info.
1066// Note: Container map is copied under lock, then external calls are made without lock.
1067// If containers are removed between copy and runtime calls, errors are handled gracefully
1068// (network becomes empty string). This trade-off avoids holding locks during I/O.
1069func (s *Service) ListRoutesWithDetails(ctx context.Context) []domain.RouteInfo {
1070	s.mu.RLock()
1071	containers := make(map[string]*domain.Container, len(s.containers))
1072	maps.Copy(containers, s.containers)
1073	s.mu.RUnlock()
1074
1075	// Fetch all attachments once to avoid N+1 queries
1076	attachmentsByDomain := s.getAllAttachments(ctx)
1077
1078	results := make([]domain.RouteInfo, 0, len(containers))
1079	for domainName, container := range containers {
1080		network := ""
1081		image := ""
1082		containerID := ""
1083		status := ""
1084		if container != nil {
1085			containerID = container.ID
1086			status = container.Status
1087			image = container.Image
1088			if container.Labels != nil {
1089				if labelImage, ok := container.Labels[domain.LabelImage]; ok && labelImage != "" {
1090					image = labelImage
1091				}
1092			}
1093			// Strip registry domain prefix from image for cleaner display
1094			image = s.stripRegistryPrefix(image)
1095			if networkName, err := s.runtime.GetContainerNetwork(ctx, container.ID); err == nil {
1096				network = networkName
1097			}
1098		}
1099
1100		results = append(results, domain.RouteInfo{
1101			Domain:          domainName,
1102			Image:           image,
1103			ContainerID:     containerID,
1104			ContainerStatus: status,
1105			Network:         network,
1106			Attachments:     attachmentsByDomain[domainName],
1107		})
1108	}
1109
1110	return results
1111}
1112
1113// stripRegistryPrefix removes the configured registry domain prefix from an image reference.
1114// For example, "reg.example.com/myapp:latest" becomes "myapp:latest" if registry domain is "reg.example.com".
1115func (s *Service) stripRegistryPrefix(image string) string {
1116	s.mu.RLock()
1117	cfg := s.config
1118	s.mu.RUnlock()
1119
1120	if cfg.RegistryDomain == "" {
1121		return image
1122	}
1123	prefix := strings.TrimSuffix(cfg.RegistryDomain, "/") + "/"
1124	if strings.HasPrefix(image, prefix) {
1125		return strings.TrimPrefix(image, prefix)
1126	}
1127	return image
1128}
1129
1130// ListAttachments returns attachments for a domain.
1131func (s *Service) ListAttachments(ctx context.Context, domainName string) []domain.Attachment {
1132	return s.getAttachmentsForDomain(ctx, domainName)
1133}
1134
1135// ListNetworks returns Gordon-managed networks.
1136func (s *Service) ListNetworks(ctx context.Context) ([]*domain.NetworkInfo, error) {
1137	s.mu.RLock()
1138	cfg := s.config
1139	s.mu.RUnlock()
1140
1141	networks, err := s.runtime.ListNetworks(ctx)
1142	if err != nil {
1143		return nil, err
1144	}
1145
1146	var filtered []*domain.NetworkInfo
1147	for _, network := range networks {
1148		if strings.HasPrefix(network.Name, cfg.NetworkPrefix+"-") && network.Labels[domain.LabelManaged] == "true" {
1149			filtered = append(filtered, network)
1150		}
1151	}
1152
1153	return filtered, nil
1154}
1155
1156// HealthCheck performs health checks on all containers.
1157func (s *Service) HealthCheck(ctx context.Context) map[string]bool {
1158	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1159		zerowrap.FieldLayer:   "usecase",
1160		zerowrap.FieldUseCase: "HealthCheck",
1161	})
1162	log := zerowrap.FromCtx(ctx)
1163
1164	// Copy container IDs under lock, then release before making Docker API calls
1165	s.mu.RLock()
1166	snapshot := make(map[string]string, len(s.containers))
1167	for d, c := range s.containers {
1168		snapshot[d] = c.ID
1169	}
1170	s.mu.RUnlock()
1171
1172	health := make(map[string]bool, len(snapshot))
1173	for d, id := range snapshot {
1174		running, err := s.runtime.IsContainerRunning(ctx, id)
1175		if err != nil {
1176			log.WrapErrWithFields(err, "health check failed", map[string]any{"domain": d, zerowrap.FieldEntityID: id})
1177			health[d] = false
1178		} else {
1179			health[d] = running
1180		}
1181	}
1182	return health
1183}
1184
1185// SyncContainers synchronizes containers with runtime state.
1186func (s *Service) SyncContainers(ctx context.Context) error {
1187	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1188		zerowrap.FieldLayer:   "usecase",
1189		zerowrap.FieldUseCase: "SyncContainers",
1190	})
1191	log := zerowrap.FromCtx(ctx)
1192
1193	// List containers without holding lock to avoid blocking during Docker API call
1194	allContainers, err := s.runtime.ListContainers(ctx, false)
1195	if err != nil {
1196		return log.WrapErr(err, "failed to list containers")
1197	}
1198
1199	managed := make(map[string]*domain.Container)
1200	attachments := make(map[string][]string)
1201	for _, c := range allContainers {
1202		if c.Labels == nil || c.Labels[domain.LabelManaged] != "true" {
1203			continue
1204		}
1205		if c.Labels[domain.LabelAttachment] == "true" {
1206			owner := c.Labels[domain.LabelAttachedTo]
1207			if owner != "" {
1208				attachments[owner] = append(attachments[owner], c.ID)
1209			}
1210			continue
1211		}
1212		if d, ok := c.Labels[domain.LabelDomain]; ok {
1213			managed[d] = c
1214		}
1215	}
1216
1217	s.mu.Lock()
1218	s.containers = managed
1219	s.attachments = attachments
1220	newCount := int64(len(managed))
1221	delta := newCount - s.managedCount
1222	s.managedCount = newCount
1223	s.mu.Unlock()
1224
1225	// Report initial/delta managed container count to OTel UpDownCounter.
1226	if delta != 0 && s.metrics != nil {
1227		s.metrics.ManagedContainers.Add(ctx, delta)
1228	}
1229
1230	log.Info().Int(zerowrap.FieldCount, len(managed)).Msg("container state synchronized")
1231	return nil
1232}
1233
1234// AutoStart starts containers for the provided routes that aren't running.
1235// It skips readiness checks to avoid blocking boot; the background monitor
1236// handles crash recovery.
1237func (s *Service) AutoStart(ctx context.Context, routes []domain.Route) error {
1238	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1239		zerowrap.FieldLayer:   "usecase",
1240		zerowrap.FieldUseCase: "AutoStart",
1241	})
1242	log := zerowrap.FromCtx(ctx)
1243
1244	log.Info().Int("route_count", len(routes)).Msg("auto-starting containers for configured routes")
1245
1246	// Filter out routes that already have a running container.
1247	var pending []domain.Route
1248	skipped := 0
1249	for _, route := range routes {
1250		if _, exists := s.Get(ctx, route.Domain); exists {
1251			log.Debug().Str("domain", route.Domain).Msg("container already running, skipping")
1252			skipped++
1253		} else {
1254			pending = append(pending, route)
1255		}
1256	}
1257
1258	if len(pending) == 0 {
1259		log.Info().Int("skipped", skipped).Msg("auto-start completed, all containers already running")
1260		return nil
1261	}
1262
1263	// Deploy all pending routes concurrently with readiness checks skipped.
1264	deployCtx := domain.WithSkipReadiness(ctx)
1265
1266	type result struct {
1267		route domain.Route
1268		err   error
1269	}
1270	results := make(chan result, len(pending))
1271	sem := make(chan struct{}, 4) // limit concurrency
1272
1273	for _, route := range pending {
1274		sem <- struct{}{}
1275		go func(r domain.Route) {
1276			defer func() { <-sem }()
1277
1278			log.Info().Str("domain", r.Domain).Str("image", r.Image).Msg("auto-starting container for route")
1279
1280			if _, err := s.Deploy(deployCtx, r); err != nil {
1281				results <- result{route: r, err: err}
1282				return
1283			}
1284			results <- result{route: r}
1285		}(route)
1286	}
1287
1288	var started, errCount int
1289	for i := 0; i < len(pending); i++ {
1290		res := <-results
1291		if res.err != nil {
1292			log.Warn().Err(res.err).Str("domain", res.route.Domain).Msg("failed to auto-start container")
1293			errCount++
1294		} else {
1295			started++
1296		}
1297	}
1298
1299	log.Info().
1300		Int("started", started).
1301		Int("skipped", skipped).
1302		Int("errors", errCount).
1303		Msg("auto-start completed")
1304
1305	if errCount > 0 {
1306		return fmt.Errorf("auto-start completed with %d errors", errCount)
1307	}
1308	return nil
1309}
1310
1311// Shutdown gracefully shuts down all managed containers.
1312func (s *Service) Shutdown(ctx context.Context) error {
1313	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1314		zerowrap.FieldLayer:   "usecase",
1315		zerowrap.FieldUseCase: "Shutdown",
1316	})
1317	log := zerowrap.FromCtx(ctx)
1318	log.Info().Msg("shutting down container manager...")
1319
1320	// Wait for any in-flight background cleanup goroutines to finish
1321	s.cleanupWg.Wait()
1322
1323	// Containers are left running across Gordon restarts.
1324	// SyncContainers + AutoStart will pick them back up on next boot.
1325
1326	// Close log writer to stop all log collection
1327	if s.logWriter != nil {
1328		if err := s.logWriter.Close(); err != nil {
1329			log.Warn().Err(err).Msg("failed to close container log writer")
1330		}
1331	}
1332
1333	log.Info().Msg("container manager shutdown complete")
1334	return nil
1335}
1336
1337// WaitForCleanup blocks until all background cleanup goroutines complete.
1338// Intended for use in tests to avoid mock assertion races.
1339func (s *Service) WaitForCleanup() {
1340	s.cleanupWg.Wait()
1341}
1342
1343// StartMonitor begins background monitoring of tracked containers.
1344// The monitor restarts crashed containers and detects crash loops.
1345// Safe to call multiple times; subsequent calls are no-ops.
1346func (s *Service) StartMonitor(ctx context.Context) {
1347	s.mu.Lock()
1348	if s.monitor != nil {
1349		s.mu.Unlock()
1350		return
1351	}
1352	s.monitor = newMonitor(s)
1353	m := s.monitor
1354	m.Start(ctx)
1355	s.mu.Unlock()
1356}
1357
1358// StopMonitor stops the background container monitor.
1359// Safe to call multiple times; subsequent calls are no-ops.
1360func (s *Service) StopMonitor() {
1361	s.mu.Lock()
1362	m := s.monitor
1363	s.monitor = nil
1364	s.mu.Unlock()
1365	if m != nil {
1366		m.Stop()
1367	}
1368}
1369
1370// UpdateConfig updates the service configuration.
1371func (s *Service) UpdateConfig(config Config) {
1372	s.mu.Lock()
1373	s.config = config
1374	s.mu.Unlock()
1375}
1376
1377// UpdateAttachments updates only the attachment configuration in the service.
1378// This is called after a config reload to propagate attachment changes without restart.
1379// The incoming map is deep-copied so external callers cannot mutate service state.
1380func (s *Service) UpdateAttachments(attachments map[string][]string) {
1381	var copied map[string][]string
1382	if attachments != nil {
1383		copied = make(map[string][]string, len(attachments))
1384		for k, v := range attachments {
1385			sl := make([]string, len(v))
1386			copy(sl, v)
1387			copied[k] = sl
1388		}
1389	}
1390	s.mu.Lock()
1391	s.config.Attachments = copied
1392	s.mu.Unlock()
1393}
1394
1395// Helper methods
1396
1397// cleanupFailedContainer stops and removes a container that failed to start properly.
1398func (s *Service) cleanupFailedContainer(ctx context.Context, containerID string) {
1399	log := zerowrap.FromCtx(ctx)
1400	if err := s.runtime.StopContainer(ctx, containerID); err != nil {
1401		log.Warn().Err(err).Str(zerowrap.FieldEntityID, containerID).Msg("failed to stop container after failure")
1402	}
1403	if err := s.runtime.RemoveContainer(ctx, containerID, true); err != nil {
1404		log.Warn().Err(err).Str(zerowrap.FieldEntityID, containerID).Msg("failed to remove container after failure")
1405	}
1406}
1407
1408// cleanupOldContainer stops and removes an old container after zero-downtime switch.
1409// It also renames the new container to the canonical name.
1410func (s *Service) cleanupOldContainer(ctx context.Context, old *domain.Container, newContainerID, domainName string) {
1411	log := zerowrap.FromCtx(ctx)
1412	log.Info().Str(zerowrap.FieldEntityID, old.ID).Msg("stopping old container after zero-downtime switch")
1413
1414	if s.logWriter != nil {
1415		if err := s.logWriter.StopLogging(old.ID); err != nil {
1416			log.Warn().Err(err).Str(zerowrap.FieldEntityID, old.ID).Msg("failed to stop logging for old container")
1417		}
1418	}
1419
1420	if err := s.runtime.StopContainer(ctx, old.ID); err != nil {
1421		log.Warn().Err(err).Str(zerowrap.FieldEntityID, old.ID).Msg("failed to stop old container")
1422	}
1423
1424	if err := s.runtime.RemoveContainer(ctx, old.ID, true); err != nil {
1425		log.Warn().Err(err).Str(zerowrap.FieldEntityID, old.ID).Msg("failed to remove old container")
1426	}
1427
1428	// Rename new container to canonical name
1429	canonicalName := fmt.Sprintf("gordon-%s", domainName)
1430	if err := s.runtime.RenameContainer(ctx, newContainerID, canonicalName); err != nil {
1431		log.Warn().Err(err).Str("canonical_name", canonicalName).Msg("failed to rename container to canonical name")
1432	}
1433}
1434
1435// startLogCollection starts log collection for a container in the background.
1436// Errors are logged but don't fail the calling operation.
1437func (s *Service) startLogCollection(ctx context.Context, containerID, domainName string) {
1438	if s.logWriter == nil {
1439		return
1440	}
1441
1442	log := zerowrap.FromCtx(ctx)
1443
1444	// Use context.WithoutCancel so the log stream outlives the HTTP request.
1445	// Without this, the Docker log stream closes when the deploy response completes.
1446	bgCtx := context.WithoutCancel(ctx)
1447
1448	logStream, err := s.runtime.GetContainerLogs(bgCtx, containerID, true)
1449	if err != nil {
1450		log.Warn().Err(err).Msg("failed to get container logs for collection")
1451		return
1452	}
1453
1454	if err := s.logWriter.StartLogging(bgCtx, containerID, domainName, logStream); err != nil {
1455		log.Warn().Err(err).Msg("failed to start container log collection")
1456		logStream.Close()
1457	}
1458}
1459
1460func (s *Service) buildImageRef(image string) string {
1461	s.mu.RLock()
1462	cfg := s.config
1463	s.mu.RUnlock()
1464
1465	if !cfg.RegistryAuthEnabled || cfg.RegistryDomain == "" {
1466		return image
1467	}
1468
1469	// Normalize registry domain by trimming any trailing slash
1470	reg := strings.TrimSuffix(cfg.RegistryDomain, "/")
1471
1472	// Check if image already has the registry domain prefix
1473	prefix := reg + "/"
1474	if strings.HasPrefix(image, prefix) {
1475		return image
1476	}
1477
1478	// Check if image already has an explicit registry (host:port/ or host.domain/).
1479	// We need to detect patterns like:
1480	// - "docker.io/library/nginx:latest" (has dot and slash)
1481	// - "localhost:5001/myapp:latest" (has host:port/ pattern)
1482	// - "myregistry:5000/app:v1" (has host:port/ pattern)
1483	// - "[fd00::1]:5000/app:v1" (IPv6 with port)
1484	if hasExplicitRegistry(image) {
1485		return image
1486	}
1487
1488	return fmt.Sprintf("%s/%s", reg, image)
1489}
1490
1491// hasExplicitRegistry checks if an image reference already includes an explicit registry.
1492// Returns true for patterns like "docker.io/image", "localhost:5000/image", "localhost/image",
1493// "registry:8080/image", "[fd00::1]/image", "[fd00::1]:5000/image".
1494func hasExplicitRegistry(image string) bool {
1495	// Find the first slash which separates registry from image name
1496	slashIdx := strings.Index(image, "/")
1497	if slashIdx == -1 {
1498		return false // No slash means no registry prefix (e.g., "myapp:latest")
1499	}
1500
1501	// Extract the part before the first slash (potential registry)
1502	registryPart := image[:slashIdx]
1503
1504	// Check for bracketed IPv6 address (e.g., "[fd00::1]" or "[fd00::1]:5000")
1505	if strings.HasPrefix(registryPart, "[") {
1506		// Look for closing bracket
1507		if closeBracket := strings.Index(registryPart, "]"); closeBracket != -1 {
1508			// Valid bracketed IPv6: either ends at ] or has ]:port
1509			return true
1510		}
1511	}
1512
1513	// Check for localhost (with or without port)
1514	if registryPart == "localhost" {
1515		return true
1516	}
1517
1518	// Check for host:port pattern (e.g., "localhost:5000", "registry:8080")
1519	if colonIdx := strings.LastIndex(registryPart, ":"); colonIdx != -1 {
1520		port := registryPart[colonIdx+1:]
1521		// If everything after colon is digits, it's a port
1522		if len(port) > 0 && isNumeric(port) {
1523			return true
1524		}
1525	}
1526
1527	// Check for domain pattern (has a dot, e.g., "docker.io", "gcr.io")
1528	if strings.Contains(registryPart, ".") {
1529		return true
1530	}
1531
1532	return false
1533}
1534
1535// isNumeric checks if a string contains only digits.
1536func isNumeric(s string) bool {
1537	for _, c := range s {
1538		if c < '0' || c > '9' {
1539			return false
1540		}
1541	}
1542	return true
1543}
1544
1545func normalizePullPolicy(policy string) string {
1546	switch strings.ToLower(strings.TrimSpace(policy)) {
1547	case PullPolicyAlways:
1548		return PullPolicyAlways
1549	case PullPolicyIfTagChanged:
1550		return PullPolicyIfTagChanged
1551	case PullPolicyIfNotPresent:
1552		return PullPolicyIfNotPresent
1553	default:
1554		return PullPolicyIfNotPresent
1555	}
1556}
1557
1558func isDigestRef(imageRef string) bool {
1559	return strings.Contains(imageRef, "@sha256:")
1560}
1561
1562func (s *Service) pullRefForDeploy(ctx context.Context, imageRef string) (string, bool) {
1563	if !domain.IsInternalDeploy(ctx) {
1564		return imageRef, false
1565	}
1566	s.mu.RLock()
1567	cfg := s.config
1568	s.mu.RUnlock()
1569	return rewriteToLocalRegistry(imageRef, cfg.RegistryDomain, cfg.RegistryPort), true
1570}
1571
1572// ensureImage ensures the image is available locally, pulling if needed.
1573// Returns the image reference to use for container operations:
1574//   - For digest references (@sha256:...), returns the pullRef since Docker can't tag digests
1575//   - For tagged images, returns the original imageRef after tagging the pulled image
1576func (s *Service) ensureImage(ctx context.Context, imageRef string) (string, error) {
1577	ctx, span := tracer.Start(ctx, "container.ensure_image",
1578		trace.WithAttributes(attribute.String("image", imageRef)))
1579	defer span.End()
1580
1581	ctx = zerowrap.CtxWithField(ctx, "image", imageRef)
1582	log := zerowrap.FromCtx(ctx)
1583
1584	// Determine if this is an internal deploy and what reference to use for pulls.
1585	pullRef, isInternal := s.pullRefForDeploy(ctx, imageRef)
1586	if pullRef != imageRef {
1587		log.Info().
1588			Str("original_ref", imageRef).
1589			Str("pull_ref", pullRef).
1590			Msg("internal deploy: using localhost registry for pull")
1591	}
1592
1593	found, err := s.ensureLocalImage(ctx, imageRef, pullRef)
1594	if err != nil {
1595		return "", err
1596	}
1597	if found {
1598		return imageRef, nil
1599	}
1600
1601	// Pull image
1602	log.Info().Msg("pulling image from registry")
1603
1604	if err := s.pullImage(ctx, pullRef, isInternal); err != nil {
1605		return "", err
1606	}
1607
1608	// For digest references, we can't create a tag (Docker doesn't allow it).
1609	// In this case, use the pullRef directly since the image is available under that reference.
1610	if strings.Contains(imageRef, "@sha256:") {
1611		log.Info().
1612			Str("pull_ref", pullRef).
1613			Msg("digest reference: using pull reference for container operations")
1614		return pullRef, nil
1615	}
1616
1617	if err := s.tagImageIfNeeded(ctx, pullRef, imageRef); err != nil {
1618		return "", err
1619	}
1620
1621	// Clean up the temporary pull reference tag to avoid duplicate entries
1622	if pullRef != imageRef {
1623		if err := s.runtime.UntagImage(ctx, pullRef); err != nil {
1624			// Log but don't fail - the canonical tag is already applied
1625			log.Debug().Err(err).Str("pull_ref", pullRef).Msg("failed to remove temporary pull tag")
1626		}
1627	}
1628
1629	log.Info().Msg("image pulled successfully")
1630	return imageRef, nil
1631}
1632
1633func (s *Service) ensureLocalImage(ctx context.Context, imageRef, pullRef string) (bool, error) {
1634	log := zerowrap.FromCtx(ctx)
1635
1636	// For internal deploys (image push events), always pull fresh image
1637	// because the same tag may reference new content (e.g., latest tag updated)
1638	if domain.IsInternalDeploy(ctx) {
1639		log.Info().Msg("internal deploy detected, forcing image pull to ensure latest content")
1640		return false, nil
1641	}
1642
1643	s.mu.RLock()
1644	cfg := s.config
1645	s.mu.RUnlock()
1646
1647	pullPolicy := normalizePullPolicy(cfg.PullPolicy)
1648	switch pullPolicy {
1649	case PullPolicyAlways:
1650		log.Info().Str("pull_policy", pullPolicy).Msg("pull policy forces image pull")
1651		return false, nil
1652	case PullPolicyIfTagChanged:
1653		if !isDigestRef(imageRef) {
1654			log.Info().Str("pull_policy", pullPolicy).Msg("tag reference detected, pulling to check for updates")
1655			return false, nil
1656		}
1657	}
1658
1659	localImages, err := s.runtime.ListImages(ctx)
1660	if err != nil {
1661		log.WrapErr(err, "failed to list local images, will attempt pull")
1662		return false, nil
1663	}
1664
1665	normalizedRef := normalizeImageRef(imageRef)
1666	normalizedPullRef := normalizeImageRef(pullRef)
1667	for _, img := range localImages {
1668		normalizedImage := normalizeImageRef(img)
1669		if normalizedImage == normalizedRef {
1670			log.Info().Msg("image found locally, skipping pull")
1671			return true, nil
1672		}
1673		if normalizedImage == normalizedPullRef {
1674			if err := s.tagImageIfNeeded(ctx, pullRef, imageRef); err != nil {
1675				return false, err
1676			}
1677			// Clean up the temporary pull reference tag
1678			if pullRef != imageRef {
1679				if err := s.runtime.UntagImage(ctx, pullRef); err != nil {
1680					log.Debug().Err(err).Str("pull_ref", pullRef).Msg("failed to remove temporary pull tag")
1681				}
1682			}
1683			log.Info().Msg("image found locally, skipping pull")
1684			return true, nil
1685		}
1686	}
1687
1688	return false, nil
1689}
1690
1691func (s *Service) pullImage(ctx context.Context, pullRef string, isInternal bool) error {
1692	s.mu.RLock()
1693	cfg := s.config
1694	s.mu.RUnlock()
1695
1696	log := zerowrap.FromCtx(ctx)
1697
1698	switch {
1699	case isInternal && cfg.RegistryAuthEnabled:
1700		if cfg.InternalRegistryUsername == "" || cfg.InternalRegistryPassword == "" {
1701			return log.WrapErr(fmt.Errorf("internal registry auth not configured"), "failed to pull image for internal deploy")
1702		}
1703		if err := s.pullImageWithRetry(ctx, func(ctx context.Context) error {
1704			return s.runtime.PullImageWithAuth(ctx, pullRef, cfg.InternalRegistryUsername, cfg.InternalRegistryPassword)
1705		}, isInternal); err != nil {
1706			return log.WrapErr(err, "failed to pull image with internal auth")
1707		}
1708	case isInternal:
1709		if err := s.pullImageWithRetry(ctx, func(ctx context.Context) error {
1710			return s.runtime.PullImage(ctx, pullRef)
1711		}, isInternal); err != nil {
1712			return log.WrapErr(err, "failed to pull image")
1713		}
1714	case cfg.RegistryAuthEnabled:
1715		if cfg.ServiceTokenUsername == "" || cfg.ServiceToken == "" {
1716			return log.WrapErr(fmt.Errorf("registry service token not configured"), "failed to pull image for registry auth")
1717		}
1718		if err := s.runtime.PullImageWithAuth(ctx, pullRef, cfg.ServiceTokenUsername, cfg.ServiceToken); err != nil {
1719			return log.WrapErr(err, "failed to pull image with auth")
1720		}
1721	default:
1722		if err := s.runtime.PullImage(ctx, pullRef); err != nil {
1723			return log.WrapErr(err, "failed to pull image")
1724		}
1725	}
1726
1727	return nil
1728}
1729
1730func (s *Service) pullImageWithRetry(ctx context.Context, pullFn func(context.Context) error, isInternal bool) error {
1731	log := zerowrap.FromCtx(ctx)
1732
1733	for attempt := 1; attempt <= internalPullMaxAttempts; attempt++ {
1734		err := pullFn(ctx)
1735		if err == nil {
1736			return nil
1737		}
1738		if !isInternal || !isConnectionRefusedError(err) || attempt == internalPullMaxAttempts {
1739			return fmt.Errorf("internal image pull failed after %d attempts: %w", attempt, err)
1740		}
1741
1742		backoff := time.Duration(attempt) * time.Second
1743		log.Warn().
1744			Err(err).
1745			Int("attempt", attempt).
1746			Dur("backoff", backoff).
1747			Msg("internal image pull failed with connection refused, retrying")
1748
1749		select {
1750		case <-time.After(backoff):
1751		case <-ctx.Done():
1752			return ctx.Err()
1753		}
1754	}
1755	return nil
1756}
1757
1758func (s *Service) tagImageIfNeeded(ctx context.Context, sourceRef, targetRef string) error {
1759	if sourceRef == targetRef {
1760		return nil
1761	}
1762
1763	// Cannot create a tag with a digest reference - Docker/Podman doesn't allow it.
1764	// When using digest references (image@sha256:...), skip tagging as the image
1765	// is already available by its digest.
1766	if strings.Contains(targetRef, "@sha256:") {
1767		return nil
1768	}
1769
1770	log := zerowrap.FromCtx(ctx)
1771	if err := s.runtime.TagImage(ctx, sourceRef, targetRef); err != nil {
1772		return log.WrapErr(err, "failed to tag image from pull reference")
1773	}
1774	return nil
1775}
1776
1777func (s *Service) loadEnvironment(ctx context.Context, preResolved []string, domainName, imageRef string) ([]string, error) {
1778	log := zerowrap.FromCtx(ctx)
1779
1780	var userEnvVars []string
1781	if len(preResolved) > 0 {
1782		userEnvVars = preResolved
1783	} else {
1784		var err error
1785		userEnvVars, err = s.envLoader.LoadEnv(ctx, domainName)
1786		if err != nil {
1787			return nil, log.WrapErr(err, "failed to load environment variables")
1788		}
1789	}
1790
1791	dockerfileEnvVars, err := s.runtime.InspectImageEnv(ctx, imageRef)
1792	if err != nil {
1793		log.WrapErr(err, "failed to inspect image environment")
1794		dockerfileEnvVars = []string{}
1795	}
1796
1797	return mergeEnvironmentVariables(dockerfileEnvVars, userEnvVars), nil
1798}
1799
1800func (s *Service) setupVolumes(ctx context.Context, domainName, imageRef string) (map[string]string, error) {
1801	s.mu.RLock()
1802	cfg := s.config
1803	s.mu.RUnlock()
1804
1805	log := zerowrap.FromCtx(ctx)
1806	volumes := make(map[string]string)
1807
1808	if !cfg.VolumeAutoCreate {
1809		return volumes, nil
1810	}
1811
1812	volumePaths, err := s.runtime.InspectImageVolumes(ctx, imageRef)
1813	if err != nil {
1814		log.WrapErr(err, "failed to inspect image volumes")
1815		return volumes, nil
1816	}
1817
1818	for _, path := range volumePaths {
1819		name := generateVolumeName(cfg.VolumePrefix, domainName, path)
1820
1821		exists, err := s.runtime.VolumeExists(ctx, name)
1822		if err != nil {
1823			log.WrapErrWithFields(err, "failed to check volume", map[string]any{"volume": name})
1824			continue
1825		}
1826
1827		if !exists {
1828			if err := s.runtime.CreateVolume(ctx, name); err != nil {
1829				log.WrapErrWithFields(err, "failed to create volume", map[string]any{"volume": name})
1830				continue
1831			}
1832			log.Info().Str("volume", name).Str(zerowrap.FieldPath, path).Msg("created volume")
1833		}
1834
1835		volumes[path] = name
1836	}
1837
1838	return volumes, nil
1839}
1840
1841func (s *Service) getNetworkForApp(domainName string) string {
1842	var networkIsolation bool
1843	var networkGroups map[string][]string
1844
1845	if s.configProvider != nil {
1846		snap := s.configProvider.GetAttachmentConfig()
1847		networkGroups = snap.NetworkGroups
1848		s.mu.RLock()
1849		networkIsolation = s.config.NetworkIsolation
1850		s.mu.RUnlock()
1851	} else {
1852		s.mu.RLock()
1853		networkIsolation = s.config.NetworkIsolation
1854		networkGroups = s.config.NetworkGroups
1855		s.mu.RUnlock()
1856	}
1857
1858	if !networkIsolation {
1859		return "bridge"
1860	}
1861
1862	for groupName, domains := range networkGroups {
1863		if slices.Contains(domains, domainName) {
1864			return s.generateNetworkName(groupName)
1865		}
1866	}
1867
1868	return s.generateNetworkName(domainName)
1869}
1870
1871func (s *Service) generateNetworkName(identifier string) string {
1872	s.mu.RLock()
1873	cfg := s.config
1874	s.mu.RUnlock()
1875
1876	return fmt.Sprintf("%s-%s", cfg.NetworkPrefix, strings.ReplaceAll(identifier, ".", "-"))
1877}
1878
1879func (s *Service) createNetworkIfNeeded(ctx context.Context, networkName string) error {
1880	if networkName == "bridge" || networkName == "default" {
1881		return nil
1882	}
1883
1884	ctx = zerowrap.CtxWithField(ctx, "network", networkName)
1885	log := zerowrap.FromCtx(ctx)
1886
1887	exists, err := s.runtime.NetworkExists(ctx, networkName)
1888	if err != nil {
1889		return log.WrapErr(err, "failed to check network existence")
1890	}
1891
1892	if !exists {
1893		if err := s.runtime.CreateNetwork(ctx, networkName, map[string]string{"driver": "bridge"}); err != nil {
1894			return log.WrapErr(err, "failed to create network")
1895		}
1896		log.Info().Msg("created network for app isolation")
1897	}
1898
1899	return nil
1900}
1901
1902func (s *Service) cleanupOrphanedContainers(ctx context.Context, domainName string, skipContainerID string) error {
1903	log := zerowrap.FromCtx(ctx)
1904	expectedName := fmt.Sprintf("gordon-%s", domainName)
1905	expectedNewName := expectedName + "-new"
1906	expectedNextName := expectedName + "-next"
1907
1908	allContainers, err := s.runtime.ListContainers(ctx, true)
1909	if err != nil {
1910		return err
1911	}
1912
1913	for _, c := range allContainers {
1914		if (c.Name == expectedName || c.Name == expectedNewName || c.Name == expectedNextName) && c.ID != skipContainerID {
1915			// Skip any running container regardless of name — a running
1916			// temp container may be actively serving traffic while the
1917			// new container stabilizes, or may be from a concurrent deploy.
1918			if c.Status == "running" {
1919				log.Debug().
1920					Str(zerowrap.FieldEntityID, c.ID).
1921					Str("container_name", c.Name).
1922					Str(zerowrap.FieldStatus, c.Status).
1923					Msg("skipping running container during orphan cleanup")
1924				continue
1925			}
1926
1927			log.Info().Str(zerowrap.FieldEntityID, c.ID).Str("container_name", c.Name).Str(zerowrap.FieldStatus, c.Status).Msg("found orphaned container, removing")
1928
1929			if err := s.runtime.StopContainer(ctx, c.ID); err != nil {
1930				log.WrapErrWithFields(err, "failed to stop orphaned container", map[string]any{zerowrap.FieldEntityID: c.ID})
1931			}
1932
1933			if err := s.runtime.RemoveContainer(ctx, c.ID, true); err != nil {
1934				log.WrapErrWithFields(err, "failed to remove orphaned container", map[string]any{zerowrap.FieldEntityID: c.ID})
1935			}
1936		}
1937	}
1938
1939	return nil
1940}
1941
1942func (s *Service) cleanupVolumesForDomain(_ context.Context, _ string) error {
1943	return nil
1944}
1945
1946func (s *Service) cleanupNetworkIfEmpty(ctx context.Context, networkName string) error {
1947	if networkName == "bridge" || networkName == "default" {
1948		return nil
1949	}
1950
1951	ctx = zerowrap.CtxWithField(ctx, "network", networkName)
1952	log := zerowrap.FromCtx(ctx)
1953
1954	networks, err := s.runtime.ListNetworks(ctx)
1955	if err != nil {
1956		return log.WrapErr(err, "failed to list networks")
1957	}
1958
1959	for _, n := range networks {
1960		if n.Name == networkName && len(n.Containers) == 0 {
1961			if err := s.runtime.RemoveNetwork(ctx, networkName); err != nil {
1962				return err
1963			}
1964			log.Info().Msg("cleaned up empty network")
1965			break
1966		}
1967	}
1968
1969	return nil
1970}
1971
1972func (s *Service) deployAttachments(ctx context.Context, domainName, networkName string) error {
1973	// Collect attachments from both domain-specific and network group configs
1974	attachments := s.resolveAttachmentsForDomain(domainName)
1975	if len(attachments) == 0 {
1976		return nil
1977	}
1978
1979	log := zerowrap.FromCtx(ctx)
1980	var deployed []string // track successfully deployed attachment IDs for rollback
1981	for _, svc := range attachments {
1982		if err := s.deployAttachedService(ctx, domainName, svc, networkName); err != nil {
1983			log.WrapErrWithFields(err, "failed to deploy attachment", map[string]any{zerowrap.FieldService: svc, "domain": domainName})
1984
1985			// Rollback: clean up already-deployed attachments
1986			for _, id := range deployed {
1987				if stopErr := s.runtime.StopContainer(ctx, id); stopErr != nil {
1988					log.Warn().Err(stopErr).Str("attachment_id", id).Msg("rollback: failed to stop attachment")
1989				}
1990				if rmErr := s.runtime.RemoveContainer(ctx, id, true); rmErr != nil {
1991					log.Warn().Err(rmErr).Str("attachment_id", id).Msg("rollback: failed to remove attachment")
1992				}
1993			}
1994			// Deregister rolled-back attachments
1995			if len(deployed) > 0 {
1996				s.mu.Lock()
1997				remaining := s.attachments[domainName]
1998				filtered := remaining[:0]
1999				rollbackSet := make(map[string]bool, len(deployed))
2000				for _, id := range deployed {
2001					rollbackSet[id] = true
2002				}
2003				for _, id := range remaining {
2004					if !rollbackSet[id] {
2005						filtered = append(filtered, id)
2006					}
2007				}
2008				s.attachments[domainName] = filtered
2009				s.mu.Unlock()
2010			}
2011
2012			return fmt.Errorf("failed to deploy attachment %q (rolled back %d already-deployed)", svc, len(deployed))
2013		}
2014		// Collect IDs of attachments tracked under this domain after successful deploy
2015		s.mu.RLock()
2016		ids := s.attachments[domainName]
2017		if len(ids) > 0 {
2018			latest := ids[len(ids)-1]
2019			deployed = append(deployed, latest)
2020		}
2021		s.mu.RUnlock()
2022	}
2023	return nil
2024}
2025
2026// resolveAttachmentsForDomain returns attachments for a domain by checking:
2027// 1. Direct domain attachments (attachments[domain])
2028// 2. Network group attachments (attachments[group] where domain is in network_groups[group])
2029func (s *Service) resolveAttachmentsForDomain(domainName string) []string {
2030	var attachments map[string][]string
2031	var networkGroups map[string][]string
2032
2033	if s.configProvider != nil {
2034		snap := s.configProvider.GetAttachmentConfig()
2035		attachments = snap.Attachments
2036		networkGroups = snap.NetworkGroups
2037	} else {
2038		s.mu.RLock()
2039		attachments = s.config.Attachments
2040		networkGroups = s.config.NetworkGroups
2041		s.mu.RUnlock()
2042	}
2043
2044	seen := make(map[string]bool)
2045	var result []string
2046
2047	// First, add domain-specific attachments
2048	if domainAttachments, ok := attachments[domainName]; ok {
2049		for _, img := range domainAttachments {
2050			if !seen[img] {
2051				seen[img] = true
2052				result = append(result, img)
2053			}
2054		}
2055	}
2056
2057	// Then, find which network group this domain belongs to and add group attachments
2058	for groupName, domains := range networkGroups {
2059		if slices.Contains(domains, domainName) {
2060			if groupAttachments, ok := attachments[groupName]; ok {
2061				for _, img := range groupAttachments {
2062					if !seen[img] {
2063						seen[img] = true
2064						result = append(result, img)
2065					}
2066				}
2067			}
2068			break // Domain can only be in one network group
2069		}
2070	}
2071
2072	return result
2073}
2074
2075func (s *Service) getAttachmentsForDomain(ctx context.Context, domainName string) []domain.Attachment {
2076	containers, err := s.runtime.ListContainers(ctx, true)
2077	if err != nil {
2078		return nil
2079	}
2080
2081	return s.filterAttachments(ctx, containers, domainName)
2082}
2083
2084// getAllAttachments fetches all containers once and returns attachments grouped by domain.
2085// This avoids N+1 queries when listing multiple routes.
2086func (s *Service) getAllAttachments(ctx context.Context) map[string][]domain.Attachment {
2087	containers, err := s.runtime.ListContainers(ctx, true)
2088	if err != nil {
2089		return nil
2090	}
2091
2092	result := make(map[string][]domain.Attachment)
2093	for _, container := range containers {
2094		if container.Labels == nil {
2095			continue
2096		}
2097		if container.Labels[domain.LabelAttachment] != "true" {
2098			continue
2099		}
2100		ownerDomain := container.Labels[domain.LabelAttachedTo]
2101		if ownerDomain == "" {
2102			continue
2103		}
2104		image := container.Image
2105		serviceName := container.Name
2106		if labelImage, ok := container.Labels[domain.LabelImage]; ok && labelImage != "" {
2107			image = labelImage
2108			serviceName = extractServiceName(labelImage)
2109		}
2110		// Get the container's network for display in the routes table
2111		network := ""
2112		if networkName, err := s.runtime.GetContainerNetwork(ctx, container.ID); err == nil {
2113			network = networkName
2114		}
2115		attachment := domain.Attachment{
2116			Name:        serviceName,
2117			Image:       s.stripRegistryPrefix(image),
2118			ContainerID: container.ID,
2119			Status:      container.Status,
2120			Network:     network,
2121			Ports:       append([]int(nil), container.Ports...),
2122		}
2123		result[ownerDomain] = append(result[ownerDomain], attachment)
2124	}
2125
2126	return result
2127}
2128
2129// filterAttachments extracts attachments for a specific domain from a container list.
2130func (s *Service) filterAttachments(ctx context.Context, containers []*domain.Container, domainName string) []domain.Attachment {
2131	attachments := make([]domain.Attachment, 0)
2132	for _, container := range containers {
2133		if container.Labels == nil {
2134			continue
2135		}
2136		if container.Labels[domain.LabelAttachment] != "true" {
2137			continue
2138		}
2139		if container.Labels[domain.LabelAttachedTo] != domainName {
2140			continue
2141		}
2142		image := container.Image
2143		serviceName := container.Name
2144		if labelImage, ok := container.Labels[domain.LabelImage]; ok && labelImage != "" {
2145			image = labelImage
2146			serviceName = extractServiceName(labelImage)
2147		}
2148		// Get the container's network for display in the routes table
2149		network := ""
2150		if networkName, err := s.runtime.GetContainerNetwork(ctx, container.ID); err == nil {
2151			network = networkName
2152		}
2153		attachment := domain.Attachment{
2154			Name:        serviceName,
2155			Image:       s.stripRegistryPrefix(image),
2156			ContainerID: container.ID,
2157			Status:      container.Status,
2158			Network:     network,
2159			Ports:       append([]int(nil), container.Ports...),
2160		}
2161		attachments = append(attachments, attachment)
2162	}
2163
2164	return attachments
2165}
2166
2167func (s *Service) deployAttachedService(ctx context.Context, ownerDomain, serviceImage, networkName string) error {
2168	log := zerowrap.FromCtx(ctx)
2169
2170	// Parse service name from image (e.g., "my-postgres:latest" → "postgres")
2171	serviceName := extractServiceName(serviceImage)
2172	containerName := fmt.Sprintf("gordon-%s-%s", sanitizeName(ownerDomain), serviceName)
2173
2174	ctx = zerowrap.CtxWithFields(ctx, map[string]any{
2175		"attachment":     serviceName,
2176		"container_name": containerName,
2177		"owner_domain":   ownerDomain,
2178	})
2179
2180	// Check if already running (idempotent)
2181	// Try new name first, then fall back to legacy name for backwards compatibility
2182	existingContainer, err := s.resolveExistingAttachment(ctx, ownerDomain, serviceName)
2183	if err != nil {
2184		return err
2185	}
2186
2187	if existingContainer != nil {
2188		shouldSkip, err := s.handleRunningAttachment(ctx, existingContainer, containerName, serviceImage)
2189		if err != nil {
2190			return err
2191		}
2192		if shouldSkip {
2193			return nil
2194		}
2195		if existingContainer.Status == string(domain.ContainerStatusRunning) {
2196			existingContainer = nil
2197		}
2198	}
2199
2200	if err := s.removeStoppedAttachment(ctx, existingContainer, containerName); err != nil {
2201		return err
2202	}
2203
2204	log.Info().Str(zerowrap.FieldService, serviceImage).Msg("deploying attached service")
2205
2206	// Ensure image (canonical ref is returned; internal pulls may use the local registry).
2207	imageRef := s.buildImageRef(serviceImage)
2208	actualImageRef, err := s.ensureImage(ctx, imageRef)
2209	if err != nil {
2210		return err
2211	}
2212
2213	// Get image metadata
2214	exposedPorts, err := s.runtime.GetImageExposedPorts(ctx, actualImageRef)
2215	if err != nil {
2216		log.WrapErr(err, "failed to get exposed ports for attachment, using defaults")
2217		exposedPorts = []int{}
2218	}
2219
2220	// Setup volumes (attachments need persistent data)
2221	volumes, err := s.setupVolumes(ctx, containerName, actualImageRef)
2222	if err != nil {
2223		log.WrapErr(err, "failed to setup volumes for attachment")
2224		volumes = make(map[string]string)
2225	}
2226
2227	// Load environment (attachment-specific env file)
2228	envVars, err := s.loadEnvironment(ctx, nil, containerName, actualImageRef)
2229	if err != nil {
2230		log.WrapErr(err, "failed to load environment for attachment")
2231		envVars = []string{}
2232	}
2233	envHash := hashEnvironment(envVars)
2234
2235	// Create container with attachment labels (use actual ref, track original in labels)
2236	config := &domain.ContainerConfig{
2237		Image:       actualImageRef,
2238		Name:        containerName,
2239		Hostname:    serviceName, // Internal DNS: postgres, redis, etc.
2240		Ports:       exposedPorts,
2241		Env:         envVars,
2242		Volumes:     volumes,
2243		NetworkMode: networkName, // Same network as main app
2244		Labels: map[string]string{
2245			domain.LabelManaged:    "true",
2246			domain.LabelAttachment: "true",
2247			domain.LabelAttachedTo: ownerDomain,
2248			domain.LabelEnvHash:    envHash,
2249			domain.LabelImage:      serviceImage,
2250		},
2251	}
2252
2253	container, err := s.runtime.CreateContainer(ctx, config)
2254	if err != nil {
2255		return log.WrapErr(err, "failed to create attachment container")
2256	}
2257
2258	// Start container
2259	if err := s.runtime.StartContainer(ctx, container.ID); err != nil {
2260		s.runtime.RemoveContainer(ctx, container.ID, true)
2261		return log.WrapErr(err, "failed to start attachment container")
2262	}
2263
2264	// Track attachment
2265	s.mu.Lock()
2266	s.attachments[ownerDomain] = append(s.attachments[ownerDomain], container.ID)
2267	s.mu.Unlock()
2268
2269	// Start log collection for attachment
2270	s.startLogCollection(ctx, container.ID, containerName)
2271
2272	log.Info().Str(zerowrap.FieldEntityID, container.ID).Msg("attachment deployed successfully")
2273	return nil
2274}
2275
2276func (s *Service) attachmentEnvDrifted(ctx context.Context, existing *domain.Container, containerName, serviceImage string) (bool, error) {
2277	currentEnv, err := s.loadEnvironment(ctx, nil, containerName, s.buildImageRef(serviceImage))
2278	if err != nil {
2279		return false, err
2280	}
2281
2282	currentHash := hashEnvironment(currentEnv)
2283	existingHash, ok := existing.Labels[domain.LabelEnvHash]
2284	if ok && existingHash == currentHash {
2285		return false, nil
2286	}
2287
2288	return true, nil
2289}
2290
2291func (s *Service) handleRunningAttachment(ctx context.Context, existing *domain.Container, containerName, serviceImage string) (bool, error) {
2292	if existing.Status != string(domain.ContainerStatusRunning) {
2293		return false, nil
2294	}
2295
2296	log := zerowrap.FromCtx(ctx)
2297
2298	// Check image drift
2299	existingImage := existing.Labels[domain.LabelImage]
2300	imageDrifted := existingImage != "" && existingImage != serviceImage
2301
2302	// Check env drift
2303	envDrifted, err := s.attachmentEnvDrifted(ctx, existing, containerName, serviceImage)
2304	if err != nil {
2305		return false, log.WrapErr(err, "failed to check attachment env drift")
2306	}
2307
2308	if !envDrifted && !imageDrifted {
2309		log.Debug().Str("container_name", containerName).Msg("attachment already running with current env and image, skipping")
2310		return true, nil
2311	}
2312
2313	if imageDrifted {
2314		log.Info().Str("container_name", containerName).Str("existing_image", existingImage).Str("desired_image", serviceImage).Msg("attachment image changed, recreating")
2315	}
2316	if envDrifted {
2317		log.Info().Str("container_name", containerName).Msg("attachment env changed, recreating")
2318	}
2319
2320	if err := s.runtime.StopContainer(ctx, existing.ID); err != nil {
2321		return false, log.WrapErr(err, "failed to stop attachment for drift update")
2322	}
2323	if err := s.runtime.RemoveContainer(ctx, existing.ID, true); err != nil {
2324		return false, log.WrapErr(err, "failed to remove attachment for drift update")
2325	}
2326
2327	return false, nil
2328}
2329
2330func (s *Service) removeStoppedAttachment(ctx context.Context, existing *domain.Container, containerName string) error {
2331	if existing == nil {
2332		return nil
2333	}
2334
2335	log := zerowrap.FromCtx(ctx)
2336	log.Info().Str("container_name", containerName).Msg("removing stopped attachment container")
2337	if err := s.runtime.RemoveContainer(ctx, existing.ID, true); err != nil {
2338		return log.WrapErr(err, "failed to remove existing attachment container")
2339	}
2340
2341	return nil
2342}
2343
2344func (s *Service) resolveExistingAttachment(ctx context.Context, ownerDomain, serviceName string) (*domain.Container, error) {
2345	log := zerowrap.FromCtx(ctx)
2346	containerName := fmt.Sprintf("gordon-%s-%s", sanitizeName(ownerDomain), serviceName)
2347	existingContainer := s.findContainerByName(ctx, containerName)
2348	if existingContainer != nil {
2349		return existingContainer, nil
2350	}
2351
2352	containerNameLegacy := fmt.Sprintf("gordon-%s-%s", sanitizeNameLegacy(ownerDomain), serviceName)
2353	existingContainer = s.findContainerByName(ctx, containerNameLegacy)
2354	if existingContainer == nil {
2355		return nil, nil
2356	}
2357
2358	log.Info().Str("container_name", containerNameLegacy).Msg("found attachment with legacy naming, will be replaced")
2359	if err := s.runtime.StopContainer(ctx, existingContainer.ID); err != nil {
2360		return nil, log.WrapErr(err, "failed to stop legacy attachment container")
2361	}
2362	if err := s.runtime.RemoveContainer(ctx, existingContainer.ID, true); err != nil {
2363		return nil, log.WrapErr(err, "failed to remove legacy attachment container")
2364	}
2365
2366	return nil, nil
2367}
2368
2369// Utility functions
2370
2371func normalizeImageRef(image string) string {
2372	// Extract repository from image reference, handling port numbers correctly.
2373	// Examples:
2374	//   nginx:latest -> docker.io/library/nginx
2375	//   user/repo:tag -> docker.io/user/repo
2376	//   localhost:5000/image:tag -> localhost:5000/image
2377	//   registry.example.com/image:tag -> registry.example.com/image
2378	//
2379	// Note: This function assumes valid Docker image references. Edge cases like
2380	// "registry.com:5000" (registry with port but no image name) are not valid
2381	// image references per Docker's naming conventions, which require at least
2382	// one path component after the registry (e.g., "registry.com:5000/image").
2383
2384	// Find the tag separator: the last colon that isn't part of a port number.
2385	// A colon is part of a port if there's no slash after it until the next colon.
2386	repo := image
2387	lastColon := strings.LastIndex(image, ":")
2388	if lastColon != -1 {
2389		afterColon := image[lastColon+1:]
2390		// If there's no slash after the colon, it's the tag separator
2391		if !strings.Contains(afterColon, "/") {
2392			repo = image[:lastColon]
2393		}
2394	}
2395
2396	if !strings.Contains(repo, "/") {
2397		return "docker.io/library/" + repo
2398	}
2399
2400	if strings.Count(repo, "/") == 1 && !strings.Contains(strings.Split(repo, "/")[0], ".") && !strings.Contains(strings.Split(repo, "/")[0], ":") {
2401		return "docker.io/" + repo
2402	}
2403
2404	return repo
2405}
2406
2407func generateVolumeName(prefix, domainName, volumePath string) string {
2408	return fmt.Sprintf("%s-%s-%s",
2409		prefix,
2410		strings.ReplaceAll(domainName, ".", "-"),
2411		strings.ReplaceAll(strings.Trim(volumePath, "/"), "/", "-"))
2412}
2413
2414func mergeEnvironmentVariables(dockerfileEnv, userEnv []string) []string {
2415	envMap := make(map[string]string)
2416
2417	for _, env := range dockerfileEnv {
2418		if k, v, ok := strings.Cut(env, "="); ok {
2419			envMap[k] = v
2420		}
2421	}
2422
2423	for _, env := range userEnv {
2424		if k, v, ok := strings.Cut(env, "="); ok {
2425			envMap[k] = v
2426		}
2427	}
2428
2429	keys := make([]string, 0, len(envMap))
2430	for k := range envMap {
2431		keys = append(keys, k)
2432	}
2433	slices.Sort(keys)
2434
2435	result := make([]string, 0, len(envMap))
2436	for _, k := range keys {
2437		result = append(result, k+"="+envMap[k])
2438	}
2439
2440	return result
2441}
2442
2443// extractServiceName gets service name from image reference.
2444// "my-postgres:latest" → "postgres", "redis:7" → "redis"
2445func extractServiceName(image string) string {
2446	// Remove tag
2447	parts := strings.Split(image, ":")
2448	name := parts[0]
2449
2450	// Remove registry prefix if present
2451	if strings.Contains(name, "/") {
2452		nameParts := strings.Split(name, "/")
2453		name = nameParts[len(nameParts)-1]
2454	}
2455
2456	// Remove common prefixes like "my-"
2457	name = strings.TrimPrefix(name, "my-")
2458
2459	return name
2460}
2461
2462// sanitizeName is a convenience wrapper around domain.SanitizeDomainForContainer.
2463func sanitizeName(d string) string {
2464	return domain.SanitizeDomainForContainer(d)
2465}
2466
2467// sanitizeNameLegacy is a convenience wrapper around domain.SanitizeDomainForContainerLegacy.
2468func sanitizeNameLegacy(d string) string {
2469	return domain.SanitizeDomainForContainerLegacy(d)
2470}
2471
2472func isContainerNotFoundError(err error) bool {
2473	if err == nil {
2474		return false
2475	}
2476	if errors.Is(err, domain.ErrContainerNotFound) {
2477		return true
2478	}
2479	msg := strings.ToLower(err.Error())
2480	return strings.Contains(msg, "no such container") ||
2481		strings.Contains(msg, "no container with name or id") ||
2482		strings.Contains(msg, "container not found")
2483}
2484
2485func isConnectionRefusedError(err error) bool {
2486	if err == nil {
2487		return false
2488	}
2489	return strings.Contains(strings.ToLower(err.Error()), "connection refused")
2490}
2491
2492// findContainerByName finds a container by its name.
2493func (s *Service) findContainerByName(ctx context.Context, name string) *domain.Container {
2494	containers, err := s.runtime.ListContainers(ctx, true)
2495	if err != nil {
2496		return nil
2497	}
2498
2499	for _, c := range containers {
2500		if c.Name == name {
2501			return c
2502		}
2503	}
2504	return nil
2505}
2506
2507func (s *Service) waitForReady(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig) error {
2508	if err := s.pollContainerRunning(ctx, containerID); err != nil {
2509		return err
2510	}
2511
2512	s.mu.RLock()
2513	cfg := s.config
2514	s.mu.RUnlock()
2515
2516	readinessMode := cfg.ReadinessMode
2517	if readinessMode == "" {
2518		readinessMode = "delay"
2519	}
2520
2521	switch readinessMode {
2522	case "delay":
2523		return s.waitForReadyByDelay(ctx, containerID)
2524	case "docker-health":
2525		_, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(ctx, containerID)
2526		if err != nil {
2527			return err
2528		}
2529		if !hasHealthcheck {
2530			return errors.New("no healthcheck detected")
2531		}
2532		return s.waitForHealthy(ctx, containerID, cfg.HealthTimeout)
2533	default: // auto
2534		return s.readinessCascade(ctx, containerID, containerConfig, cfg)
2535	}
2536}
2537
2538// readinessCascade auto-detects the strongest available readiness signal:
2539//  1. Docker healthcheck (if present) → wait for healthy status
2540//  2. HTTP probe (if gordon.health label set) → GET until 2xx/3xx
2541//  3. Default HTTP probe (GET / on exposed port) → wait for 2xx/3xx
2542//  4. TCP probe (if port info available) → connect until accepted
2543//  5. Delay fallback (last resort) → waitForReadyByDelay
2544func (s *Service) readinessCascade(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) error {
2545	log := zerowrap.FromCtx(ctx)
2546
2547	// 1. Docker healthcheck
2548	_, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(ctx, containerID)
2549	if err != nil {
2550		return err
2551	}
2552	if hasHealthcheck {
2553		log.Info().Msg("readiness cascade: using Docker healthcheck")
2554		return s.waitForHealthy(ctx, containerID, cfg.HealthTimeout)
2555	}
2556
2557	// 2. HTTP probe via gordon.health label
2558	if probed, probeErr := s.tryHTTPProbe(ctx, containerID, containerConfig, cfg); probed {
2559		return probeErr
2560	}
2561
2562	// 3. Default HTTP probe on root path (if port info available)
2563	if probed, probeErr := s.tryDefaultHTTPProbe(ctx, containerID, containerConfig, cfg); probed {
2564		return probeErr
2565	}
2566
2567	// 4. TCP probe (if port info available)
2568	if probed, probeErr := s.tryTCPProbe(ctx, containerID, containerConfig, cfg); probed {
2569		return probeErr
2570	}
2571
2572	// 5. Delay fallback
2573	log.Info().Msg("readiness cascade: using delay fallback")
2574	return s.waitForReadyByDelay(ctx, containerID)
2575}
2576
2577// tryHTTPProbe attempts an HTTP probe if the gordon.health label is set.
2578// Returns (true, err) if the probe was attempted, (false, nil) if skipped.
2579func (s *Service) tryHTTPProbe(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) (bool, error) {
2580	log := zerowrap.FromCtx(ctx)
2581	if containerConfig == nil {
2582		return false, nil
2583	}
2584	healthPath, ok := containerConfig.Labels[domain.LabelHealth]
2585	if !ok || healthPath == "" {
2586		return false, nil
2587	}
2588	ip, port, probeErr := s.resolveProbeEndpoint(ctx, containerID, containerConfig)
2589	if probeErr != nil || ip == "" || port <= 0 {
2590		log.Debug().Err(probeErr).Msg("readiness cascade: HTTP probe skipped, could not resolve container endpoint")
2591		return false, nil
2592	}
2593	url := fmt.Sprintf("http://%s:%d%s", ip, port, healthPath)
2594	timeout := cfg.HTTPProbeTimeout
2595	if timeout == 0 {
2596		timeout = 60 * time.Second
2597	}
2598	log.Info().Str("url", url).Dur("timeout", timeout).Msg("readiness cascade: using HTTP probe")
2599	return true, httpProbe(ctx, url, timeout)
2600}
2601
2602// tryDefaultHTTPProbe attempts an HTTP GET on "/" using the container's
2603// exposed port. This is the default readiness check for web containers —
2604// it verifies the app is actually serving HTTP, not just accepting TCP.
2605// Returns (true, err) if the probe was attempted, (false, nil) if skipped.
2606func (s *Service) tryDefaultHTTPProbe(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) (bool, error) {
2607	log := zerowrap.FromCtx(ctx)
2608	if containerConfig == nil || len(containerConfig.Ports) == 0 {
2609		return false, nil
2610	}
2611	ip, port, probeErr := s.resolveProbeEndpoint(ctx, containerID, containerConfig)
2612	if probeErr != nil || ip == "" || port <= 0 {
2613		log.Debug().Err(probeErr).Msg("readiness cascade: default HTTP probe skipped, could not resolve container endpoint")
2614		return false, nil
2615	}
2616	url := fmt.Sprintf("http://%s:%d/", ip, port)
2617	timeout := cfg.HTTPProbeTimeout
2618	if timeout == 0 {
2619		timeout = 60 * time.Second
2620	}
2621	log.Info().Str("url", url).Dur("timeout", timeout).Msg("readiness cascade: using default HTTP alive probe")
2622	return true, httpAliveProbe(ctx, url, timeout)
2623}
2624
2625// tryTCPProbe attempts a TCP probe if port info is available.
2626// Returns (true, err) if the probe was attempted, (false, nil) if skipped.
2627func (s *Service) tryTCPProbe(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) (bool, error) {
2628	log := zerowrap.FromCtx(ctx)
2629	if containerConfig == nil || len(containerConfig.Ports) == 0 {
2630		return false, nil
2631	}
2632	ip, port, probeErr := s.resolveProbeEndpoint(ctx, containerID, containerConfig)
2633	if probeErr != nil || ip == "" || port <= 0 {
2634		log.Debug().Err(probeErr).Msg("readiness cascade: TCP probe skipped, could not resolve container endpoint")
2635		return false, nil
2636	}
2637	addr := fmt.Sprintf("%s:%d", ip, port)
2638	timeout := cfg.TCPProbeTimeout
2639	if timeout == 0 {
2640		timeout = 30 * time.Second
2641	}
2642	log.Info().Str("addr", addr).Dur("timeout", timeout).Msg("readiness cascade: using TCP probe")
2643	return true, tcpProbe(ctx, addr, timeout)
2644}
2645
2646// resolveContainerEndpoint returns a host-reachable address for probing.
2647// In rootless podman/Docker setups, container internal IPs are not routable
2648// from the host. We use the host port binding (127.0.0.1:<mapped_port>)
2649// which is always reachable. Falls back to the container's internal IP
2650// only if no host port mapping exists (e.g. host-network mode).
2651func (s *Service) resolveProbeEndpoint(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig) (string, int, error) {
2652	log := zerowrap.FromCtx(ctx)
2653
2654	internalPort := 0
2655	if containerConfig != nil {
2656		if labels := containerConfig.Labels; labels != nil {
2657			for _, key := range []string{domain.LabelProxyPort, domain.LabelPort} {
2658				if portStr, ok := labels[key]; ok && portStr != "" {
2659					port, err := strconv.Atoi(portStr)
2660					if err == nil && port > 0 && port <= 65535 {
2661						internalPort = port
2662						break
2663					}
2664					log.Warn().Str("label", key).Str("port_value", portStr).Msg("invalid probe port label value")
2665				}
2666			}
2667		}
2668	}
2669
2670	if internalPort == 0 {
2671		_, resolvedPort, err := s.runtime.GetContainerNetworkInfo(ctx, containerID)
2672		if err != nil {
2673			return "", 0, err
2674		}
2675		internalPort = resolvedPort
2676	}
2677
2678	hostPort, hostErr := s.runtime.GetContainerPort(ctx, containerID, internalPort)
2679	if hostErr == nil && hostPort > 0 {
2680		log.Debug().
2681			Int("internal_port", internalPort).
2682			Int("host_port", hostPort).
2683			Msg("resolved container endpoint via host port binding")
2684		return "127.0.0.1", hostPort, nil
2685	}
2686
2687	ip, _, fallbackErr := s.runtime.GetContainerNetworkInfo(ctx, containerID)
2688	if fallbackErr != nil {
2689		return "", 0, fallbackErr
2690	}
2691	log.Debug().
2692		Str("ip", ip).
2693		Int("port", internalPort).
2694		Msg("resolved container endpoint via internal IP (no host port binding)")
2695	return ip, internalPort, nil
2696}
2697
2698// waitForReadyByDelay waits using the legacy running+delay strategy.
2699func (s *Service) waitForReadyByDelay(ctx context.Context, containerID string) error {
2700	log := zerowrap.FromCtx(ctx)
2701
2702	s.mu.RLock()
2703	cfg := s.config
2704	s.mu.RUnlock()
2705
2706	delay := cfg.ReadinessDelay
2707	if delay == 0 {
2708		delay = 5 * time.Second
2709	}
2710
2711	log.Debug().Dur("delay", delay).Msg("waiting for container readiness")
2712	select {
2713	case <-time.After(delay):
2714	case <-ctx.Done():
2715		return ctx.Err()
2716	}
2717
2718	// Verify still running after delay
2719	running, err := s.runtime.IsContainerRunning(ctx, containerID)
2720	if err != nil {
2721		return err
2722	}
2723	if running {
2724		return nil
2725	}
2726
2727	return s.waitForRecovery(ctx, containerID)
2728}
2729
2730func (s *Service) waitForHealthy(ctx context.Context, containerID string, timeout time.Duration) error {
2731	if timeout == 0 {
2732		timeout = 90 * time.Second
2733	}
2734	waitCtx, cancel := context.WithTimeout(ctx, timeout)
2735	defer cancel()
2736
2737	var lastStatus string
2738	for {
2739		status, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(waitCtx, containerID)
2740		if err != nil {
2741			return err
2742		}
2743		if !hasHealthcheck {
2744			return errors.New("no healthcheck detected")
2745		}
2746		// Treat empty health status as transitional startup. Some runtimes can
2747		// temporarily report an empty status before first probe results.
2748		if status == "" {
2749			status = "starting"
2750		}
2751		if status == "healthy" {
2752			return nil
2753		}
2754		lastStatus = status
2755
2756		select {
2757		case <-time.After(time.Second):
2758		case <-waitCtx.Done():
2759			return fmt.Errorf("container healthcheck timeout after %s (last status: %s)", timeout, lastStatus)
2760		}
2761	}
2762}
2763
2764// pollContainerRunning polls until the container is running (max 30 seconds).
2765func (s *Service) pollContainerRunning(ctx context.Context, containerID string) error {
2766	for i := 0; i < 30; i++ {
2767		running, err := s.runtime.IsContainerRunning(ctx, containerID)
2768		if err != nil {
2769			return err
2770		}
2771		if running {
2772			return nil
2773		}
2774		if i == 29 {
2775			return fmt.Errorf("container did not start within 30 seconds")
2776		}
2777		select {
2778		case <-time.After(time.Second):
2779		case <-ctx.Done():
2780			return ctx.Err()
2781		}
2782	}
2783	return fmt.Errorf("container did not start within 30 seconds")
2784}
2785
2786// waitForRecovery waits for a container to recover within the recovery window.
2787func (s *Service) waitForRecovery(ctx context.Context, containerID string) error {
2788	log := zerowrap.FromCtx(ctx)
2789	log.Warn().
2790		Dur("recovery_window", readinessRecoveryWindow).
2791		Msg("container not running after readiness delay, waiting for recovery")
2792
2793	deadline := time.Now().Add(readinessRecoveryWindow)
2794	for time.Now().Before(deadline) {
2795		select {
2796		case <-time.After(time.Second):
2797		case <-ctx.Done():
2798			return ctx.Err()
2799		}
2800
2801		running, err := s.runtime.IsContainerRunning(ctx, containerID)
2802		if err != nil {
2803			return err
2804		}
2805		if running {
2806			log.Info().Msg("container recovered during readiness recovery window")
2807			return nil
2808		}
2809	}
2810
2811	return fmt.Errorf("container not running after readiness delay and recovery window")
2812}
2813
2814// publishContainerDeployed publishes a container.deployed event.
2815func (s *Service) publishContainerDeployed(ctx context.Context, domainName, containerID string) {
2816	payload := &domain.ContainerEventPayload{
2817		ContainerID: containerID,
2818		Domain:      domainName,
2819		Action:      "deployed",
2820	}
2821
2822	if err := s.eventBus.Publish(domain.EventContainerDeployed, payload); err != nil {
2823		log := zerowrap.FromCtx(ctx)
2824		log.Warn().Err(err).Msg("failed to publish container deployed event")
2825	}
2826}
2827
2828// rewriteToRegistryDomain rewrites an image reference to use the configured registry domain.
2829// e.g., "myapp:latest" -> "registry.example.com/myapp:latest"
2830func rewriteToRegistryDomain(imageRef, registryDomain string) string {
2831	if registryDomain == "" {
2832		return imageRef
2833	}
2834
2835	prefix := registryDomain + "/"
2836	if strings.HasPrefix(imageRef, prefix) {
2837		return imageRef
2838	}
2839
2840	return prefix + imageRef
2841}
2842
2843// rewriteToLocalRegistry rewrites an image reference to use the local registry address.
2844// e.g., "registry.example.com/myapp:latest" -> "localhost:5000/myapp:latest"
2845func rewriteToLocalRegistry(imageRef, registryDomain string, registryPort int) string {
2846	if imageRef == "" {
2847		return imageRef
2848	}
2849
2850	localRegistry := fmt.Sprintf("localhost:%d", registryPort)
2851	localPrefix := localRegistry + "/"
2852	imageRef = strings.TrimPrefix(imageRef, localPrefix)
2853
2854	registryDomain = strings.TrimSuffix(registryDomain, "/")
2855	if registryDomain != "" {
2856		prefix := registryDomain + "/"
2857		imageRef = strings.TrimPrefix(imageRef, prefix)
2858	}
2859
2860	return localPrefix + imageRef
2861}