1// Package container implements the container management use case.
2package container
3
4import (
5 "bufio"
6 "context"
7 "encoding/binary"
8 "encoding/hex"
9 "errors"
10 "fmt"
11 "io"
12 "maps"
13 "net"
14 "slices"
15 "strconv"
16 "strings"
17 "sync"
18 "time"
19
20 "github.com/bnema/zerowrap"
21 "go.opentelemetry.io/otel"
22 "go.opentelemetry.io/otel/attribute"
23 "go.opentelemetry.io/otel/codes"
24 "go.opentelemetry.io/otel/metric"
25 "go.opentelemetry.io/otel/trace"
26
27 "github.com/bnema/gordon/internal/adapters/out/telemetry"
28 "github.com/bnema/gordon/internal/boundaries/out"
29 "github.com/bnema/gordon/internal/domain"
30)
31
32// Config holds configuration needed by the container service.
33type Config struct {
34 RegistryAuthEnabled bool
35 RegistryDomain string
36 LegacyRegistryDomains []string
37 RegistryPort int
38 ServiceTokenUsername string
39 ServiceToken string
40 InternalRegistryUsername string
41 InternalRegistryPassword string
42 PullPolicy string
43 VolumeAutoCreate bool
44 VolumePrefix string
45 VolumePreserve bool
46 NetworkIsolation bool
47 NetworkPrefix string
48 NetworkGroups map[string][]string
49 NetworkInternal bool
50 Attachments map[string][]string
51 AllowedRegistries []string
52 RequireImageDigest bool
53 SecurityProfile string
54 ReadinessDelay time.Duration // Delay after container starts before considering it ready
55 ReadinessMode string // Readiness strategy: auto, docker-health, delay
56 HealthTimeout time.Duration // Max wait for health-based readiness
57 DrainDelay time.Duration // Grace period after cache invalidation before stopping old container
58 DrainDelayConfigured bool // True when deploy.drain_delay was explicitly configured
59 DrainMode string // Drain strategy: auto, inflight, delay
60 DrainTimeout time.Duration // Max wait for in-flight requests to drain
61 StabilizationDelay time.Duration // Post-switch monitoring window (default 2s)
62 TCPProbeTimeout time.Duration // TCP probe timeout (default 30s)
63 HTTPProbeTimeout time.Duration // HTTP probe timeout (default 60s)
64 AttachmentReadinessTimeout time.Duration // Max wait for attachment readiness (default 30s)
65 DefaultMemoryLimit int64 // Default memory limit in bytes for containers (0 = no limit)
66 DefaultNanoCPUs int64 // Default CPU quota in nanoseconds for containers (0 = no limit)
67 DefaultPidsLimit int64 // Default max PIDs for containers (0 = no limit)
68}
69
70var tracer = otel.Tracer("gordon.container")
71
72const (
73 PullPolicyAlways = "always"
74 PullPolicyIfNotPresent = "if-not-present"
75 PullPolicyIfTagChanged = "if-tag-changed"
76
77 // readinessRecoveryWindow is an additional grace window used when a container
78 // is briefly not running at the end of readiness delay. This avoids false
79 // negatives during short startup flaps.
80 readinessRecoveryWindow = 30 * time.Second
81 failedContainerCleanupTimeout = 30 * time.Second
82 internalPullMaxAttempts = 3
83 deployFailureLogLimit = 20
84)
85
86// Service implements the ContainerService interface.
87type Service struct {
88 runtime out.ContainerRuntime
89 envLoader out.EnvLoader
90 eventBus out.EventPublisher
91 logWriter out.ContainerLogWriter
92 cacheInvalidator out.ProxyCacheInvalidator
93 drainWaiter out.ProxyDrainWaiter
94 config Config
95 configProvider AttachmentConfigProvider // live config reads for attachments/networks (may be nil)
96 metrics *telemetry.Metrics
97 containers map[string]*domain.Container
98 attachments map[string][]string // ownerDomain → []containerIDs
99 managedCount int64 // tracks UpDownCounter value for delta computation
100 mu sync.RWMutex
101 deployMu sync.Map // per-domain deploy locks (domain → *domainDeployLock)
102 cleanupWg sync.WaitGroup // tracks background old-container cleanup goroutines
103 monitor *Monitor
104}
105
106// domainDeployLock is a context-aware mutex using a buffered channel.
107// It allows Lock() to be interrupted by context cancellation.
108type domainDeployLock struct {
109 ch chan struct{}
110}
111
112// newDomainDeployLock creates a new lock that is initially unlocked.
113func newDomainDeployLock() *domainDeployLock {
114 l := &domainDeployLock{ch: make(chan struct{}, 1)}
115 l.ch <- struct{}{}
116 return l
117}
118
119// Lock acquires the lock or returns an error if the context is cancelled.
120func (l *domainDeployLock) Lock(ctx context.Context) error {
121 // Fast path: honor already-canceled contexts before blocking.
122 if err := ctx.Err(); err != nil {
123 return err
124 }
125 select {
126 case <-ctx.Done():
127 return ctx.Err()
128 case <-l.ch:
129 // Re-check context after acquisition: if canceled while we were
130 // in the select, release the token so the lock isn't consumed.
131 if err := ctx.Err(); err != nil {
132 l.ch <- struct{}{}
133 return err
134 }
135 return nil
136 }
137}
138
139// Unlock releases the lock.
140func (l *domainDeployLock) Unlock() {
141 select {
142 case l.ch <- struct{}{}:
143 default:
144 // This should never happen if Lock/Unlock are used correctly.
145 // Panic to detect double-unlocks or other misuse early.
146 panic("domainDeployLock: double unlock detected")
147 }
148}
149
150// NewService creates a new container service.
151func NewService(
152 runtime out.ContainerRuntime,
153 envLoader out.EnvLoader,
154 eventBus out.EventPublisher,
155 logWriter out.ContainerLogWriter,
156 config Config,
157 configProvider AttachmentConfigProvider,
158) *Service {
159 return &Service{
160 runtime: runtime,
161 envLoader: envLoader,
162 eventBus: eventBus,
163 logWriter: logWriter,
164 config: config,
165 configProvider: configProvider,
166 containers: make(map[string]*domain.Container),
167 attachments: make(map[string][]string),
168 }
169}
170
171// SetMetrics sets the telemetry metrics for the container service.
172func (s *Service) SetMetrics(m *telemetry.Metrics) {
173 s.metrics = m
174}
175
176// SetProxyCacheInvalidator sets the proxy cache invalidator for synchronous
177// cache invalidation during zero-downtime deployments. This must be called
178// after construction because the proxy service is created after the container
179// service during application initialization.
180func (s *Service) SetProxyCacheInvalidator(inv out.ProxyCacheInvalidator) {
181 s.mu.Lock()
182 s.cacheInvalidator = inv
183 s.mu.Unlock()
184}
185
186// SetProxyDrainWaiter sets the proxy in-flight drain waiter used during
187// zero-downtime replacement before stopping the old container.
188func (s *Service) SetProxyDrainWaiter(waiter out.ProxyDrainWaiter) {
189 s.mu.Lock()
190 s.drainWaiter = waiter
191 s.mu.Unlock()
192}
193
194// domainDeployMu returns a per-domain lock for serializing deploy operations.
195func (s *Service) domainDeployMu(domain string) *domainDeployLock {
196 v, _ := s.deployMu.LoadOrStore(domain, newDomainDeployLock())
197 // Type assertion is safe: LoadOrStore always stores *domainDeployLock values,
198 // and any concurrent LoadOrStore calls will also store *domainDeployLock.
199 return v.(*domainDeployLock)
200}
201
202// acquireDomainDeployLock acquires the per-domain deploy lock with context cancellation support.
203// Returns an unlock function that must be called to release the lock.
204func (s *Service) acquireDomainDeployLock(ctx context.Context, domain string) (func(), error) {
205 lock := s.domainDeployMu(domain)
206 if err := lock.Lock(ctx); err != nil {
207 return nil, err
208 }
209 return func() { lock.Unlock() }, nil
210}
211
212// buildContainerConfig constructs the container configuration for deployment.
213func (s *Service) deploymentContainerName(containerDomain string, existing *domain.Container) string {
214 canonicalName := managedContainerName(containerDomain)
215 if existing == nil {
216 return canonicalName
217 }
218
219 // Alternate temporary names if the tracked container was left with a temp suffix.
220 // This prevents name collisions after interrupted zero-downtime deploys.
221 newName := canonicalName + "-new"
222 nextName := canonicalName + "-next"
223 switch existing.Name {
224 case newName:
225 return nextName
226 case nextName:
227 return newName
228 default:
229 return newName
230 }
231}
232
233// containerConfigInput groups the parameters for building a container configuration.
234type containerConfigInput struct {
235 Domain string
236 Image string
237 ImageRef string
238 ExposedPorts []int
239 EnvVars []string
240 EnvHash string
241 Volumes map[string]string
242 NetworkName string
243 ImageLabels map[string]string
244 Existing *domain.Container
245}
246
247func (s *Service) buildContainerConfig(in containerConfigInput) *domain.ContainerConfig {
248 s.mu.RLock()
249 cfg := s.config
250 s.mu.RUnlock()
251
252 containerName := s.deploymentContainerName(in.Domain, in.Existing)
253 labels := map[string]string{
254 domain.LabelDomain: in.Domain,
255 domain.LabelEnvHash: in.EnvHash,
256 domain.LabelImage: in.Image,
257 domain.LabelManaged: "true",
258 domain.LabelRoute: in.Domain,
259 }
260
261 // Propagate proxy/port labels from image so readiness probes can find them
262 for _, key := range []string{domain.LabelProxyPort, domain.LabelPort, domain.LabelHealth} {
263 if v, ok := in.ImageLabels[key]; ok && v != "" {
264 labels[key] = v
265 }
266 }
267
268 // Ensure the label-specified proxy port is included in exposed ports
269 // so Docker creates a host port binding for it.
270 ports := in.ExposedPorts
271 if portStr, ok := labels[domain.LabelProxyPort]; ok {
272 if port, err := strconv.Atoi(portStr); err == nil && port > 0 && port <= 65535 {
273 if !slices.Contains(ports, port) {
274 ports = append(ports, port)
275 }
276 }
277 }
278
279 containerConfig := &domain.ContainerConfig{
280 Image: in.ImageRef,
281 Name: containerName,
282 Ports: ports,
283 Env: in.EnvVars,
284 Volumes: in.Volumes,
285 NetworkMode: in.NetworkName,
286 Hostname: in.Domain,
287 Labels: labels,
288 AutoRemove: false,
289 RestartPolicy: domain.RestartPolicyAlways,
290 MemoryLimit: cfg.DefaultMemoryLimit,
291 NanoCPUs: cfg.DefaultNanoCPUs,
292 PidsLimit: cfg.DefaultPidsLimit,
293 }
294 applySecurityProfile(containerConfig, cfg)
295 return containerConfig
296}
297
298// Deploy creates and starts a container for the given route.
299// Implements zero-downtime deployment: new container starts before old one stops.
300func (s *Service) Deploy(ctx context.Context, route domain.Route) (*domain.Container, error) {
301 ctx, span := tracer.Start(ctx, "container.deploy",
302 trace.WithAttributes(
303 attribute.String("domain", route.Domain),
304 attribute.String("image", route.Image),
305 ))
306 defer span.End()
307
308 deployStart := time.Now()
309
310 // Serialize deploys for the same domain to prevent race conditions
311 // (e.g. multiple image.pushed events + explicit deploy call from CLI).
312 unlock, err := s.acquireDomainDeployLock(ctx, route.Domain)
313 if err != nil {
314 return nil, err
315 }
316 defer unlock()
317
318 // Enrich context with use case fields for all downstream logs
319 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
320 zerowrap.FieldLayer: "usecase",
321 zerowrap.FieldUseCase: "Deploy",
322 "domain": route.Domain,
323 })
324 log := zerowrap.FromCtx(ctx)
325
326 // Record deploy metrics and trace status
327 defer s.recordDeployMetrics(ctx, span, route, deployStart, &err)
328
329 existing, hasExisting := s.resolveExistingContainer(ctx, route.Domain)
330
331 resources, err := s.prepareDeployResources(ctx, route, existing)
332 if err != nil {
333 if deployErr := s.wrapPullDeployFailure(route, existing, err); deployErr != nil {
334 err = deployErr
335 return nil, err
336 }
337 return nil, err
338 }
339
340 // Skip redundant deploy: if the existing container is already running
341 // the exact same image (by Docker image ID), return it immediately.
342 // This prevents the double-deploy caused by the event-based deploy
343 // (triggered by image.pushed) racing with the explicit CLI deploy call.
344 if hasExisting {
345 existingForSkip := existing
346 if existingForSkip.ImageID == "" && existingForSkip.Image != "" && normalizeImageRef(existingForSkip.Image) == normalizeImageRef(route.Image) {
347 existingForSkip = s.containerForRedundantCheck(ctx, existingForSkip)
348 }
349 if existingForSkip.ImageID != "" {
350 if skip, container := s.skipRedundantDeploy(ctx, existingForSkip, resources.actualImageRef, resources.envHash); skip {
351 return container, nil
352 }
353 }
354 }
355
356 newContainer, err := s.createStartedContainer(ctx, route, existing, resources)
357 if err != nil {
358 return nil, err
359 }
360
361 invalidated := s.activateDeployedContainer(ctx, route.Domain, newContainer)
362
363 // Post-switch stabilization: verify new container stays running
364 if hasExisting {
365 stable, stabilizeErr := s.stabilizeNewContainer(ctx, route.Domain, newContainer, existing)
366 if stabilizeErr != nil {
367 // Both old and new containers are dead; assign to named return so
368 // the deferred recordDeployMetrics and span see the failure.
369 err = stabilizeErr
370 return nil, err
371 }
372 if !stable {
373 // Rollback performed — old container is restored
374 return existing, nil
375 }
376 }
377
378 // Finalize old container in the background — the new container is already
379 // serving traffic, so there's no reason to block the deploy response while
380 // waiting for the old container to stop (which can take 20s if the app
381 // doesn't handle SIGTERM).
382 s.cleanupWg.Add(1)
383 go func() {
384 defer s.cleanupWg.Done()
385 s.finalizePreviousContainer(context.WithoutCancel(ctx), route.Domain, existing, hasExisting, invalidated, newContainer.ID)
386 }()
387
388 // Start container log collection (non-blocking, errors don't fail deployment)
389 s.startLogCollection(ctx, newContainer.ID, route.Domain)
390
391 log.Info().
392 Str("image", route.Image).
393 Str(zerowrap.FieldEntityID, newContainer.ID).
394 Ints("ports", newContainer.Ports).
395 Str("network", resources.networkName).
396 Bool("zero_downtime", hasExisting).
397 Msg("container deployed successfully")
398
399 return newContainer, nil
400}
401
402// recordDeployMetrics records span error status and deploy metrics at the end of a Deploy call.
403// It is called via defer and receives a pointer to the named return error so it can observe
404// the final error value after all deferred functions have run.
405func (s *Service) recordDeployMetrics(ctx context.Context, span trace.Span, route domain.Route, start time.Time, errPtr *error) {
406 err := *errPtr
407 if err != nil {
408 span.RecordError(err)
409 span.SetStatus(codes.Error, err.Error())
410 }
411 if s.metrics == nil {
412 return
413 }
414 attrs := metric.WithAttributes(
415 attribute.String("domain", route.Domain),
416 attribute.String("image", route.Image),
417 )
418 s.metrics.DeployTotal.Add(ctx, 1, attrs)
419 s.metrics.DeployDuration.Record(ctx, time.Since(start).Seconds(), attrs)
420 if err != nil {
421 s.metrics.DeployErrors.Add(ctx, 1, attrs)
422 }
423}
424
425// skipRedundantDeploy checks whether the existing container is already running
426// the same image (by Docker image ID) as the one we are about to deploy.
427// When a push triggers both an event-based deploy and an explicit CLI deploy,
428// the second one arrives after the first has already completed; this avoids
429// a full redundant create-start-readiness cycle.
430func (s *Service) skipRedundantDeploy(ctx context.Context, existing *domain.Container, actualImageRef, envHash string) (bool, *domain.Container) {
431 log := zerowrap.FromCtx(ctx)
432
433 newImageID, err := s.runtime.GetImageID(ctx, actualImageRef)
434 if err != nil {
435 log.Debug().Err(err).Msg("cannot resolve image ID for redundancy check, proceeding with deploy")
436 return false, nil
437 }
438
439 if existing.ImageID != newImageID {
440 return false, nil
441 }
442
443 // Verify the container is actually still running before skipping.
444 running, err := s.runtime.IsContainerRunning(ctx, existing.ID)
445 if err != nil || !running {
446 return false, nil
447 }
448
449 if existing.Labels != nil {
450 existingEnvHash, hasEnvHash := existing.Labels[domain.LabelEnvHash]
451 if !hasEnvHash || existingEnvHash != envHash {
452 log.Info().
453 Str("container_id", existing.ID).
454 Bool("has_env_hash", hasEnvHash).
455 Msg("env changed, proceeding with deploy despite same image")
456 return false, nil
457 }
458 } else {
459 return false, nil
460 }
461
462 log.Info().
463 Str("container_id", existing.ID).
464 Str("image_id", newImageID).
465 Msg("skipping redundant deploy: container already running this image")
466
467 return true, existing
468}
469
470func (s *Service) containerForRedundantCheck(ctx context.Context, existing *domain.Container) *domain.Container {
471 if existing == nil || existing.ImageID != "" || existing.ID == "" {
472 return existing
473 }
474
475 log := zerowrap.FromCtx(ctx)
476 inspected, err := s.runtime.InspectContainer(ctx, existing.ID)
477 if err != nil {
478 log.Debug().Err(err).Str("container_id", existing.ID).Msg("cannot inspect existing container for redundancy check")
479 return existing
480 }
481 if inspected == nil || inspected.ImageID == "" {
482 return existing
483 }
484
485 existingCopy := *existing
486 existingCopy.ImageID = inspected.ImageID
487 return &existingCopy
488}
489
490type deployResources struct {
491 networkName string
492 actualImageRef string
493 exposedPorts []int
494 imageLabels map[string]string
495 envVars []string
496 envHash string
497 volumes map[string]string
498}
499
500// resolveExistingContainer returns the currently running container for a domain.
501// It first checks the in-memory map (fast path). If not found, it queries the
502// runtime for a running container with matching name and managed label. This
503// handles cases where Gordon restarted and in-memory state is stale.
504//
505// The slow path checks canonical, -new, and -next names because a previous
506// deploy may have been interrupted (e.g. eventbus timeout, Gordon restart)
507// leaving the active container under a temp name. Canonical is preferred when
508// multiple matches exist.
509func (s *Service) resolveExistingContainer(ctx context.Context, domainName string) (*domain.Container, bool) {
510 // Fast path: check in-memory state
511 s.mu.RLock()
512 container, ok := s.containers[domainName]
513 s.mu.RUnlock()
514 if ok {
515 return container, true
516 }
517
518 // Slow path: query runtime for running containers
519 log := zerowrap.FromCtx(ctx)
520 canonicalName := managedContainerName(domainName)
521 candidateNames := map[string]bool{
522 canonicalName: true,
523 canonicalName + "-new": true,
524 canonicalName + "-next": true,
525 }
526
527 running, err := s.runtime.ListContainers(ctx, false)
528 if err != nil {
529 log.Warn().Err(err).Msg("failed to list running containers for existing container resolution")
530 return nil, false
531 }
532
533 // Prefer canonical name; fall back to any running managed temp container.
534 var best *domain.Container
535 for _, c := range running {
536 if !candidateNames[c.Name] || c.Labels[domain.LabelManaged] != "true" {
537 continue
538 }
539 if best == nil || c.Name == canonicalName {
540 best = c
541 }
542 }
543
544 if best != nil {
545 log.Info().
546 Str("container_id", best.ID).
547 Str("container_name", best.Name).
548 Msg("resolved existing container from runtime (in-memory state was stale)")
549
550 // Update in-memory state so subsequent lookups are fast
551 s.mu.Lock()
552 if _, alreadyTracked := s.containers[domainName]; !alreadyTracked {
553 s.managedCount++
554 }
555 s.containers[domainName] = best
556 s.mu.Unlock()
557
558 return best, true
559 }
560
561 return nil, false
562}
563
564func (s *Service) prepareDeployResources(ctx context.Context, route domain.Route, existing *domain.Container) (*deployResources, error) {
565 log := zerowrap.FromCtx(ctx)
566
567 existingID := ""
568 if existing != nil {
569 existingID = existing.ID
570 }
571 if err := s.cleanupOrphanedContainers(ctx, route.Domain, existingID); err != nil {
572 log.WrapErr(err, "failed to cleanup orphaned containers")
573 }
574
575 imageRef, err := s.buildValidatedImageRef(ctx, route.Image)
576 if err != nil {
577 return nil, err
578 }
579 actualImageRef, err := s.ensureImage(ctx, imageRef)
580 if err != nil {
581 return nil, err
582 }
583
584 networkName := s.getNetworkForApp(route.Domain)
585 if err := s.createNetworkIfNeeded(ctx, networkName); err != nil {
586 return nil, log.WrapErr(err, "failed to create network")
587 }
588 if err := s.deployAttachments(ctx, route.Domain, networkName); err != nil {
589 return nil, log.WrapErr(err, "failed to deploy attachments")
590 }
591
592 exposedPorts, err := s.runtime.GetImageExposedPorts(ctx, actualImageRef)
593 if err != nil {
594 log.WrapErr(err, "failed to get exposed ports, using defaults")
595 exposedPorts = []int{80, 8080, 3000}
596 }
597
598 imageLabels, err := s.runtime.GetImageLabels(ctx, actualImageRef)
599 if err != nil {
600 log.Warn().Err(err).Msg("failed to get image labels, skipping label propagation")
601 imageLabels = nil
602 }
603
604 envVars, err := s.loadEnvironment(ctx, route.Env, route.Domain, actualImageRef)
605 if err != nil {
606 return nil, err
607 }
608 envHash := hashEnvironment(envVars)
609
610 volumes, err := s.setupVolumes(ctx, route.Domain, actualImageRef, nil)
611 if err != nil {
612 return nil, err
613 }
614
615 return &deployResources{
616 networkName: networkName,
617 actualImageRef: actualImageRef,
618 exposedPorts: exposedPorts,
619 imageLabels: imageLabels,
620 envVars: envVars,
621 envHash: envHash,
622 volumes: volumes,
623 }, nil
624}
625
626func (s *Service) createStartedContainer(ctx context.Context, route domain.Route, existing *domain.Container, resources *deployResources) (*domain.Container, error) {
627 ctx, span := tracer.Start(ctx, "container.create_and_start")
628 defer span.End()
629
630 log := zerowrap.FromCtx(ctx)
631
632 containerConfig := s.buildContainerConfig(containerConfigInput{
633 Domain: route.Domain,
634 Image: route.Image,
635 ImageRef: resources.actualImageRef,
636 ExposedPorts: resources.exposedPorts,
637 EnvVars: resources.envVars,
638 EnvHash: resources.envHash,
639 Volumes: resources.volumes,
640 NetworkName: resources.networkName,
641 ImageLabels: resources.imageLabels,
642 Existing: existing,
643 })
644
645 newContainer, err := s.runtime.CreateContainer(ctx, containerConfig)
646 if err != nil {
647 return nil, log.WrapErr(err, "failed to create container")
648 }
649 if err := s.runtime.StartContainer(ctx, newContainer.ID); err != nil {
650 s.runtime.RemoveContainer(ctx, newContainer.ID, true)
651 return nil, log.WrapErr(err, "failed to start container")
652 }
653 if !domain.IsSkipReadiness(ctx) {
654 if err := s.waitForReady(ctx, newContainer.ID, containerConfig); err != nil {
655 deployErr := s.newDeployFailure(newContainer, "container failed readiness check", err, s.captureRecentContainerLogs(ctx, newContainer.ID))
656 s.cleanupFailedContainer(ctx, newContainer.ID)
657 return nil, deployErr
658 }
659 }
660
661 inspected, err := s.runtime.InspectContainer(ctx, newContainer.ID)
662 if err != nil {
663 s.cleanupFailedContainer(ctx, newContainer.ID)
664 return nil, log.WrapErr(err, "failed to inspect started container")
665 }
666
667 return inspected, nil
668}
669
670func (s *Service) newDeployFailure(container *domain.Container, cause string, err error, logs []string) error {
671 deployErr := &domain.DeployFailureError{
672 Summary: "failed to deploy",
673 Cause: cause,
674 Hint: inferDeployFailureHint(err),
675 Logs: logs,
676 Err: err,
677 }
678 if container != nil {
679 deployErr.ContainerName = container.Name
680 deployErr.ContainerID = container.ID
681 }
682
683 return deployErr
684}
685
686func (s *Service) wrapPullDeployFailure(route domain.Route, existing *domain.Container, err error) error {
687 if !isPullFailure(err) {
688 return nil
689 }
690
691 return s.newDeployFailure(&domain.Container{Name: s.deploymentContainerName(route.Domain, existing)}, "failed to pull image", err, nil)
692}
693
694func (s *Service) captureRecentContainerLogs(ctx context.Context, containerID string) []string {
695 if ctx.Err() != nil {
696 return nil
697 }
698
699 logStream, err := s.runtime.GetContainerLogs(ctx, containerID, false)
700 if err != nil {
701 return nil
702 }
703 defer func() {
704 _ = logStream.Close()
705 }()
706
707 logs, err := parseDockerLogLines(logStream, deployFailureLogLimit)
708 if err != nil {
709 return nil
710 }
711
712 return logs
713}
714
715func parseDockerLogLines(r io.Reader, maxLines int) ([]string, error) {
716 reader := bufio.NewReader(r)
717 collector := newLogLineCollector(maxLines)
718
719 header, err := reader.Peek(8)
720 if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, bufio.ErrBufferFull) {
721 return nil, err
722 }
723
724 if len(header) >= 8 && looksLikeDockerLogStream(header) {
725 return parseDockerFramedLogLines(reader, collector)
726 }
727
728 return parsePlainLogLines(reader, collector)
729}
730
731func parseDockerFramedLogLines(reader *bufio.Reader, collector *logLineCollector) ([]string, error) {
732 for {
733 frameHeader := make([]byte, 8)
734 if _, err := io.ReadFull(reader, frameHeader); err != nil {
735 if isLogStreamEOF(err) {
736 collector.flush()
737 return collector.lines, nil
738 }
739 return nil, err
740 }
741
742 size := int(binary.BigEndian.Uint32(frameHeader[4:8]))
743 if size <= 0 {
744 continue
745 }
746
747 payload, err := readLogPayload(reader, size)
748 if err != nil {
749 collector.addChunk(string(payload))
750 collector.flush()
751 if isLogStreamEOF(err) {
752 return collector.lines, nil
753 }
754 return nil, err
755 }
756
757 collector.addChunk(string(payload))
758 }
759}
760
761func parsePlainLogLines(reader *bufio.Reader, collector *logLineCollector) ([]string, error) {
762
763 scanner := bufio.NewScanner(reader)
764 for scanner.Scan() {
765 collector.addLine(scanner.Text())
766 }
767 if err := scanner.Err(); err != nil {
768 return nil, err
769 }
770
771 collector.flush()
772 return collector.lines, nil
773}
774
775func readLogPayload(reader *bufio.Reader, size int) ([]byte, error) {
776 payload := make([]byte, size)
777 _, err := io.ReadFull(reader, payload)
778 return payload, err
779}
780
781func isLogStreamEOF(err error) bool {
782 return errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF)
783}
784
785func looksLikeDockerLogStream(data []byte) bool {
786 return len(data) >= 8 && data[1] == 0 && data[2] == 0 && data[3] == 0
787}
788
789type logLineCollector struct {
790 lines []string
791 pending string
792 maxLines int
793}
794
795func newLogLineCollector(maxLines int) *logLineCollector {
796 return &logLineCollector{maxLines: maxLines}
797}
798
799func (c *logLineCollector) addChunk(chunk string) {
800 for _, part := range strings.SplitAfter(chunk, "\n") {
801 c.pending += part
802 if strings.HasSuffix(part, "\n") {
803 c.flush()
804 }
805 }
806}
807
808func (c *logLineCollector) addLine(line string) {
809 line = strings.TrimRight(line, "\r")
810 if line == "" {
811 return
812 }
813 c.lines = append(c.lines, line)
814 if c.maxLines > 0 && len(c.lines) > c.maxLines {
815 c.lines = c.lines[len(c.lines)-c.maxLines:]
816 }
817}
818
819func (c *logLineCollector) flush() {
820 if c.pending == "" {
821 return
822 }
823 c.addLine(strings.TrimRight(c.pending, "\r\n"))
824 c.pending = ""
825}
826
827func inferDeployFailureHint(err error) string {
828 if err == nil {
829 return ""
830 }
831
832 msg := strings.ToLower(err.Error())
833 switch {
834 case strings.Contains(msg, "healthcheck timeout"):
835 return "check startup logs and confirm the health endpoint becomes ready within the configured timeout"
836 case strings.Contains(msg, "unhealthy"):
837 return "inspect recent app logs and verify the healthcheck command reflects actual readiness"
838 case strings.Contains(msg, "no healthcheck detected"):
839 return "add a Docker healthcheck or switch readiness mode"
840 case isPullFailure(err), strings.Contains(msg, "access denied"), strings.Contains(msg, "unauthorized"):
841 return "verify registry auth and confirm the image exists at the requested tag"
842 default:
843 return ""
844 }
845}
846
847func isPullFailure(err error) bool {
848 if err == nil {
849 return false
850 }
851
852 if errors.Is(err, domain.ErrImagePullFailed) {
853 return true
854 }
855
856 msg := strings.ToLower(err.Error())
857 return strings.Contains(msg, "failed to pull image") ||
858 strings.Contains(msg, "pull access denied")
859}
860
861func (s *Service) activateDeployedContainer(ctx context.Context, domainName string, container *domain.Container) bool {
862 s.mu.Lock()
863 _, wasTracked := s.containers[domainName]
864 s.containers[domainName] = container
865 if !wasTracked {
866 s.managedCount++
867 }
868 s.mu.Unlock()
869
870 // Track managed container count (only increment for new domains, not replacements)
871 if !wasTracked && s.metrics != nil {
872 s.metrics.ManagedContainers.Add(ctx, 1)
873 }
874
875 s.publishContainerDeployed(ctx, domainName, container.ID)
876
877 s.mu.RLock()
878 inv := s.cacheInvalidator
879 s.mu.RUnlock()
880 if inv != nil {
881 inv.InvalidateTarget(ctx, domainName)
882 return true
883 }
884
885 return false
886}
887
888// stabilizeNewContainer monitors the new container briefly after traffic switch.
889// If it crashes during this window, rolls back to old container.
890// Returns true if stabilization succeeded, false if rollback was performed.
891// Returns an error only when both new and old containers are dead.
892func (s *Service) stabilizeNewContainer(ctx context.Context, domainName string, newContainer, oldContainer *domain.Container) (bool, error) {
893 log := zerowrap.FromCtx(ctx)
894
895 if oldContainer == nil {
896 return true, nil
897 }
898
899 s.mu.RLock()
900 delay := s.config.StabilizationDelay
901 s.mu.RUnlock()
902 if delay == 0 {
903 delay = 2 * time.Second
904 }
905
906 // Brief stabilization: verify new container is still running after delay
907 select {
908 case <-time.After(delay):
909 case <-ctx.Done():
910 return true, nil
911 }
912
913 running, err := s.runtime.IsContainerRunning(ctx, newContainer.ID)
914 if err != nil || !running {
915 log.Error().
916 Str("new_container", newContainer.ID).
917 Str("old_container", oldContainer.ID).
918 Msg("new container crashed during stabilization, rolling back to old")
919
920 // Verify old container is still running before restoring it.
921 // If both old and new are dead (e.g., OOM), returning a dead
922 // container as "existing" would leave the domain in a broken state.
923 oldRunning, oldErr := s.runtime.IsContainerRunning(ctx, oldContainer.ID)
924 if oldErr != nil || !oldRunning {
925 log.Error().
926 Str("old_container", oldContainer.ID).
927 Msg("old container is also not running, cannot rollback")
928
929 // Cleanup failed new container
930 if stopErr := s.runtime.StopContainer(ctx, newContainer.ID); stopErr != nil {
931 log.WrapErrWithFields(stopErr, "failed to stop failed new container during rollback", map[string]any{zerowrap.FieldEntityID: newContainer.ID})
932 }
933 if removeErr := s.runtime.RemoveContainer(ctx, newContainer.ID, true); removeErr != nil {
934 log.WrapErrWithFields(removeErr, "failed to remove failed new container during rollback", map[string]any{zerowrap.FieldEntityID: newContainer.ID})
935 }
936
937 return false, fmt.Errorf("stabilization failed: new container crashed and old container %s is also not running", oldContainer.ID)
938 }
939
940 // Rollback: restore old container as tracked
941 s.mu.Lock()
942 s.containers[domainName] = oldContainer
943 s.mu.Unlock()
944
945 // Re-invalidate proxy cache to point back to old
946 s.mu.RLock()
947 inv := s.cacheInvalidator
948 s.mu.RUnlock()
949 if inv != nil {
950 inv.InvalidateTarget(ctx, domainName)
951 }
952
953 // Cleanup failed new container
954 if stopErr := s.runtime.StopContainer(ctx, newContainer.ID); stopErr != nil {
955 log.WrapErrWithFields(stopErr, "failed to stop failed new container during rollback", map[string]any{zerowrap.FieldEntityID: newContainer.ID})
956 }
957 if removeErr := s.runtime.RemoveContainer(ctx, newContainer.ID, true); removeErr != nil {
958 log.WrapErrWithFields(removeErr, "failed to remove failed new container during rollback", map[string]any{zerowrap.FieldEntityID: newContainer.ID})
959 }
960
961 return false, nil
962 }
963
964 return true, nil
965}
966
967func (s *Service) finalizePreviousContainer(ctx context.Context, domainName string, existing *domain.Container, hasExisting, invalidated bool, newContainerID string) {
968 if !hasExisting {
969 return
970 }
971
972 if invalidated {
973 s.waitForDrain(ctx, existing.ID)
974 }
975
976 s.cleanupOldContainer(ctx, existing, newContainerID, domainName)
977}
978
979func (s *Service) waitForDrain(ctx context.Context, oldContainerID string) {
980 log := zerowrap.FromCtx(ctx)
981
982 s.mu.RLock()
983 cfg := s.config
984 waiter := s.drainWaiter
985 s.mu.RUnlock()
986
987 drainMode := cfg.DrainMode
988 if drainMode == "" {
989 drainMode = "delay"
990 }
991
992 shouldUseInFlight := (drainMode == "inflight" || drainMode == "auto") && waiter != nil
993 if shouldUseInFlight {
994 timeout := cfg.DrainTimeout
995 if timeout == 0 {
996 timeout = 30 * time.Second
997 }
998 drained := waiter.WaitForNoInFlight(ctx, oldContainerID, timeout)
999 if !drained {
1000 log.Warn().
1001 Str("old_container_id", oldContainerID).
1002 Dur("drain_timeout", timeout).
1003 Msg("drain wait timed out; old container may still have in-flight traffic")
1004 }
1005 return
1006 }
1007
1008 drainDelay := 2 * time.Second
1009 if cfg.DrainDelayConfigured {
1010 drainDelay = cfg.DrainDelay
1011 }
1012 if drainDelay <= 0 {
1013 return
1014 }
1015 select {
1016 case <-time.After(drainDelay):
1017 case <-ctx.Done():
1018 }
1019}
1020
1021// restartContainerWithRecovery restarts a container, reconciling stale state if needed.
1022// It returns the (possibly refreshed) container and updated attachment IDs.
1023func (s *Service) restartContainerWithRecovery(ctx context.Context, domainName string, container *domain.Container, attachmentIDs []string) (*domain.Container, []string, error) {
1024 log := zerowrap.FromCtx(ctx)
1025
1026 if err := s.runtime.RestartContainer(ctx, container.ID); err != nil {
1027 if !isContainerNotFoundError(err) {
1028 return nil, nil, log.WrapErr(err, "failed to restart container")
1029 }
1030
1031 // In-memory container ID can become stale after external runtime changes.
1032 // Re-sync and retry once with the latest tracked container.
1033 log.Warn().
1034 Err(err).
1035 Str(zerowrap.FieldEntityID, container.ID).
1036 Msg("tracked container missing during restart, attempting state reconciliation")
1037
1038 if syncErr := s.SyncContainers(ctx); syncErr != nil {
1039 log.Warn().Err(syncErr).Msg("failed to sync container state during restart recovery")
1040 }
1041
1042 s.mu.RLock()
1043 refreshed, refreshedExists := s.containers[domainName]
1044 attachmentIDs = append([]string{}, s.attachments[domainName]...)
1045 s.mu.RUnlock()
1046
1047 if !refreshedExists || refreshed == nil {
1048 return nil, nil, domain.ErrContainerNotFound
1049 }
1050 if err := s.runtime.RestartContainer(ctx, refreshed.ID); err != nil {
1051 return nil, nil, log.WrapErr(err, "failed to restart container after state reconciliation")
1052 }
1053 return refreshed, attachmentIDs, nil
1054 }
1055
1056 return container, attachmentIDs, nil
1057}
1058
1059// Restart restarts a running container for the given domain.
1060func (s *Service) Restart(ctx context.Context, domainName string, withAttachments bool) error {
1061 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1062 zerowrap.FieldLayer: "usecase",
1063 zerowrap.FieldUseCase: "Restart",
1064 "domain": domainName,
1065 })
1066 log := zerowrap.FromCtx(ctx)
1067
1068 s.mu.RLock()
1069 container, exists := s.containers[domainName]
1070 attachments := s.attachments[domainName]
1071 var attachmentIDs []string
1072 if len(attachments) > 0 {
1073 attachmentIDs = make([]string, len(attachments))
1074 copy(attachmentIDs, attachments)
1075 }
1076 s.mu.RUnlock()
1077
1078 if !exists || container == nil {
1079 return domain.ErrContainerNotFound
1080 }
1081
1082 // When attachments are requested, sync runtime state first to get accurate
1083 // attachment tracking, then check if configured attachments are deployed.
1084 // If fewer attachment containers are tracked than configured, fail early with
1085 // guidance rather than restarting the main container into a broken state
1086 // (e.g., missing database hostname).
1087 if withAttachments {
1088 if syncErr := s.SyncContainers(ctx); syncErr != nil {
1089 log.Warn().Err(syncErr).Msg("failed to sync container state before attachment check")
1090 }
1091
1092 // Re-read attachment IDs after sync
1093 s.mu.RLock()
1094 if freshAttachments := s.attachments[domainName]; len(freshAttachments) > 0 {
1095 attachmentIDs = make([]string, len(freshAttachments))
1096 copy(attachmentIDs, freshAttachments)
1097 } else {
1098 attachmentIDs = nil
1099 }
1100 s.mu.RUnlock()
1101
1102 configuredAttachments := s.resolveAttachmentsForDomain(domainName)
1103 if len(configuredAttachments) > 0 && len(attachmentIDs) < len(configuredAttachments) {
1104 missing := len(configuredAttachments) - len(attachmentIDs)
1105 return fmt.Errorf("%w: domain %q has %d configured attachment(s) but only %d deployed (%d missing)",
1106 domain.ErrAttachmentNotDeployed, domainName, len(configuredAttachments), len(attachmentIDs), missing)
1107 }
1108 }
1109
1110 // Restart the main container (with stale-state recovery)
1111 container, attachmentIDs, err := s.restartContainerWithRecovery(ctx, domainName, container, attachmentIDs)
1112 if err != nil {
1113 return err
1114 }
1115 log.Info().Str(zerowrap.FieldEntityID, container.ID).Msg("container restarted")
1116
1117 // Record restart metric
1118 if s.metrics != nil {
1119 s.metrics.ContainerRestarts.Add(ctx, 1, metric.WithAttributes(
1120 attribute.String("domain", domainName),
1121 attribute.String("source", "api"),
1122 ))
1123 }
1124
1125 // Restart attachments if requested
1126 if withAttachments && len(attachmentIDs) > 0 {
1127 for _, attachID := range attachmentIDs {
1128 if err := s.runtime.RestartContainer(ctx, attachID); err != nil {
1129 log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to restart attachment")
1130 continue // Don't fail the whole operation for one attachment
1131 }
1132 log.Info().Str("attachment_id", attachID).Msg("attachment restarted")
1133 }
1134 }
1135
1136 return nil
1137}
1138
1139// Stop stops a running container.
1140func (s *Service) Stop(ctx context.Context, containerID string) error {
1141 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1142 zerowrap.FieldLayer: "usecase",
1143 zerowrap.FieldUseCase: "Stop",
1144 zerowrap.FieldEntityID: containerID,
1145 })
1146 log := zerowrap.FromCtx(ctx)
1147
1148 // Stop log collection before stopping container
1149 if s.logWriter != nil {
1150 if err := s.logWriter.StopLogging(containerID); err != nil {
1151 log.Warn().Err(err).Msg("failed to stop container log collection")
1152 }
1153 }
1154
1155 if err := s.runtime.StopContainer(ctx, containerID); err != nil {
1156 return log.WrapErr(err, "failed to stop container")
1157 }
1158
1159 log.Info().Msg("container stopped")
1160 return nil
1161}
1162
1163// PreviewRemovedRouteCleanup reports retained runtime state for a route that is
1164// no longer configured, without mutating containers or in-memory tracking.
1165func (s *Service) PreviewRemovedRouteCleanup(ctx context.Context, domainName string) (*domain.CleanupReport, error) {
1166 canonicalDomain, ok := domain.CanonicalRouteDomain(domainName)
1167 if !ok {
1168 return &domain.CleanupReport{Domain: domainName}, domain.ErrRouteDomainInvalid
1169 }
1170 domainName = canonicalDomain
1171
1172 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1173 zerowrap.FieldLayer: "usecase",
1174 zerowrap.FieldUseCase: "PreviewRemovedRouteCleanup",
1175 "domain": domainName,
1176 })
1177 log := zerowrap.FromCtx(ctx)
1178
1179 report := &domain.CleanupReport{Domain: domainName}
1180
1181 unlock, err := s.acquireDomainDeployLock(ctx, domainName)
1182 if err != nil {
1183 return report, err
1184 }
1185 defer unlock()
1186
1187 allContainers, err := s.runtime.ListContainers(ctx, true)
1188 if err != nil {
1189 return report, log.WrapErr(err, "failed to list containers for removed route cleanup preview")
1190 }
1191
1192 routeContainers := routeContainersForDomain(allContainers, domainName)
1193 report.PreservedAttachments = preservedAttachmentsForDomain(allContainers, domainName)
1194 if len(report.PreservedAttachments) > 0 {
1195 report.Hints = append(report.Hints, "attachments preserved; remove or purge attachments explicitly if they are no longer needed")
1196 }
1197 for _, c := range routeContainers {
1198 report.OrphanedEntities = append(report.OrphanedEntities, domain.CleanupOrphanedEntity{
1199 Kind: "route_container",
1200 ID: c.ID,
1201 Name: c.Name,
1202 Status: c.Status,
1203 Reason: "route runtime state still exists after route removal",
1204 })
1205 }
1206
1207 return report, nil
1208}
1209
1210// ReconcileRemovedRoute removes active runtime containers for a route that was
1211// removed from configuration while preserving stateful resources for explicit
1212// follow-up cleanup. It intentionally removes only main route containers, never
1213// attachment containers or volumes.
1214func (s *Service) ReconcileRemovedRoute(ctx context.Context, domainName string) (*domain.CleanupReport, error) {
1215 canonicalDomain, ok := domain.CanonicalRouteDomain(domainName)
1216 if !ok {
1217 return &domain.CleanupReport{Domain: domainName}, domain.ErrRouteDomainInvalid
1218 }
1219 domainName = canonicalDomain
1220
1221 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1222 zerowrap.FieldLayer: "usecase",
1223 zerowrap.FieldUseCase: "ReconcileRemovedRoute",
1224 "domain": domainName,
1225 })
1226 log := zerowrap.FromCtx(ctx)
1227
1228 report := &domain.CleanupReport{Domain: domainName}
1229
1230 unlock, err := s.acquireDomainDeployLock(ctx, domainName)
1231 if err != nil {
1232 return report, err
1233 }
1234 defer unlock()
1235
1236 allContainers, err := s.runtime.ListContainers(ctx, true)
1237 if err != nil {
1238 return report, log.WrapErr(err, "failed to list containers for removed route cleanup")
1239 }
1240
1241 routeContainers := routeContainersForDomain(allContainers, domainName)
1242
1243 // Clear in-memory state before stopping containers so the monitor cannot
1244 // interpret the deliberate removal as a crash that should be restarted.
1245 s.clearRouteContainerTracking(ctx, domainName)
1246 if inv := s.proxyCacheInvalidator(); inv != nil {
1247 inv.InvalidateTarget(ctx, domainName)
1248 }
1249
1250 report.PreservedAttachments = preservedAttachmentsForDomain(allContainers, domainName)
1251 if len(report.PreservedAttachments) > 0 {
1252 report.Hints = append(report.Hints, "attachments preserved; remove or purge attachments explicitly if they are no longer needed")
1253 }
1254
1255 if len(routeContainers) == 0 {
1256 report.Warnings = append(report.Warnings, "no active route container found for removed route")
1257 return report, nil
1258 }
1259
1260 s.removeRouteContainersForCleanup(ctx, log, routeContainers, report)
1261
1262 if len(report.PartialFailures) > 0 {
1263 report.Warnings = append(report.Warnings, "route cleanup completed with partial failures; manual follow-up may be required")
1264 }
1265
1266 return report, nil
1267}
1268
1269func routeContainersForDomain(containers []*domain.Container, domainName string) []*domain.Container {
1270 var routeContainers []*domain.Container
1271 for _, c := range containers {
1272 if isManagedRouteContainerForDomain(c, domainName) {
1273 routeContainers = append(routeContainers, c)
1274 }
1275 }
1276 return routeContainers
1277}
1278
1279func preservedAttachmentsForDomain(containers []*domain.Container, domainName string) []domain.CleanupAttachment {
1280 var attachments []domain.CleanupAttachment
1281 for _, c := range containers {
1282 if isPreservedAttachmentForDomain(c, domainName) {
1283 attachments = append(attachments, domain.CleanupAttachment{
1284 Name: c.Name,
1285 Image: c.Image,
1286 ContainerID: c.ID,
1287 Status: c.Status,
1288 Owner: c.Labels[domain.LabelAttachedTo],
1289 Reason: "attachments are preserved by default; remove or purge them explicitly",
1290 })
1291 }
1292 }
1293 return attachments
1294}
1295
1296func (s *Service) removeRouteContainersForCleanup(ctx context.Context, log zerowrap.Logger, containers []*domain.Container, report *domain.CleanupReport) {
1297 for _, c := range containers {
1298 s.stopRouteLogCollectionForCleanup(log, c, report)
1299 s.stopRouteContainerForCleanup(ctx, log, c, report)
1300 if err := s.runtime.RemoveContainer(ctx, c.ID, true); err != nil {
1301 report.PartialFailures = append(report.PartialFailures, domain.CleanupFailure{Action: "remove", Kind: "container", ID: c.ID, Name: c.Name, Error: err.Error()})
1302 log.Warn().Err(err).Str(zerowrap.FieldEntityID, c.ID).Msg("failed to remove removed route container")
1303 continue
1304 }
1305 report.RemovedContainers = append(report.RemovedContainers, domain.CleanupContainer{ID: c.ID, Name: c.Name, Image: c.Image, Status: c.Status})
1306 }
1307}
1308
1309func (s *Service) stopRouteLogCollectionForCleanup(log zerowrap.Logger, c *domain.Container, report *domain.CleanupReport) {
1310 if s.logWriter == nil {
1311 return
1312 }
1313 if err := s.logWriter.StopLogging(c.ID); err != nil {
1314 report.PartialFailures = append(report.PartialFailures, domain.CleanupFailure{Action: "stop_logging", Kind: "container", ID: c.ID, Name: c.Name, Error: err.Error()})
1315 log.Warn().Err(err).Str(zerowrap.FieldEntityID, c.ID).Msg("failed to stop removed route log collection")
1316 }
1317}
1318
1319func (s *Service) stopRouteContainerForCleanup(ctx context.Context, log zerowrap.Logger, c *domain.Container, report *domain.CleanupReport) {
1320 if c.Status != string(domain.ContainerStatusRunning) {
1321 return
1322 }
1323 if err := s.runtime.StopContainer(ctx, c.ID); err != nil {
1324 report.PartialFailures = append(report.PartialFailures, domain.CleanupFailure{Action: "stop", Kind: "container", ID: c.ID, Name: c.Name, Error: err.Error()})
1325 log.Warn().Err(err).Str(zerowrap.FieldEntityID, c.ID).Msg("failed to stop removed route container")
1326 }
1327}
1328
1329func isManagedRouteContainerForDomain(c *domain.Container, domainName string) bool {
1330 if c == nil || c.Labels == nil || c.Labels[domain.LabelManaged] != "true" {
1331 return false
1332 }
1333 if c.Labels[domain.LabelAttachment] == "true" {
1334 return false
1335 }
1336 return c.Labels[domain.LabelRoute] == domainName || c.Labels[domain.LabelDomain] == domainName || c.Name == managedContainerName(domainName)
1337}
1338
1339func isPreservedAttachmentForDomain(c *domain.Container, domainName string) bool {
1340 if c == nil || c.Labels == nil || c.Labels[domain.LabelManaged] != "true" {
1341 return false
1342 }
1343 return c.Labels[domain.LabelAttachment] == "true" && c.Labels[domain.LabelAttachedTo] == domainName
1344}
1345
1346func (s *Service) proxyCacheInvalidator() out.ProxyCacheInvalidator {
1347 s.mu.RLock()
1348 defer s.mu.RUnlock()
1349 return s.cacheInvalidator
1350}
1351
1352func (s *Service) clearRouteContainerTracking(ctx context.Context, domainName string) {
1353 s.mu.Lock()
1354 removed := false
1355 if _, exists := s.containers[domainName]; exists {
1356 delete(s.containers, domainName)
1357 if s.managedCount > 0 {
1358 s.managedCount--
1359 }
1360 removed = true
1361 }
1362 s.mu.Unlock()
1363
1364 if removed && s.metrics != nil {
1365 s.metrics.ManagedContainers.Add(ctx, -1)
1366 }
1367}
1368
1369// Remove removes a container.
1370func (s *Service) Remove(ctx context.Context, containerID string, force bool) error {
1371 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1372 zerowrap.FieldLayer: "usecase",
1373 zerowrap.FieldUseCase: "Remove",
1374 zerowrap.FieldEntityID: containerID,
1375 })
1376 log := zerowrap.FromCtx(ctx)
1377
1378 // Stop log collection before removing container
1379 if s.logWriter != nil {
1380 if err := s.logWriter.StopLogging(containerID); err != nil {
1381 log.Warn().Err(err).Msg("failed to stop container log collection")
1382 }
1383 }
1384
1385 // Find domain and attachment IDs for cleanup
1386 containerDomain, attachmentIDs, cfg := s.findContainerContext(containerID)
1387
1388 if err := s.runtime.RemoveContainer(ctx, containerID, force); err != nil {
1389 return log.WrapErr(err, "failed to remove container")
1390 }
1391
1392 // Cleanup volumes
1393 if containerDomain != "" && !cfg.VolumePreserve {
1394 if err := s.cleanupVolumesForDomain(ctx, containerDomain); err != nil {
1395 log.WrapErrWithFields(err, "failed to cleanup volumes", map[string]any{"domain": containerDomain})
1396 }
1397 }
1398
1399 // Clean up attachment containers
1400 s.removeAttachments(ctx, attachmentIDs)
1401
1402 // Remove from tracking
1403 s.mu.Lock()
1404 var removedDomain string
1405 for d, c := range s.containers {
1406 if c.ID == containerID {
1407 delete(s.containers, d)
1408 removedDomain = d
1409 s.managedCount--
1410 log.Info().Str("domain", d).Msg("container removed")
1411 break
1412 }
1413 }
1414 if removedDomain != "" {
1415 delete(s.attachments, removedDomain)
1416 }
1417 s.mu.Unlock()
1418
1419 // Decrement managed container count
1420 if removedDomain != "" && s.metrics != nil {
1421 s.metrics.ManagedContainers.Add(ctx, -1)
1422 }
1423
1424 // Note: We intentionally do NOT clean up deployMu entries to avoid a race condition:
1425 // If Remove deletes the mutex entry while a concurrent Deploy is acquiring the lock,
1426 // the Deploy would create a fresh mutex, breaking serialization and allowing concurrent
1427 // deploys to the same domain. The memory footprint is acceptable (one small struct per
1428 // domain ever deployed), and this is the safer choice for correctness.
1429
1430 // Cleanup network
1431 if removedDomain != "" && cfg.NetworkIsolation {
1432 networkName := s.getNetworkForApp(removedDomain)
1433 if err := s.cleanupNetworkIfEmpty(ctx, networkName); err != nil {
1434 log.WrapErrWithFields(err, "failed to cleanup network", map[string]any{"network": networkName})
1435 }
1436 }
1437
1438 return nil
1439}
1440
1441// findContainerContext looks up the domain, attachment IDs, and config snapshot for a container.
1442func (s *Service) findContainerContext(containerID string) (string, []string, Config) {
1443 s.mu.RLock()
1444 defer s.mu.RUnlock()
1445 cfg := s.config
1446 var containerDomain string
1447 for d, c := range s.containers {
1448 if c.ID == containerID {
1449 containerDomain = d
1450 break
1451 }
1452 }
1453 var attachmentIDs []string
1454 if containerDomain != "" {
1455 attachmentIDs = append(attachmentIDs, s.attachments[containerDomain]...)
1456 }
1457 return containerDomain, attachmentIDs, cfg
1458}
1459
1460// removeAttachments stops and removes attachment containers best-effort.
1461func (s *Service) removeAttachments(ctx context.Context, attachmentIDs []string) {
1462 log := zerowrap.FromCtx(ctx)
1463 for _, attachID := range attachmentIDs {
1464 if s.logWriter != nil {
1465 if err := s.logWriter.StopLogging(attachID); err != nil {
1466 log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to stop attachment log collection")
1467 }
1468 }
1469 if err := s.runtime.StopContainer(ctx, attachID); err != nil {
1470 log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to stop attachment container")
1471 }
1472 if err := s.runtime.RemoveContainer(ctx, attachID, true); err != nil {
1473 log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to remove attachment container")
1474 }
1475 }
1476}
1477
1478// Get retrieves a container by domain.
1479func (s *Service) Get(_ context.Context, domainName string) (*domain.Container, bool) {
1480 s.mu.RLock()
1481 defer s.mu.RUnlock()
1482 container, exists := s.containers[domainName]
1483 return container, exists
1484}
1485
1486// List returns all managed containers.
1487func (s *Service) List(_ context.Context) map[string]*domain.Container {
1488 s.mu.RLock()
1489 defer s.mu.RUnlock()
1490
1491 result := make(map[string]*domain.Container, len(s.containers))
1492 maps.Copy(result, s.containers)
1493 return result
1494}
1495
1496// ListRoutesWithDetails returns routes with network and attachment info.
1497// Note: Container map is copied under lock, then external calls are made without lock.
1498// If containers are removed between copy and runtime calls, errors are handled gracefully
1499// (network becomes empty string). This trade-off avoids holding locks during I/O.
1500func (s *Service) ListRoutesWithDetails(ctx context.Context) []domain.RouteInfo {
1501 s.mu.RLock()
1502 containers := make(map[string]*domain.Container, len(s.containers))
1503 maps.Copy(containers, s.containers)
1504 s.mu.RUnlock()
1505
1506 // Fetch all attachments once to avoid N+1 queries
1507 attachmentsByDomain := s.getAllAttachments(ctx)
1508
1509 results := make([]domain.RouteInfo, 0, len(containers))
1510 for domainName, container := range containers {
1511 network := ""
1512 image := ""
1513 containerID := ""
1514 status := ""
1515 if container != nil {
1516 containerID = container.ID
1517 status = container.Status
1518 image = container.Image
1519 if container.Labels != nil {
1520 if labelImage, ok := container.Labels[domain.LabelImage]; ok && labelImage != "" {
1521 image = labelImage
1522 }
1523 }
1524 // Strip registry domain prefix from image for cleaner display
1525 image = s.stripRegistryPrefix(image)
1526 if networkName, err := s.runtime.GetContainerNetwork(ctx, container.ID); err == nil {
1527 network = networkName
1528 }
1529 }
1530
1531 results = append(results, domain.RouteInfo{
1532 Domain: domainName,
1533 Image: image,
1534 ContainerID: containerID,
1535 ContainerStatus: status,
1536 Network: network,
1537 Attachments: attachmentsByDomain[domainName],
1538 })
1539 }
1540
1541 return results
1542}
1543
1544// stripRegistryPrefix removes the configured current or legacy Gordon registry
1545// domain prefix from an image reference. For example,
1546// "reg.example.com/myapp:latest" becomes "myapp:latest" when
1547// "reg.example.com" is a current or legacy Gordon registry domain.
1548func (s *Service) stripRegistryPrefix(image string) string {
1549 s.mu.RLock()
1550 cfg := s.config
1551 s.mu.RUnlock()
1552
1553 return domain.StripKnownGordonRegistry(image, cfg.RegistryDomain, cfg.LegacyRegistryDomains)
1554}
1555
1556// ListAttachments returns attachments for a domain.
1557func (s *Service) ListAttachments(ctx context.Context, domainName string) []domain.Attachment {
1558 return s.getAttachmentsForDomain(ctx, domainName)
1559}
1560
1561// ListOrphanedAttachments returns managed attachment containers no longer configured.
1562func (s *Service) ListOrphanedAttachments(ctx context.Context) ([]domain.CleanupAttachment, error) {
1563 containers, err := s.runtime.ListContainers(ctx, true)
1564 if err != nil {
1565 return nil, fmt.Errorf("failed to list containers with runtime for orphaned attachments: %w", err)
1566 }
1567 return s.orphanedAttachmentsFromContainers(containers), nil
1568}
1569
1570// CleanupOrphanedAttachments optionally stops/removes orphaned attachment containers.
1571// When owner is non-empty, cleanup is scoped to that attachment owner.
1572func (s *Service) CleanupOrphanedAttachments(ctx context.Context, owner string, stop bool) (*domain.CleanupReport, error) {
1573 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1574 zerowrap.FieldLayer: "usecase",
1575 zerowrap.FieldUseCase: "CleanupOrphanedAttachments",
1576 "owner": owner,
1577 })
1578 log := zerowrap.FromCtx(ctx)
1579 report := &domain.CleanupReport{Domain: owner}
1580
1581 containers, err := s.runtime.ListContainers(ctx, true)
1582 if err != nil {
1583 return report, fmt.Errorf("failed to list containers with runtime for orphaned attachment cleanup: %w", err)
1584 }
1585 orphans := filterAttachmentContainersByOwner(s.orphanedAttachmentContainers(containers), owner)
1586 if !stop {
1587 report.PreservedAttachments = cleanupAttachmentsFromContainers(orphans)
1588 if len(report.PreservedAttachments) > 0 {
1589 report.Hints = append(report.Hints, "orphaned attachments preserved; rerun with --stop to stop/remove containers while preserving volumes")
1590 }
1591 return report, nil
1592 }
1593
1594 for _, c := range orphans {
1595 s.stopRouteLogCollectionForCleanup(log, c, report)
1596 s.stopRouteContainerForCleanup(ctx, log, c, report)
1597 if err := s.runtime.RemoveContainer(ctx, c.ID, true); err != nil {
1598 report.PartialFailures = append(report.PartialFailures, domain.CleanupFailure{Action: "remove", Kind: "attachment", ID: c.ID, Name: c.Name, Error: err.Error()})
1599 log.Warn().Err(err).Str(zerowrap.FieldEntityID, c.ID).Msg("failed to remove orphaned attachment")
1600 continue
1601 }
1602 report.RemovedContainers = append(report.RemovedContainers, domain.CleanupContainer{ID: c.ID, Name: c.Name, Image: attachmentImage(c), Status: c.Status})
1603 s.untrackAttachmentContainer(c.Labels[domain.LabelAttachedTo], c.ID)
1604 }
1605 if len(report.RemovedContainers) > 0 {
1606 report.Hints = append(report.Hints, "attachment volumes and data were preserved")
1607 }
1608 if len(report.PartialFailures) > 0 {
1609 report.Warnings = append(report.Warnings, "attachment cleanup completed with partial failures; manual follow-up may be required")
1610 }
1611 return report, nil
1612}
1613
1614func (s *Service) orphanedAttachmentsFromContainers(containers []*domain.Container) []domain.CleanupAttachment {
1615 return cleanupAttachmentsFromContainers(s.orphanedAttachmentContainers(containers))
1616}
1617
1618func (s *Service) orphanedAttachmentContainers(containers []*domain.Container) []*domain.Container {
1619 var out []*domain.Container
1620 for _, c := range containers {
1621 if !isManagedAttachment(c) {
1622 continue
1623 }
1624 owner := c.Labels[domain.LabelAttachedTo]
1625 image := attachmentImage(c)
1626 if owner == "" || image == "" || !s.isAttachmentConfiguredForOwner(owner, image) {
1627 out = append(out, c)
1628 }
1629 }
1630 return out
1631}
1632
1633func filterAttachmentContainersByOwner(containers []*domain.Container, owner string) []*domain.Container {
1634 if owner == "" {
1635 return containers
1636 }
1637 var out []*domain.Container
1638 for _, c := range containers {
1639 if c != nil && c.Labels != nil && strings.EqualFold(c.Labels[domain.LabelAttachedTo], owner) {
1640 out = append(out, c)
1641 }
1642 }
1643 return out
1644}
1645
1646func cleanupAttachmentsFromContainers(containers []*domain.Container) []domain.CleanupAttachment {
1647 attachments := make([]domain.CleanupAttachment, 0, len(containers))
1648 for _, c := range containers {
1649 attachments = append(attachments, domain.CleanupAttachment{
1650 Name: c.Name,
1651 Image: attachmentImage(c),
1652 ContainerID: c.ID,
1653 Status: c.Status,
1654 Owner: c.Labels[domain.LabelAttachedTo],
1655 Reason: "attachment container is no longer configured",
1656 })
1657 }
1658 return attachments
1659}
1660
1661func isManagedAttachment(c *domain.Container) bool {
1662 return c != nil && c.Labels != nil && c.Labels[domain.LabelManaged] == "true" && c.Labels[domain.LabelAttachment] == "true"
1663}
1664
1665func attachmentImage(c *domain.Container) string {
1666 if c == nil {
1667 return ""
1668 }
1669 if c.Labels != nil && c.Labels[domain.LabelImage] != "" {
1670 return c.Labels[domain.LabelImage]
1671 }
1672 return c.Image
1673}
1674
1675func (s *Service) isAttachmentConfiguredForOwner(owner, image string) bool {
1676 return slices.Contains(s.resolveAttachmentsForDomain(owner), image)
1677}
1678
1679func (s *Service) untrackAttachmentContainer(owner, containerID string) {
1680 s.mu.Lock()
1681 defer s.mu.Unlock()
1682 ids := s.attachments[owner]
1683 for i, id := range ids {
1684 if id == containerID {
1685 s.attachments[owner] = slices.Delete(ids, i, i+1)
1686 break
1687 }
1688 }
1689 if len(s.attachments[owner]) == 0 {
1690 delete(s.attachments, owner)
1691 }
1692}
1693
1694// ListNetworks returns Gordon-managed networks.
1695func (s *Service) ListNetworks(ctx context.Context) ([]*domain.NetworkInfo, error) {
1696 s.mu.RLock()
1697 cfg := s.config
1698 s.mu.RUnlock()
1699
1700 networks, err := s.runtime.ListNetworks(ctx)
1701 if err != nil {
1702 return nil, err
1703 }
1704
1705 var filtered []*domain.NetworkInfo
1706 for _, network := range networks {
1707 if strings.HasPrefix(network.Name, cfg.NetworkPrefix+"-") && network.Labels[domain.LabelManaged] == "true" {
1708 filtered = append(filtered, network)
1709 }
1710 }
1711
1712 return filtered, nil
1713}
1714
1715// HealthCheck performs health checks on all containers.
1716func (s *Service) HealthCheck(ctx context.Context) map[string]bool {
1717 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1718 zerowrap.FieldLayer: "usecase",
1719 zerowrap.FieldUseCase: "HealthCheck",
1720 })
1721 log := zerowrap.FromCtx(ctx)
1722
1723 // Copy container IDs under lock, then release before making Docker API calls
1724 s.mu.RLock()
1725 snapshot := make(map[string]string, len(s.containers))
1726 for d, c := range s.containers {
1727 snapshot[d] = c.ID
1728 }
1729 s.mu.RUnlock()
1730
1731 health := make(map[string]bool, len(snapshot))
1732 for d, id := range snapshot {
1733 running, err := s.runtime.IsContainerRunning(ctx, id)
1734 if err != nil {
1735 log.WrapErrWithFields(err, "health check failed", map[string]any{"domain": d, zerowrap.FieldEntityID: id})
1736 health[d] = false
1737 } else {
1738 health[d] = running
1739 }
1740 }
1741 return health
1742}
1743
1744// SyncContainers synchronizes containers with runtime state.
1745func isTrackedManagedContainerStatus(status string) bool {
1746 return status == "" || status == string(domain.ContainerStatusRunning) || status == "restarting"
1747}
1748
1749func (s *Service) SyncContainers(ctx context.Context) error {
1750 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1751 zerowrap.FieldLayer: "usecase",
1752 zerowrap.FieldUseCase: "SyncContainers",
1753 })
1754 log := zerowrap.FromCtx(ctx)
1755
1756 // List containers without holding lock to avoid blocking during the runtime call.
1757 // Use all=true so we can rebuild the running-container maps from a single
1758 // snapshot without a second list call.
1759 allContainers, err := s.runtime.ListContainers(ctx, true)
1760 if err != nil {
1761 return log.WrapErr(err, "failed to list containers")
1762 }
1763
1764 managed := make(map[string]*domain.Container)
1765 attachments := make(map[string][]string)
1766 for _, c := range allContainers {
1767 if !isTrackedManagedContainerStatus(c.Status) {
1768 continue
1769 }
1770 if c.Labels == nil || c.Labels[domain.LabelManaged] != "true" {
1771 continue
1772 }
1773 if c.Labels[domain.LabelAttachment] == "true" {
1774 owner := c.Labels[domain.LabelAttachedTo]
1775 if owner != "" {
1776 attachments[owner] = append(attachments[owner], c.ID)
1777 }
1778 continue
1779 }
1780 if d, ok := c.Labels[domain.LabelDomain]; ok {
1781 managed[d] = c
1782 }
1783 }
1784
1785 s.mu.Lock()
1786 s.containers = managed
1787 s.attachments = attachments
1788 newCount := int64(len(managed))
1789 delta := newCount - s.managedCount
1790 s.managedCount = newCount
1791 s.mu.Unlock()
1792
1793 // Report initial/delta managed container count to OTel UpDownCounter.
1794 if delta != 0 && s.metrics != nil {
1795 s.metrics.ManagedContainers.Add(ctx, delta)
1796 }
1797
1798 log.Info().Int(zerowrap.FieldCount, len(managed)).Msg("container state synchronized")
1799 return nil
1800}
1801
1802// AutoStart starts containers for the provided routes that aren't running.
1803// It skips readiness checks to avoid blocking boot; the background monitor
1804// handles crash recovery.
1805func (s *Service) AutoStart(ctx context.Context, routes []domain.Route) error {
1806 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1807 zerowrap.FieldLayer: "usecase",
1808 zerowrap.FieldUseCase: "AutoStart",
1809 })
1810 log := zerowrap.FromCtx(ctx)
1811
1812 log.Info().Int("route_count", len(routes)).Msg("auto-starting containers for configured routes")
1813
1814 // Filter out routes that already have a running container.
1815 var pending []domain.Route
1816 skipped := 0
1817 for _, route := range routes {
1818 if _, exists := s.Get(ctx, route.Domain); exists {
1819 log.Debug().Str("domain", route.Domain).Msg("container already running, skipping")
1820 skipped++
1821 } else {
1822 pending = append(pending, route)
1823 }
1824 }
1825
1826 if len(pending) == 0 {
1827 log.Info().Int("skipped", skipped).Msg("auto-start completed, all containers already running")
1828 return nil
1829 }
1830
1831 // Deploy all pending routes concurrently with readiness checks skipped.
1832 deployCtx := domain.WithSkipReadiness(ctx)
1833
1834 type result struct {
1835 route domain.Route
1836 err error
1837 }
1838 results := make(chan result, len(pending))
1839 sem := make(chan struct{}, 4) // limit concurrency
1840
1841 for _, route := range pending {
1842 sem <- struct{}{}
1843 go func(r domain.Route) {
1844 defer func() { <-sem }()
1845
1846 log.Info().Str("domain", r.Domain).Str("image", r.Image).Msg("auto-starting container for route")
1847
1848 if _, err := s.Deploy(deployCtx, r); err != nil {
1849 results <- result{route: r, err: err}
1850 return
1851 }
1852 results <- result{route: r}
1853 }(route)
1854 }
1855
1856 var started, errCount int
1857 for i := 0; i < len(pending); i++ {
1858 res := <-results
1859 if res.err != nil {
1860 log.Warn().Err(res.err).Str("domain", res.route.Domain).Msg("failed to auto-start container")
1861 errCount++
1862 } else {
1863 started++
1864 }
1865 }
1866
1867 log.Info().
1868 Int("started", started).
1869 Int("skipped", skipped).
1870 Int("errors", errCount).
1871 Msg("auto-start completed")
1872
1873 if errCount > 0 {
1874 return fmt.Errorf("auto-start completed with %d errors", errCount)
1875 }
1876 return nil
1877}
1878
1879// Shutdown gracefully shuts down all managed containers.
1880func (s *Service) Shutdown(ctx context.Context) error {
1881 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1882 zerowrap.FieldLayer: "usecase",
1883 zerowrap.FieldUseCase: "Shutdown",
1884 })
1885 log := zerowrap.FromCtx(ctx)
1886 log.Info().Msg("shutting down container manager...")
1887
1888 // Wait for any in-flight background cleanup goroutines to finish
1889 s.cleanupWg.Wait()
1890
1891 // Containers are left running across Gordon restarts.
1892 // SyncContainers + AutoStart will pick them back up on next boot.
1893
1894 // Close log writer to stop all log collection
1895 if s.logWriter != nil {
1896 if err := s.logWriter.Close(); err != nil {
1897 log.Warn().Err(err).Msg("failed to close container log writer")
1898 }
1899 }
1900
1901 log.Info().Msg("container manager shutdown complete")
1902 return nil
1903}
1904
1905// WaitForCleanup blocks until all background cleanup goroutines complete.
1906// Intended for use in tests to avoid mock assertion races.
1907func (s *Service) WaitForCleanup() {
1908 s.cleanupWg.Wait()
1909}
1910
1911// StartMonitor begins background monitoring of tracked containers.
1912// The monitor restarts crashed containers and detects crash loops.
1913// Safe to call multiple times; subsequent calls are no-ops.
1914func (s *Service) StartMonitor(ctx context.Context) {
1915 s.mu.Lock()
1916 if s.monitor != nil {
1917 s.mu.Unlock()
1918 return
1919 }
1920 s.monitor = newMonitor(s)
1921 m := s.monitor
1922 m.Start(ctx)
1923 s.mu.Unlock()
1924}
1925
1926// StopMonitor stops the background container monitor.
1927// Safe to call multiple times; subsequent calls are no-ops.
1928func (s *Service) StopMonitor() {
1929 s.mu.Lock()
1930 m := s.monitor
1931 s.monitor = nil
1932 s.mu.Unlock()
1933 if m != nil {
1934 m.Stop()
1935 }
1936}
1937
1938// UpdateConfig updates the service configuration.
1939func (s *Service) UpdateConfig(config Config) {
1940 s.mu.Lock()
1941 s.config = config
1942 s.mu.Unlock()
1943}
1944
1945// UpdateAttachments updates only the attachment configuration in the service.
1946// This is called after a config reload to propagate attachment changes without restart.
1947// The incoming map is deep-copied so external callers cannot mutate service state.
1948func (s *Service) UpdateAttachments(attachments map[string][]string) {
1949 var copied map[string][]string
1950 if attachments != nil {
1951 copied = make(map[string][]string, len(attachments))
1952 for k, v := range attachments {
1953 sl := make([]string, len(v))
1954 copy(sl, v)
1955 copied[k] = sl
1956 }
1957 }
1958 s.mu.Lock()
1959 s.config.Attachments = copied
1960 s.mu.Unlock()
1961}
1962
1963// Helper methods
1964
1965// cleanupFailedContainer stops and removes a container that failed to start properly.
1966func (s *Service) cleanupFailedContainer(ctx context.Context, containerID string) {
1967 cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), failedContainerCleanupTimeout)
1968 defer cancel()
1969
1970 log := zerowrap.FromCtx(cleanupCtx)
1971 if err := s.runtime.StopContainer(cleanupCtx, containerID); err != nil {
1972 log.Warn().Err(err).Str(zerowrap.FieldEntityID, containerID).Msg("failed to stop container after failure")
1973 }
1974 if err := s.runtime.RemoveContainer(cleanupCtx, containerID, true); err != nil {
1975 log.Warn().Err(err).Str(zerowrap.FieldEntityID, containerID).Msg("failed to remove container after failure")
1976 }
1977}
1978
1979// cleanupOldContainer stops and removes an old container after zero-downtime switch.
1980// It also renames the new container to the canonical name.
1981func (s *Service) cleanupOldContainer(ctx context.Context, old *domain.Container, newContainerID, domainName string) {
1982 log := zerowrap.FromCtx(ctx)
1983 log.Info().Str(zerowrap.FieldEntityID, old.ID).Msg("stopping old container after zero-downtime switch")
1984
1985 if s.logWriter != nil {
1986 if err := s.logWriter.StopLogging(old.ID); err != nil {
1987 log.Warn().Err(err).Str(zerowrap.FieldEntityID, old.ID).Msg("failed to stop logging for old container")
1988 }
1989 }
1990
1991 if err := s.runtime.StopContainer(ctx, old.ID); err != nil {
1992 log.Warn().Err(err).Str(zerowrap.FieldEntityID, old.ID).Msg("failed to stop old container")
1993 }
1994
1995 if err := s.runtime.RemoveContainer(ctx, old.ID, true); err != nil {
1996 log.Warn().Err(err).Str(zerowrap.FieldEntityID, old.ID).Msg("failed to remove old container")
1997 }
1998
1999 // Rename new container to canonical name
2000 canonicalName := managedContainerName(domainName)
2001 if err := s.runtime.RenameContainer(ctx, newContainerID, canonicalName); err != nil {
2002 log.Warn().Err(err).Str("canonical_name", canonicalName).Msg("failed to rename container to canonical name")
2003 }
2004}
2005
2006// startLogCollection starts log collection for a container in the background.
2007// Errors are logged but don't fail the calling operation.
2008func (s *Service) startLogCollection(ctx context.Context, containerID, domainName string) {
2009 if s.logWriter == nil {
2010 return
2011 }
2012
2013 log := zerowrap.FromCtx(ctx)
2014
2015 // Use context.WithoutCancel so the log stream outlives the HTTP request.
2016 // Without this, the Docker log stream closes when the deploy response completes.
2017 bgCtx := context.WithoutCancel(ctx)
2018
2019 logStream, err := s.runtime.GetContainerLogs(bgCtx, containerID, true)
2020 if err != nil {
2021 log.Warn().Err(err).Msg("failed to get container logs for collection")
2022 return
2023 }
2024
2025 if err := s.logWriter.StartLogging(bgCtx, containerID, domainName, logStream); err != nil {
2026 log.Warn().Err(err).Msg("failed to start container log collection")
2027 logStream.Close()
2028 }
2029}
2030
2031func (s *Service) buildValidatedImageRef(ctx context.Context, image string) (string, error) {
2032 imageRef := s.buildImageRef(image)
2033 if err := s.validateImageRef(ctx, imageRef); err != nil {
2034 return "", err
2035 }
2036 return imageRef, nil
2037}
2038
2039func (s *Service) buildImageRef(image string) string {
2040 s.mu.RLock()
2041 cfg := s.config
2042 s.mu.RUnlock()
2043
2044 if !cfg.RegistryAuthEnabled || cfg.RegistryDomain == "" {
2045 return image
2046 }
2047
2048 // Normalize registry domain by trimming any trailing slash
2049 reg := strings.TrimSuffix(cfg.RegistryDomain, "/")
2050
2051 // Check if image already has the registry domain prefix
2052 prefix := reg + "/"
2053 if strings.HasPrefix(image, prefix) {
2054 return image
2055 }
2056
2057 // Check if image already has an explicit registry (host:port/ or host.domain/).
2058 // We need to detect patterns like:
2059 // - "docker.io/library/nginx:latest" (has dot and slash)
2060 // - "localhost:5001/myapp:latest" (has host:port/ pattern)
2061 // - "myregistry:5000/app:v1" (has host:port/ pattern)
2062 // - "[fd00::1]:5000/app:v1" (IPv6 with port)
2063 if hasExplicitRegistry(image) {
2064 return image
2065 }
2066
2067 return fmt.Sprintf("%s/%s", reg, image)
2068}
2069
2070// hasExplicitRegistry checks if an image reference already includes an explicit registry.
2071// Returns true for patterns like "docker.io/image", "localhost:5000/image", "localhost/image",
2072// "registry:8080/image", "[fd00::1]/image", "[fd00::1]:5000/image".
2073func hasExplicitRegistry(image string) bool {
2074 // Find the first slash which separates registry from image name
2075 slashIdx := strings.Index(image, "/")
2076 if slashIdx == -1 {
2077 return false // No slash means no registry prefix (e.g., "myapp:latest")
2078 }
2079
2080 // Extract the part before the first slash (potential registry)
2081 registryPart := image[:slashIdx]
2082
2083 // Check for bracketed IPv6 address (e.g., "[fd00::1]" or "[fd00::1]:5000")
2084 if strings.HasPrefix(registryPart, "[") {
2085 // Look for closing bracket
2086 if closeBracket := strings.Index(registryPart, "]"); closeBracket != -1 {
2087 // Valid bracketed IPv6: either ends at ] or has ]:port
2088 return true
2089 }
2090 }
2091
2092 // Check for localhost (with or without port)
2093 if registryPart == "localhost" {
2094 return true
2095 }
2096
2097 // Check for host:port pattern (e.g., "localhost:5000", "registry:8080")
2098 if colonIdx := strings.LastIndex(registryPart, ":"); colonIdx != -1 {
2099 port := registryPart[colonIdx+1:]
2100 // If everything after colon is digits, it's a port
2101 if len(port) > 0 && isNumeric(port) {
2102 return true
2103 }
2104 }
2105
2106 // Check for domain pattern (has a dot, e.g., "docker.io", "gcr.io")
2107 if strings.Contains(registryPart, ".") {
2108 return true
2109 }
2110
2111 return false
2112}
2113
2114func (s *Service) validateImageRef(ctx context.Context, imageRef string) error {
2115 if !hasExplicitRegistry(imageRef) {
2116 return nil
2117 }
2118 return s.validateExternalImageRef(ctx, imageRef, imageRegistry(imageRef))
2119}
2120
2121func (s *Service) validateImagePullRef(ctx context.Context, imageRef string) error {
2122 return s.validateExternalImageRef(ctx, imageRef, imageRegistryForPolicy(imageRef))
2123}
2124
2125func (s *Service) validateExternalImageRef(ctx context.Context, imageRef, registry string) error {
2126 s.mu.RLock()
2127 cfg := s.config
2128 s.mu.RUnlock()
2129
2130 if registry == "" {
2131 return fmt.Errorf("image %q rejected: invalid image reference", imageRef)
2132 }
2133
2134 if domain.IsGordonRegistryImageRef(imageRef, cfg.RegistryDomain, cfg.LegacyRegistryDomains) || sameRegistry(registry, cfg.RegistryDomain) {
2135 return nil
2136 }
2137 dangerous, err := isDangerousRegistryHost(ctx, imageRegistryHost(registry))
2138 if err != nil {
2139 return fmt.Errorf("image %q rejected: registry %q could not be verified: %w", imageRef, registry, err)
2140 }
2141 if dangerous {
2142 return fmt.Errorf("image %q rejected: registry %q is not allowed", imageRef, registry)
2143 }
2144 if len(cfg.AllowedRegistries) == 0 {
2145 return fmt.Errorf("image %q rejected: registry %q is not in images.allowed_registries", imageRef, registry)
2146 }
2147 for _, allowed := range cfg.AllowedRegistries {
2148 if sameRegistry(registry, allowed) {
2149 if cfg.RequireImageDigest && !isValidSha256DigestRef(imageRef) {
2150 return fmt.Errorf("image %q rejected: digest reference required", imageRef)
2151 }
2152 return nil
2153 }
2154 }
2155 return fmt.Errorf("image %q rejected: registry %q is not in images.allowed_registries", imageRef, registry)
2156}
2157
2158func imageRegistry(imageRef string) string {
2159 slashIdx := strings.Index(imageRef, "/")
2160 if slashIdx == -1 {
2161 return ""
2162 }
2163 return normalizeRegistry(imageRef[:slashIdx])
2164}
2165
2166func imageRegistryForPolicy(imageRef string) string {
2167 if hasExplicitRegistry(imageRef) {
2168 return imageRegistry(imageRef)
2169 }
2170 return "docker.io"
2171}
2172
2173// normalizeRegistry normalizes a registry string for comparison purposes only.
2174// This function is used by sameRegistry to determine if two registry strings
2175// refer to the same registry. The output intentionally strips IPv6 brackets
2176// (e.g., "[fd00::1]:5000" -> "fd00::1:5000") which is acceptable for equality
2177// checks but the result MUST NOT be used to construct or emit valid registry URLs.
2178func normalizeRegistry(registry string) string {
2179 registry = strings.ToLower(strings.TrimSuffix(strings.TrimSpace(registry), "."))
2180 registry = strings.TrimSuffix(registry, "/")
2181 if strings.HasPrefix(registry, "[") {
2182 if end := strings.Index(registry, "]"); end != -1 {
2183 host := strings.Trim(registry[:end+1], "[]")
2184 port := registry[end+1:]
2185 return host + port
2186 }
2187 }
2188 return registry
2189}
2190
2191func imageRegistryHost(registry string) string {
2192 if ip := net.ParseIP(registry); ip != nil {
2193 return registry
2194 }
2195 if h, _, err := net.SplitHostPort(registry); err == nil {
2196 return strings.Trim(h, "[]")
2197 }
2198 if strings.HasPrefix(registry, "[") && strings.Contains(registry, "]") {
2199 return strings.TrimPrefix(strings.Split(registry, "]")[0], "[")
2200 }
2201 if colon := strings.LastIndex(registry, ":"); colon != -1 && isNumeric(registry[colon+1:]) {
2202 return registry[:colon]
2203 }
2204 return registry
2205}
2206
2207func sameRegistry(a, b string) bool {
2208 return normalizeRegistry(a) == normalizeRegistry(b)
2209}
2210
2211func isDangerousRegistryHost(ctx context.Context, host string) (bool, error) {
2212 host = strings.ToLower(strings.TrimSuffix(host, "."))
2213 if host == "localhost" || host == "metadata.google.internal" {
2214 return true, nil
2215 }
2216 if ip := net.ParseIP(host); ip != nil {
2217 return isDangerousRegistryIP(ip), nil
2218 }
2219 addrs, err := net.DefaultResolver.LookupIPAddr(ctx, host)
2220 if err != nil {
2221 return false, err
2222 }
2223 for _, addr := range addrs {
2224 if isDangerousRegistryIP(addr.IP) {
2225 return true, nil
2226 }
2227 }
2228 return false, nil
2229}
2230
2231func isDangerousRegistryIP(ip net.IP) bool {
2232 return ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsUnspecified()
2233}
2234
2235func applySecurityProfile(config *domain.ContainerConfig, cfg Config) {
2236 if strings.EqualFold(cfg.SecurityProfile, "strict") {
2237 config.ReadOnlyRootFS = true
2238 config.CapDrop = []string{"ALL"}
2239 config.CapAdd = []string{"NET_BIND_SERVICE"}
2240 }
2241}
2242
2243// isNumeric checks if a string contains only digits.
2244func isNumeric(s string) bool {
2245 for _, c := range s {
2246 if c < '0' || c > '9' {
2247 return false
2248 }
2249 }
2250 return true
2251}
2252
2253func normalizePullPolicy(policy string) string {
2254 switch strings.ToLower(strings.TrimSpace(policy)) {
2255 case PullPolicyAlways:
2256 return PullPolicyAlways
2257 case PullPolicyIfTagChanged:
2258 return PullPolicyIfTagChanged
2259 case PullPolicyIfNotPresent:
2260 return PullPolicyIfNotPresent
2261 default:
2262 return PullPolicyIfNotPresent
2263 }
2264}
2265
2266func isDigestRef(imageRef string) bool {
2267 return isValidSha256DigestRef(imageRef)
2268}
2269
2270func isValidSha256DigestRef(imageRef string) bool {
2271 _, digest, ok := strings.Cut(imageRef, "@sha256:")
2272 if !ok || len(digest) != 64 {
2273 return false
2274 }
2275 _, err := hex.DecodeString(digest)
2276 return err == nil
2277}
2278
2279func (s *Service) pullRefForDeploy(ctx context.Context, imageRef string) (string, bool) {
2280 if !domain.IsInternalDeploy(ctx) {
2281 return imageRef, false
2282 }
2283 s.mu.RLock()
2284 cfg := s.config
2285 s.mu.RUnlock()
2286 return rewriteToLocalRegistry(imageRef, cfg.RegistryDomain, cfg.LegacyRegistryDomains, cfg.RegistryPort), true
2287}
2288
2289// ensureImage ensures the image is available locally, pulling if needed.
2290// Returns the image reference to use for container operations:
2291// - For digest references (@sha256:...), returns the pullRef since Docker can't tag digests
2292// - For tagged images, returns the original imageRef after tagging the pulled image
2293func (s *Service) ensureImage(ctx context.Context, imageRef string) (string, error) {
2294 ctx, span := tracer.Start(ctx, "container.ensure_image",
2295 trace.WithAttributes(attribute.String("image", imageRef)))
2296 defer span.End()
2297
2298 ctx = zerowrap.CtxWithField(ctx, "image", imageRef)
2299 log := zerowrap.FromCtx(ctx)
2300
2301 // Determine if this is an internal deploy and what reference to use for pulls.
2302 pullRef, isInternal := s.pullRefForDeploy(ctx, imageRef)
2303 if err := s.validateImagePullRef(ctx, imageRef); err != nil {
2304 return "", err
2305 }
2306 if pullRef != imageRef {
2307 log.Info().
2308 Str("original_ref", imageRef).
2309 Str("pull_ref", pullRef).
2310 Msg("internal deploy: using localhost registry for pull")
2311 }
2312
2313 found, err := s.ensureLocalImage(ctx, imageRef, pullRef)
2314 if err != nil {
2315 return "", err
2316 }
2317 if found {
2318 return imageRef, nil
2319 }
2320
2321 // Pull image
2322 log.Info().Msg("pulling image from registry")
2323
2324 if err := s.pullImage(ctx, pullRef, isInternal); err != nil {
2325 return "", err
2326 }
2327
2328 // For digest references, we can't create a tag (Docker doesn't allow it).
2329 // In this case, use the pullRef directly since the image is available under that reference.
2330 if strings.Contains(imageRef, "@sha256:") {
2331 log.Info().
2332 Str("pull_ref", pullRef).
2333 Msg("digest reference: using pull reference for container operations")
2334 return pullRef, nil
2335 }
2336
2337 if err := s.tagImageIfNeeded(ctx, pullRef, imageRef); err != nil {
2338 return "", err
2339 }
2340
2341 // Clean up the temporary pull reference tag to avoid duplicate entries
2342 if pullRef != imageRef {
2343 if err := s.runtime.UntagImage(ctx, pullRef); err != nil {
2344 // Log but don't fail - the canonical tag is already applied
2345 log.Debug().Err(err).Str("pull_ref", pullRef).Msg("failed to remove temporary pull tag")
2346 }
2347 }
2348
2349 log.Info().Msg("image pulled successfully")
2350 return imageRef, nil
2351}
2352
2353func (s *Service) ensureLocalImage(ctx context.Context, imageRef, pullRef string) (bool, error) {
2354 log := zerowrap.FromCtx(ctx)
2355
2356 // For internal deploys (image push events), always pull fresh image
2357 // because the same tag may reference new content (e.g., latest tag updated)
2358 if domain.IsInternalDeploy(ctx) {
2359 log.Info().Msg("internal deploy detected, forcing image pull to ensure latest content")
2360 return false, nil
2361 }
2362
2363 s.mu.RLock()
2364 cfg := s.config
2365 s.mu.RUnlock()
2366
2367 pullPolicy := normalizePullPolicy(cfg.PullPolicy)
2368 switch pullPolicy {
2369 case PullPolicyAlways:
2370 log.Info().Str("pull_policy", pullPolicy).Msg("pull policy forces image pull")
2371 return false, nil
2372 case PullPolicyIfTagChanged:
2373 if !isDigestRef(imageRef) {
2374 log.Info().Str("pull_policy", pullPolicy).Msg("tag reference detected, pulling to check for updates")
2375 return false, nil
2376 }
2377 }
2378
2379 localImages, err := s.runtime.ListImages(ctx)
2380 if err != nil {
2381 log.WrapErr(err, "failed to list local images, will attempt pull")
2382 return false, nil
2383 }
2384
2385 normalizedRef := normalizeImageRef(imageRef)
2386 normalizedPullRef := normalizeImageRef(pullRef)
2387 for _, img := range localImages {
2388 normalizedImage := normalizeImageRef(img)
2389 if normalizedImage == normalizedRef {
2390 log.Info().Msg("image found locally, skipping pull")
2391 return true, nil
2392 }
2393 if normalizedImage == normalizedPullRef {
2394 if err := s.tagImageIfNeeded(ctx, pullRef, imageRef); err != nil {
2395 return false, err
2396 }
2397 // Clean up the temporary pull reference tag
2398 if pullRef != imageRef {
2399 if err := s.runtime.UntagImage(ctx, pullRef); err != nil {
2400 log.Debug().Err(err).Str("pull_ref", pullRef).Msg("failed to remove temporary pull tag")
2401 }
2402 }
2403 log.Info().Msg("image found locally, skipping pull")
2404 return true, nil
2405 }
2406 }
2407
2408 return false, nil
2409}
2410
2411func (s *Service) pullImage(ctx context.Context, pullRef string, isInternal bool) error {
2412 s.mu.RLock()
2413 cfg := s.config
2414 s.mu.RUnlock()
2415
2416 log := zerowrap.FromCtx(ctx)
2417
2418 switch {
2419 case isInternal && cfg.RegistryAuthEnabled:
2420 if cfg.InternalRegistryUsername == "" || cfg.InternalRegistryPassword == "" {
2421 return log.WrapErr(fmt.Errorf("internal registry auth not configured"), "failed to pull image for internal deploy")
2422 }
2423 if err := s.pullImageWithRetry(ctx, func(ctx context.Context) error {
2424 return s.runtime.PullImageWithAuth(ctx, pullRef, cfg.InternalRegistryUsername, cfg.InternalRegistryPassword)
2425 }, isInternal); err != nil {
2426 return log.WrapErr(fmt.Errorf("%w: %w", domain.ErrImagePullFailed, err), "failed to pull image with internal auth")
2427 }
2428 case isInternal:
2429 if err := s.pullImageWithRetry(ctx, func(ctx context.Context) error {
2430 return s.runtime.PullImage(ctx, pullRef)
2431 }, isInternal); err != nil {
2432 return log.WrapErr(fmt.Errorf("%w: %w", domain.ErrImagePullFailed, err), "failed to pull image")
2433 }
2434 default:
2435 // Never send Gordon's service credentials to an external registry.
2436 // External pulls are anonymous until per-registry credentials have an
2437 // explicit trust-boundary-aware configuration path.
2438 if err := s.runtime.PullImage(ctx, pullRef); err != nil {
2439 return log.WrapErr(fmt.Errorf("%w: %w", domain.ErrImagePullFailed, err), "failed to pull image")
2440 }
2441 }
2442
2443 return nil
2444}
2445
2446func (s *Service) pullImageWithRetry(ctx context.Context, pullFn func(context.Context) error, isInternal bool) error {
2447 log := zerowrap.FromCtx(ctx)
2448
2449 for attempt := 1; attempt <= internalPullMaxAttempts; attempt++ {
2450 err := pullFn(ctx)
2451 if err == nil {
2452 return nil
2453 }
2454 if !isInternal || !isConnectionRefusedError(err) || attempt == internalPullMaxAttempts {
2455 return fmt.Errorf("internal image pull failed after %d attempts: %w", attempt, err)
2456 }
2457
2458 backoff := time.Duration(attempt) * time.Second
2459 log.Warn().
2460 Err(err).
2461 Int("attempt", attempt).
2462 Dur("backoff", backoff).
2463 Msg("internal image pull failed with connection refused, retrying")
2464
2465 select {
2466 case <-time.After(backoff):
2467 case <-ctx.Done():
2468 return ctx.Err()
2469 }
2470 }
2471 return nil
2472}
2473
2474func (s *Service) tagImageIfNeeded(ctx context.Context, sourceRef, targetRef string) error {
2475 if sourceRef == targetRef {
2476 return nil
2477 }
2478
2479 // Cannot create a tag with a digest reference - Docker/Podman doesn't allow it.
2480 // When using digest references (image@sha256:...), skip tagging as the image
2481 // is already available by its digest.
2482 if strings.Contains(targetRef, "@sha256:") {
2483 return nil
2484 }
2485
2486 log := zerowrap.FromCtx(ctx)
2487 if err := s.runtime.TagImage(ctx, sourceRef, targetRef); err != nil {
2488 return log.WrapErr(err, "failed to tag image from pull reference")
2489 }
2490 return nil
2491}
2492
2493func (s *Service) loadEnvironment(ctx context.Context, preResolved []string, domainName, imageRef string) ([]string, error) {
2494 log := zerowrap.FromCtx(ctx)
2495
2496 var userEnvVars []string
2497 if len(preResolved) > 0 {
2498 userEnvVars = preResolved
2499 } else {
2500 var err error
2501 userEnvVars, err = s.envLoader.LoadEnv(ctx, domainName)
2502 if err != nil {
2503 return nil, log.WrapErr(err, "failed to load environment variables")
2504 }
2505 }
2506
2507 dockerfileEnvVars, err := s.runtime.InspectImageEnv(ctx, imageRef)
2508 if err != nil {
2509 log.WrapErr(err, "failed to inspect image environment")
2510 dockerfileEnvVars = []string{}
2511 }
2512
2513 return mergeEnvironmentVariables(dockerfileEnvVars, userEnvVars), nil
2514}
2515
2516func (s *Service) setupVolumes(ctx context.Context, domainName, imageRef string, preferredVolumes map[string]namedVolumeMount) (map[string]string, error) {
2517 s.mu.RLock()
2518 cfg := s.config
2519 s.mu.RUnlock()
2520
2521 log := zerowrap.FromCtx(ctx)
2522 volumes, err := s.validatePreferredVolumes(ctx, preferredVolumes)
2523 if err != nil {
2524 return nil, err
2525 }
2526
2527 if !cfg.VolumeAutoCreate {
2528 return volumes, nil
2529 }
2530
2531 volumePaths, err := s.runtime.InspectImageVolumes(ctx, imageRef)
2532 if err != nil {
2533 log.WrapErr(err, "failed to inspect image volumes")
2534 return volumes, nil
2535 }
2536
2537 for _, path := range volumePaths {
2538 if _, preferred := volumes[path]; preferred {
2539 continue
2540 }
2541
2542 name := generateVolumeName(cfg.VolumePrefix, domainName, path)
2543 legacyName := legacyVolumeName(cfg.VolumePrefix, domainName, path)
2544 legacyExists, err := s.runtime.VolumeExists(ctx, legacyName)
2545 if err != nil {
2546 log.WrapErrWithFields(err, "failed to check legacy volume", map[string]any{"volume": legacyName})
2547 continue
2548 }
2549 legacyOwnership := legacyVolumeOwnershipUnverified
2550 if legacyExists {
2551 legacyOwnership, err = s.legacyVolumeOwnership(ctx, legacyName, domainName)
2552 if err != nil {
2553 log.Warn().Err(err).Str("volume", legacyName).Msg("could not verify legacy volume ownership; using stable volume name")
2554 } else if legacyOwnership == legacyVolumeOwnershipOwned {
2555 volumes[path] = legacyName
2556 continue
2557 }
2558 }
2559
2560 exists, err := s.runtime.VolumeExists(ctx, name)
2561 if err != nil {
2562 log.WrapErrWithFields(err, "failed to check volume", map[string]any{"volume": name})
2563 continue
2564 }
2565
2566 if !exists {
2567 if legacyExists && legacyOwnership != legacyVolumeOwnershipForeign {
2568 // No verified mount may mean the owning container was already removed;
2569 // it does not prove that the preserved volume belongs elsewhere.
2570 return nil, fmt.Errorf("legacy volume %q exists but its ownership cannot be verified; refusing to create a replacement volume: %w", legacyName, domain.ErrVolumeOwnershipUnverified)
2571 }
2572 if err := s.runtime.CreateVolume(ctx, name); err != nil {
2573 log.WrapErrWithFields(err, "failed to create volume", map[string]any{"volume": name})
2574 continue
2575 }
2576 log.Info().Str("volume", name).Str(zerowrap.FieldPath, path).Msg("created volume")
2577 }
2578
2579 volumes[path] = name
2580 }
2581
2582 return volumes, nil
2583}
2584
2585func (s *Service) validatePreferredVolumes(ctx context.Context, preferredVolumes map[string]namedVolumeMount) (map[string]string, error) {
2586 volumes := make(map[string]string)
2587 for path, preferred := range preferredVolumes {
2588 if preferred.Name == "" {
2589 continue
2590 }
2591 exists, err := s.runtime.VolumeExists(ctx, preferred.Name)
2592 if err != nil {
2593 return nil, fmt.Errorf("check previously mounted volume %q: %w", preferred.Name, err)
2594 }
2595 if !exists {
2596 return nil, fmt.Errorf("previously mounted volume %q no longer exists: %w", preferred.Name, domain.ErrVolumeNotFound)
2597 }
2598 volumes[path] = preferred.Name
2599 }
2600 return volumes, nil
2601}
2602
2603func (s *Service) getNetworkForApp(domainName string) string {
2604 var networkIsolation bool
2605 var networkGroups map[string][]string
2606
2607 if s.configProvider != nil {
2608 snap := s.configProvider.GetAttachmentConfig()
2609 networkGroups = snap.NetworkGroups
2610 s.mu.RLock()
2611 networkIsolation = s.config.NetworkIsolation
2612 s.mu.RUnlock()
2613 } else {
2614 s.mu.RLock()
2615 networkIsolation = s.config.NetworkIsolation
2616 networkGroups = s.config.NetworkGroups
2617 s.mu.RUnlock()
2618 }
2619
2620 if !networkIsolation {
2621 return "bridge"
2622 }
2623
2624 for groupName, domains := range networkGroups {
2625 if slices.Contains(domains, domainName) {
2626 return s.generateNetworkName(groupName)
2627 }
2628 }
2629
2630 return s.generateNetworkName(domainName)
2631}
2632
2633func (s *Service) generateNetworkName(identifier string) string {
2634 s.mu.RLock()
2635 cfg := s.config
2636 s.mu.RUnlock()
2637
2638 return fmt.Sprintf("%s-%s", cfg.NetworkPrefix, domain.StableResourceName(identifier))
2639}
2640
2641func (s *Service) createNetworkIfNeeded(ctx context.Context, networkName string) error {
2642 if networkName == "bridge" || networkName == "default" {
2643 return nil
2644 }
2645
2646 ctx = zerowrap.CtxWithField(ctx, "network", networkName)
2647 log := zerowrap.FromCtx(ctx)
2648
2649 exists, err := s.runtime.NetworkExists(ctx, networkName)
2650 if err != nil {
2651 return log.WrapErr(err, "failed to check network existence")
2652 }
2653
2654 if !exists {
2655 s.mu.RLock()
2656 internal := s.config.NetworkInternal
2657 s.mu.RUnlock()
2658 if err := s.runtime.CreateNetwork(ctx, networkName, domain.NetworkConfig{
2659 Driver: "bridge",
2660 Internal: internal,
2661 Labels: map[string]string{domain.LabelManaged: "true"},
2662 }); err != nil {
2663 return log.WrapErr(err, "failed to create network")
2664 }
2665 log.Info().Msg("created network for app isolation")
2666 }
2667
2668 return nil
2669}
2670
2671func (s *Service) cleanupOrphanedContainers(ctx context.Context, domainName string, skipContainerID string) error {
2672 log := zerowrap.FromCtx(ctx)
2673 expectedName := managedContainerName(domainName)
2674 expectedNewName := expectedName + "-new"
2675 expectedNextName := expectedName + "-next"
2676
2677 allContainers, err := s.runtime.ListContainers(ctx, true)
2678 if err != nil {
2679 return err
2680 }
2681
2682 for _, c := range allContainers {
2683 if (c.Name == expectedName || c.Name == expectedNewName || c.Name == expectedNextName) && c.ID != skipContainerID {
2684 // Without a selected active container, preserve anything that may still
2685 // be serving traffic. When an active container is selected, other active
2686 // temp-name containers are stale candidates left by interrupted deploys.
2687 isActive := c.Status == "running" || c.Status == "restarting"
2688 if isActive && (skipContainerID == "" || c.Name == expectedName) {
2689 log.Debug().
2690 Str(zerowrap.FieldEntityID, c.ID).
2691 Str("container_name", c.Name).
2692 Str(zerowrap.FieldStatus, c.Status).
2693 Msg("skipping active container during orphan cleanup")
2694 continue
2695 }
2696
2697 log.Info().Str(zerowrap.FieldEntityID, c.ID).Str("container_name", c.Name).Str(zerowrap.FieldStatus, c.Status).Msg("found orphaned container, removing")
2698
2699 if err := s.runtime.StopContainer(ctx, c.ID); err != nil {
2700 log.WrapErrWithFields(err, "failed to stop orphaned container", map[string]any{zerowrap.FieldEntityID: c.ID})
2701 }
2702
2703 if err := s.runtime.RemoveContainer(ctx, c.ID, true); err != nil {
2704 log.WrapErrWithFields(err, "failed to remove orphaned container", map[string]any{zerowrap.FieldEntityID: c.ID})
2705 }
2706 }
2707 }
2708
2709 return nil
2710}
2711
2712func (s *Service) cleanupVolumesForDomain(_ context.Context, _ string) error {
2713 return nil
2714}
2715
2716func (s *Service) cleanupNetworkIfEmpty(ctx context.Context, networkName string) error {
2717 if networkName == "bridge" || networkName == "default" {
2718 return nil
2719 }
2720
2721 ctx = zerowrap.CtxWithField(ctx, "network", networkName)
2722 log := zerowrap.FromCtx(ctx)
2723
2724 networks, err := s.runtime.ListNetworks(ctx)
2725 if err != nil {
2726 return log.WrapErr(err, "failed to list networks")
2727 }
2728
2729 for _, n := range networks {
2730 if n.Name == networkName && len(n.Containers) == 0 && n.Labels[domain.LabelManaged] == "true" {
2731 if err := s.runtime.RemoveNetwork(ctx, networkName); err != nil {
2732 return err
2733 }
2734 log.Info().Msg("cleaned up empty network")
2735 break
2736 }
2737 }
2738
2739 return nil
2740}
2741
2742func (s *Service) deployAttachments(ctx context.Context, domainName, networkName string) error {
2743 // Collect attachments from both domain-specific and network group configs
2744 attachments := s.resolveAttachmentsForDomain(domainName)
2745 if len(attachments) == 0 {
2746 return nil
2747 }
2748
2749 log := zerowrap.FromCtx(ctx)
2750 var deployed []string // track successfully deployed attachment IDs for rollback
2751 for _, svc := range attachments {
2752 if err := s.deployAttachedService(ctx, domainName, svc, networkName); err != nil {
2753 log.WrapErrWithFields(err, "failed to deploy attachment", map[string]any{zerowrap.FieldService: svc, "domain": domainName})
2754
2755 // Rollback: clean up already-deployed attachments
2756 for _, id := range deployed {
2757 if stopErr := s.runtime.StopContainer(ctx, id); stopErr != nil {
2758 log.Warn().Err(stopErr).Str("attachment_id", id).Msg("rollback: failed to stop attachment")
2759 }
2760 if rmErr := s.runtime.RemoveContainer(ctx, id, true); rmErr != nil {
2761 log.Warn().Err(rmErr).Str("attachment_id", id).Msg("rollback: failed to remove attachment")
2762 }
2763 }
2764 // Deregister rolled-back attachments
2765 if len(deployed) > 0 {
2766 s.mu.Lock()
2767 remaining := s.attachments[domainName]
2768 filtered := remaining[:0]
2769 rollbackSet := make(map[string]bool, len(deployed))
2770 for _, id := range deployed {
2771 rollbackSet[id] = true
2772 }
2773 for _, id := range remaining {
2774 if !rollbackSet[id] {
2775 filtered = append(filtered, id)
2776 }
2777 }
2778 s.attachments[domainName] = filtered
2779 s.mu.Unlock()
2780 }
2781
2782 return fmt.Errorf("failed to deploy attachment %q (rolled back %d already-deployed)", svc, len(deployed))
2783 }
2784 // Collect IDs of attachments tracked under this domain after successful deploy
2785 s.mu.RLock()
2786 ids := s.attachments[domainName]
2787 if len(ids) > 0 {
2788 latest := ids[len(ids)-1]
2789 deployed = append(deployed, latest)
2790 }
2791 s.mu.RUnlock()
2792 }
2793 return nil
2794}
2795
2796// resolveAttachmentsForDomain returns attachments for a domain by checking:
2797// 1. Direct domain attachments (attachments[domain])
2798// 2. Network group attachments (attachments[group] where domain is in network_groups[group])
2799func (s *Service) resolveAttachmentsForDomain(domainName string) []string {
2800 var attachments map[string][]string
2801 var networkGroups map[string][]string
2802
2803 if s.configProvider != nil {
2804 snap := s.configProvider.GetAttachmentConfig()
2805 attachments = snap.Attachments
2806 networkGroups = snap.NetworkGroups
2807 } else {
2808 s.mu.RLock()
2809 attachments = s.config.Attachments
2810 networkGroups = s.config.NetworkGroups
2811 s.mu.RUnlock()
2812 }
2813
2814 seen := make(map[string]bool)
2815 var result []string
2816
2817 // First, add domain-specific attachments
2818 if domainAttachments, ok := attachments[domainName]; ok {
2819 for _, img := range domainAttachments {
2820 if !seen[img] {
2821 seen[img] = true
2822 result = append(result, img)
2823 }
2824 }
2825 }
2826
2827 // Then, find which network group this domain belongs to and add group attachments
2828 for groupName, domains := range networkGroups {
2829 if slices.Contains(domains, domainName) {
2830 if groupAttachments, ok := attachments[groupName]; ok {
2831 for _, img := range groupAttachments {
2832 if !seen[img] {
2833 seen[img] = true
2834 result = append(result, img)
2835 }
2836 }
2837 }
2838 break // Domain can only be in one network group
2839 }
2840 }
2841
2842 return result
2843}
2844
2845func (s *Service) getAttachmentsForDomain(ctx context.Context, domainName string) []domain.Attachment {
2846 containers, err := s.runtime.ListContainers(ctx, true)
2847 if err != nil {
2848 return nil
2849 }
2850
2851 return s.filterAttachments(ctx, containers, domainName)
2852}
2853
2854// getAllAttachments fetches all containers once and returns attachments grouped by domain.
2855// This avoids N+1 queries when listing multiple routes.
2856func (s *Service) getAllAttachments(ctx context.Context) map[string][]domain.Attachment {
2857 containers, err := s.runtime.ListContainers(ctx, true)
2858 if err != nil {
2859 return nil
2860 }
2861
2862 result := make(map[string][]domain.Attachment)
2863 for _, container := range containers {
2864 if container.Labels == nil {
2865 continue
2866 }
2867 if container.Labels[domain.LabelAttachment] != "true" {
2868 continue
2869 }
2870 ownerDomain := container.Labels[domain.LabelAttachedTo]
2871 if ownerDomain == "" {
2872 continue
2873 }
2874 image := container.Image
2875 serviceName := container.Name
2876 if labelImage, ok := container.Labels[domain.LabelImage]; ok && labelImage != "" {
2877 image = labelImage
2878 serviceName = extractServiceName(labelImage)
2879 }
2880 // Get the container's network for display in the routes table
2881 network := ""
2882 if networkName, err := s.runtime.GetContainerNetwork(ctx, container.ID); err == nil {
2883 network = networkName
2884 }
2885 attachment := domain.Attachment{
2886 Name: serviceName,
2887 Image: s.stripRegistryPrefix(image),
2888 ContainerID: container.ID,
2889 Status: container.Status,
2890 Network: network,
2891 Ports: append([]int(nil), container.Ports...),
2892 }
2893 result[ownerDomain] = append(result[ownerDomain], attachment)
2894 }
2895
2896 return result
2897}
2898
2899// filterAttachments extracts attachments for a specific domain from a container list.
2900func (s *Service) filterAttachments(ctx context.Context, containers []*domain.Container, domainName string) []domain.Attachment {
2901 attachments := make([]domain.Attachment, 0)
2902 for _, container := range containers {
2903 if container.Labels == nil {
2904 continue
2905 }
2906 if container.Labels[domain.LabelAttachment] != "true" {
2907 continue
2908 }
2909 if container.Labels[domain.LabelAttachedTo] != domainName {
2910 continue
2911 }
2912 image := container.Image
2913 serviceName := container.Name
2914 if labelImage, ok := container.Labels[domain.LabelImage]; ok && labelImage != "" {
2915 image = labelImage
2916 serviceName = extractServiceName(labelImage)
2917 }
2918 // Get the container's network for display in the routes table
2919 network := ""
2920 if networkName, err := s.runtime.GetContainerNetwork(ctx, container.ID); err == nil {
2921 network = networkName
2922 }
2923 attachment := domain.Attachment{
2924 Name: serviceName,
2925 Image: s.stripRegistryPrefix(image),
2926 ContainerID: container.ID,
2927 Status: container.Status,
2928 Network: network,
2929 Ports: append([]int(nil), container.Ports...),
2930 }
2931 attachments = append(attachments, attachment)
2932 }
2933
2934 return attachments
2935}
2936
2937func (s *Service) deployAttachedService(ctx context.Context, ownerDomain, serviceImage, networkName string) error {
2938 log := zerowrap.FromCtx(ctx)
2939
2940 // Parse service name from image (e.g., "my-postgres:latest" → "postgres")
2941 serviceName := extractServiceName(serviceImage)
2942 containerName := fmt.Sprintf("gordon-%s-%s", sanitizeName(ownerDomain), serviceName)
2943
2944 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
2945 "attachment": serviceName,
2946 "container_name": containerName,
2947 "owner_domain": ownerDomain,
2948 })
2949
2950 // Check if already running (idempotent)
2951 // Try new name first, then fall back to legacy name for backwards compatibility
2952 existingContainer, err := s.resolveExistingAttachment(ctx, ownerDomain, serviceName)
2953 if err != nil {
2954 return err
2955 }
2956 preferredVolumes := namedVolumeMounts(existingContainer)
2957
2958 if existingContainer != nil && existingContainer.Name == containerName {
2959 shouldSkip, err := s.handleRunningAttachment(ctx, existingContainer, containerName, serviceImage)
2960 if err != nil {
2961 return err
2962 }
2963 if shouldSkip {
2964 return nil
2965 }
2966 }
2967
2968 log.Info().Str(zerowrap.FieldService, serviceImage).Msg("deploying attached service")
2969
2970 config, err := s.buildAttachmentContainerConfig(ctx, ownerDomain, serviceName, serviceImage, containerName, networkName, preferredVolumes)
2971 if err != nil {
2972 return err
2973 }
2974
2975 if err := s.removeAttachmentForReplacement(ctx, existingContainer, containerName); err != nil {
2976 return err
2977 }
2978
2979 container, err := s.runtime.CreateContainer(ctx, config)
2980 if err != nil {
2981 return log.WrapErr(err, "failed to create attachment container")
2982 }
2983
2984 // Start container
2985 if err := s.runtime.StartContainer(ctx, container.ID); err != nil {
2986 s.runtime.RemoveContainer(ctx, container.ID, true)
2987 return log.WrapErr(err, "failed to start attachment container")
2988 }
2989
2990 // Wait for attachment to be ready before proceeding
2991 if err := s.waitForAttachmentReady(ctx, container.ID, config); err != nil {
2992 log.WrapErr(err, "attachment readiness check failed, cleaning up")
2993 if stopErr := s.runtime.StopContainer(ctx, container.ID); stopErr != nil {
2994 log.Warn().Err(stopErr).Str(zerowrap.FieldEntityID, container.ID).Msg("failed to stop unready attachment")
2995 }
2996 s.runtime.RemoveContainer(ctx, container.ID, true)
2997 return fmt.Errorf("attachment %q not ready: %w", serviceName, err)
2998 }
2999 // Track attachment
3000 s.mu.Lock()
3001 s.attachments[ownerDomain] = append(s.attachments[ownerDomain], container.ID)
3002 s.mu.Unlock()
3003
3004 // Start log collection for attachment
3005 s.startLogCollection(ctx, container.ID, containerName)
3006
3007 log.Info().Str(zerowrap.FieldEntityID, container.ID).Msg("attachment deployed successfully")
3008 return nil
3009}
3010
3011func (s *Service) buildAttachmentContainerConfig(ctx context.Context, ownerDomain, serviceName, serviceImage, containerName, networkName string, preferredVolumes map[string]namedVolumeMount) (*domain.ContainerConfig, error) {
3012 imageRef, err := s.buildValidatedImageRef(ctx, serviceImage)
3013 if err != nil {
3014 return nil, err
3015 }
3016 actualImageRef, err := s.ensureImage(ctx, imageRef)
3017 if err != nil {
3018 return nil, err
3019 }
3020
3021 exposedPorts := s.attachmentExposedPorts(ctx, actualImageRef)
3022 volumes, readOnlyVolumes, err := s.attachmentVolumes(ctx, containerName, actualImageRef, preferredVolumes)
3023 if err != nil {
3024 return nil, err
3025 }
3026 envVars, err := s.attachmentEnv(ctx, containerName, actualImageRef)
3027 if err != nil {
3028 return nil, err
3029 }
3030
3031 s.mu.RLock()
3032 cfg := s.config
3033 s.mu.RUnlock()
3034
3035 config := &domain.ContainerConfig{
3036 Image: actualImageRef,
3037 Name: containerName,
3038 Hostname: serviceName,
3039 Aliases: []string{serviceName},
3040 Ports: exposedPorts,
3041 Env: envVars,
3042 Volumes: volumes,
3043 ReadOnlyVolumes: readOnlyVolumes,
3044 NetworkMode: networkName,
3045 Labels: map[string]string{
3046 domain.LabelManaged: "true",
3047 domain.LabelAttachment: "true",
3048 domain.LabelAttachedTo: ownerDomain,
3049 domain.LabelEnvHash: hashEnvironment(envVars),
3050 domain.LabelImage: serviceImage,
3051 },
3052 MemoryLimit: cfg.DefaultMemoryLimit,
3053 NanoCPUs: cfg.DefaultNanoCPUs,
3054 PidsLimit: cfg.DefaultPidsLimit,
3055 RestartPolicy: domain.RestartPolicyAlways,
3056 }
3057 applySecurityProfile(config, cfg)
3058 return config, nil
3059}
3060
3061func (s *Service) attachmentExposedPorts(ctx context.Context, imageRef string) []int {
3062 log := zerowrap.FromCtx(ctx)
3063 exposedPorts, err := s.runtime.GetImageExposedPorts(ctx, imageRef)
3064 if err != nil {
3065 log.WrapErr(err, "failed to get exposed ports for attachment, using defaults")
3066 return []int{}
3067 }
3068 return exposedPorts
3069}
3070
3071func (s *Service) attachmentVolumes(ctx context.Context, containerName, imageRef string, preferredVolumes map[string]namedVolumeMount) (map[string]string, map[string]string, error) {
3072 volumes, err := s.setupVolumes(ctx, containerName, imageRef, preferredVolumes)
3073 if err != nil {
3074 return nil, nil, fmt.Errorf("failed to setup volumes for attachment %s with image %s: %w", containerName, imageRef, err)
3075 }
3076 readOnlyVolumes := make(map[string]string)
3077 for path, preferred := range preferredVolumes {
3078 if preferred.ReadOnly && volumes[path] == preferred.Name {
3079 delete(volumes, path)
3080 readOnlyVolumes[path] = preferred.Name
3081 }
3082 }
3083 return volumes, readOnlyVolumes, nil
3084}
3085
3086type namedVolumeMount struct {
3087 Name string
3088 ReadOnly bool
3089}
3090
3091func namedVolumeMounts(container *domain.Container) map[string]namedVolumeMount {
3092 volumes := make(map[string]namedVolumeMount)
3093 if container == nil {
3094 return volumes
3095 }
3096 for _, mount := range container.VolumeMounts {
3097 if mount.Type == "volume" && mount.Name != "" && mount.Destination != "" {
3098 volumes[mount.Destination] = namedVolumeMount{Name: mount.Name, ReadOnly: mount.ReadOnly}
3099 }
3100 }
3101 return volumes
3102}
3103
3104func (s *Service) attachmentEnv(ctx context.Context, containerName, imageRef string) ([]string, error) {
3105 envVars, err := s.loadEnvironment(ctx, nil, containerName, imageRef)
3106 if err != nil {
3107 return nil, fmt.Errorf("failed to load environment for attachment %s with image %s: %w", containerName, imageRef, err)
3108 }
3109 return envVars, nil
3110}
3111
3112func (s *Service) attachmentEnvDrifted(ctx context.Context, existing *domain.Container, containerName, serviceImage string) (bool, error) {
3113 imageRef, err := s.buildValidatedImageRef(ctx, serviceImage)
3114 if err != nil {
3115 return false, err
3116 }
3117 currentEnv, err := s.loadEnvironment(ctx, nil, containerName, imageRef)
3118 if err != nil {
3119 return false, err
3120 }
3121
3122 currentHash := hashEnvironment(currentEnv)
3123 existingHash, ok := existing.Labels[domain.LabelEnvHash]
3124 if ok && existingHash == currentHash {
3125 return false, nil
3126 }
3127
3128 return true, nil
3129}
3130
3131func (s *Service) handleRunningAttachment(ctx context.Context, existing *domain.Container, containerName, serviceImage string) (bool, error) {
3132 if existing.Status != string(domain.ContainerStatusRunning) {
3133 return false, nil
3134 }
3135
3136 log := zerowrap.FromCtx(ctx)
3137
3138 // Check image drift
3139 existingImage := existing.Labels[domain.LabelImage]
3140 imageDrifted := existingImage != "" && existingImage != serviceImage
3141
3142 // Check env drift
3143 envDrifted, err := s.attachmentEnvDrifted(ctx, existing, containerName, serviceImage)
3144 if err != nil {
3145 return false, log.WrapErr(err, "failed to check attachment env drift")
3146 }
3147
3148 if !envDrifted && !imageDrifted {
3149 log.Debug().Str("container_name", containerName).Msg("attachment already running with current env and image, skipping")
3150 return true, nil
3151 }
3152
3153 if imageDrifted {
3154 log.Info().Str("container_name", containerName).Str("existing_image", existingImage).Str("desired_image", serviceImage).Msg("attachment image changed, recreating")
3155 }
3156 if envDrifted {
3157 log.Info().Str("container_name", containerName).Msg("attachment env changed, recreating")
3158 }
3159
3160 return false, nil
3161}
3162
3163func (s *Service) removeAttachmentForReplacement(ctx context.Context, existing *domain.Container, containerName string) error {
3164 if existing == nil {
3165 return nil
3166 }
3167
3168 log := zerowrap.FromCtx(ctx)
3169 if existing.Status == string(domain.ContainerStatusRunning) {
3170 log.Info().Str("container_name", containerName).Msg("stopping attachment container for replacement")
3171 if err := s.runtime.StopContainer(ctx, existing.ID); err != nil {
3172 return log.WrapErr(err, "failed to stop attachment for replacement")
3173 }
3174 }
3175 log.Info().Str("container_name", containerName).Msg("removing attachment container for replacement")
3176 if err := s.runtime.RemoveContainer(ctx, existing.ID, true); err != nil {
3177 return log.WrapErr(err, "failed to remove attachment for replacement")
3178 }
3179
3180 return nil
3181}
3182
3183func (s *Service) resolveExistingAttachment(ctx context.Context, ownerDomain, serviceName string) (*domain.Container, error) {
3184 log := zerowrap.FromCtx(ctx)
3185 containerName := fmt.Sprintf("gordon-%s-%s", sanitizeName(ownerDomain), serviceName)
3186 existingContainer := s.findContainerByName(ctx, containerName)
3187 if existingContainer != nil {
3188 if err := validateAttachmentOwnership(existingContainer, ownerDomain, containerName); err != nil {
3189 return nil, err
3190 }
3191 return existingContainer, nil
3192 }
3193
3194 containerNameLegacy := fmt.Sprintf("gordon-%s-%s", sanitizeNameLegacy(ownerDomain), serviceName)
3195 existingContainer = s.findContainerByName(ctx, containerNameLegacy)
3196 if existingContainer == nil {
3197 return nil, nil
3198 }
3199
3200 if err := validateAttachmentOwnership(existingContainer, ownerDomain, containerNameLegacy); err != nil {
3201 return nil, err
3202 }
3203 log.Info().Str("container_name", containerNameLegacy).Msg("found attachment with legacy naming, will be replaced")
3204 return existingContainer, nil
3205}
3206
3207func validateAttachmentOwnership(container *domain.Container, ownerDomain, containerName string) error {
3208 if container.Labels[domain.LabelManaged] != "true" ||
3209 container.Labels[domain.LabelAttachment] != "true" ||
3210 container.Labels[domain.LabelAttachedTo] != ownerDomain {
3211 return fmt.Errorf("%w: attachment name %q is owned by another workload", domain.ErrAttachmentOwnershipMismatch, containerName)
3212 }
3213 return nil
3214}
3215
3216// Utility functions
3217
3218func managedContainerName(domainName string) string {
3219 return "gordon-" + domainName
3220}
3221
3222func normalizeImageRef(image string) string {
3223 if isDigestRef(image) {
3224 return image
3225 }
3226
3227 // Extract repository from image reference, handling port numbers correctly.
3228 // Examples:
3229 // nginx:latest -> docker.io/library/nginx
3230 // user/repo:tag -> docker.io/user/repo
3231 // localhost:5000/image:tag -> localhost:5000/image
3232 // registry.example.com/image:tag -> registry.example.com/image
3233 //
3234 // Note: This function assumes valid Docker image references. Edge cases like
3235 // "registry.com:5000" (registry with port but no image name) are not valid
3236 // image references per Docker's naming conventions, which require at least
3237 // one path component after the registry (e.g., "registry.com:5000/image").
3238
3239 // Find the tag separator: the last colon that isn't part of a port number.
3240 // A colon is part of a port if there's no slash after it until the next colon.
3241 repo := image
3242 lastColon := strings.LastIndex(image, ":")
3243 if lastColon != -1 {
3244 afterColon := image[lastColon+1:]
3245 // If there's no slash after the colon, it's the tag separator
3246 if !strings.Contains(afterColon, "/") {
3247 repo = image[:lastColon]
3248 }
3249 }
3250
3251 if !strings.Contains(repo, "/") {
3252 return "docker.io/library/" + repo
3253 }
3254
3255 if strings.Count(repo, "/") == 1 && !strings.Contains(strings.Split(repo, "/")[0], ".") && !strings.Contains(strings.Split(repo, "/")[0], ":") {
3256 return "docker.io/" + repo
3257 }
3258
3259 return repo
3260}
3261
3262func generateVolumeName(prefix, domainName, volumePath string) string {
3263 return fmt.Sprintf("%s-%s-%s",
3264 prefix,
3265 domain.StableResourceName(domainName),
3266 domain.StableResourceName(strings.Trim(volumePath, "/")))
3267}
3268
3269func legacyVolumeName(prefix, domainName, volumePath string) string {
3270 return fmt.Sprintf("%s-%s-%s",
3271 prefix,
3272 strings.ReplaceAll(domainName, ".", "-"),
3273 strings.ReplaceAll(strings.Trim(volumePath, "/"), "/", "-"))
3274}
3275
3276type legacyVolumeOwnership uint8
3277
3278const (
3279 legacyVolumeOwnershipUnverified legacyVolumeOwnership = iota
3280 legacyVolumeOwnershipOwned
3281 legacyVolumeOwnershipForeign
3282)
3283
3284func (s *Service) legacyVolumeOwnership(ctx context.Context, volumeName, domainName string) (legacyVolumeOwnership, error) {
3285 containers, err := s.runtime.ListContainers(ctx, true)
3286 if err != nil {
3287 return legacyVolumeOwnershipUnverified, fmt.Errorf("list containers for legacy volume ownership: %w", err)
3288 }
3289 ownerFound := false
3290 foreignFound := false
3291 for _, container := range containers {
3292 if container.Labels[domain.LabelManaged] != "true" || !containerMountsVolume(container, volumeName) {
3293 continue
3294 }
3295 if container.Name == domainName || container.Labels[domain.LabelRoute] == domainName || container.Labels[domain.LabelDomain] == domainName {
3296 ownerFound = true
3297 } else {
3298 foreignFound = true
3299 }
3300 }
3301 switch {
3302 case ownerFound && !foreignFound:
3303 return legacyVolumeOwnershipOwned, nil
3304 case foreignFound && !ownerFound:
3305 return legacyVolumeOwnershipForeign, nil
3306 default:
3307 return legacyVolumeOwnershipUnverified, nil
3308 }
3309}
3310
3311func containerMountsVolume(container *domain.Container, volumeName string) bool {
3312 for _, mount := range container.VolumeMounts {
3313 if mount.Name == volumeName || mount.Source == volumeName {
3314 return true
3315 }
3316 }
3317 return false
3318}
3319
3320func mergeEnvironmentVariables(dockerfileEnv, userEnv []string) []string {
3321 envMap := make(map[string]string)
3322
3323 for _, env := range dockerfileEnv {
3324 if k, v, ok := strings.Cut(env, "="); ok {
3325 envMap[k] = v
3326 }
3327 }
3328
3329 for _, env := range userEnv {
3330 if k, v, ok := strings.Cut(env, "="); ok {
3331 envMap[k] = v
3332 }
3333 }
3334
3335 keys := make([]string, 0, len(envMap))
3336 for k := range envMap {
3337 keys = append(keys, k)
3338 }
3339 slices.Sort(keys)
3340
3341 result := make([]string, 0, len(envMap))
3342 for _, k := range keys {
3343 result = append(result, k+"="+envMap[k])
3344 }
3345
3346 return result
3347}
3348
3349// extractServiceName gets service name from image reference.
3350// "my-postgres:latest" → "postgres", "redis:7" → "redis"
3351func extractServiceName(image string) string {
3352 // Remove tag
3353 parts := strings.Split(image, ":")
3354 name := parts[0]
3355
3356 // Remove registry prefix if present
3357 if strings.Contains(name, "/") {
3358 nameParts := strings.Split(name, "/")
3359 name = nameParts[len(nameParts)-1]
3360 }
3361
3362 // Remove common prefixes like "my-"
3363 name = strings.TrimPrefix(name, "my-")
3364
3365 return name
3366}
3367
3368// sanitizeName is a convenience wrapper around domain.SanitizeDomainForContainer.
3369func sanitizeName(d string) string {
3370 return domain.SanitizeDomainForContainer(d)
3371}
3372
3373// sanitizeNameLegacy is a convenience wrapper around domain.SanitizeDomainForContainerLegacy.
3374func sanitizeNameLegacy(d string) string {
3375 return domain.SanitizeDomainForContainerLegacy(d)
3376}
3377
3378func isContainerNotFoundError(err error) bool {
3379 if err == nil {
3380 return false
3381 }
3382 if errors.Is(err, domain.ErrContainerNotFound) {
3383 return true
3384 }
3385 msg := strings.ToLower(err.Error())
3386 return strings.Contains(msg, "no such container") ||
3387 strings.Contains(msg, "no container with name or id") ||
3388 strings.Contains(msg, "container not found")
3389}
3390
3391func isConnectionRefusedError(err error) bool {
3392 if err == nil {
3393 return false
3394 }
3395 return strings.Contains(strings.ToLower(err.Error()), "connection refused")
3396}
3397
3398// findContainerByName finds a container by its name.
3399func (s *Service) findContainerByName(ctx context.Context, name string) *domain.Container {
3400 containers, err := s.runtime.ListContainers(ctx, true)
3401 if err != nil {
3402 return nil
3403 }
3404
3405 for _, c := range containers {
3406 if c.Name == name {
3407 return c
3408 }
3409 }
3410 return nil
3411}
3412
3413func (s *Service) waitForReady(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig) error {
3414 if err := s.pollContainerRunning(ctx, containerID); err != nil {
3415 return err
3416 }
3417
3418 s.mu.RLock()
3419 cfg := s.config
3420 s.mu.RUnlock()
3421
3422 readinessMode := cfg.ReadinessMode
3423 if readinessMode == "" {
3424 readinessMode = "delay"
3425 }
3426
3427 switch readinessMode {
3428 case "delay":
3429 return s.waitForReadyByDelay(ctx, containerID)
3430 case "docker-health":
3431 _, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(ctx, containerID)
3432 if err != nil {
3433 return err
3434 }
3435 if !hasHealthcheck {
3436 return errors.New("no healthcheck detected")
3437 }
3438 return s.waitForHealthy(ctx, containerID, cfg.HealthTimeout)
3439 default: // auto
3440 return s.readinessCascade(ctx, containerID, containerConfig, cfg)
3441 }
3442}
3443
3444// readinessCascade auto-detects the strongest available readiness signal:
3445// 1. Docker healthcheck (if present) → wait for healthy status
3446// 2. HTTP probe (if gordon.health label set) → GET until 2xx/3xx
3447// 3. Default HTTP probe (GET / on exposed port) → wait for 2xx/3xx
3448// 4. TCP probe (if port info available) → connect until accepted
3449// 5. Delay fallback (last resort) → waitForReadyByDelay
3450func (s *Service) readinessCascade(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) error {
3451 log := zerowrap.FromCtx(ctx)
3452
3453 // 1. Docker healthcheck
3454 _, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(ctx, containerID)
3455 if err != nil {
3456 return err
3457 }
3458 if hasHealthcheck {
3459 log.Info().Msg("readiness cascade: using Docker healthcheck")
3460 return s.waitForHealthy(ctx, containerID, cfg.HealthTimeout)
3461 }
3462
3463 // 2. HTTP probe via gordon.health label
3464 if probed, probeErr := s.tryHTTPProbe(ctx, containerID, containerConfig, cfg); probed {
3465 return probeErr
3466 }
3467
3468 // 3. Default HTTP probe on root path (if port info available)
3469 if probed, probeErr := s.tryDefaultHTTPProbe(ctx, containerID, containerConfig, cfg); probed {
3470 return probeErr
3471 }
3472
3473 // 4. TCP probe (if port info available)
3474 if probed, probeErr := s.tryTCPProbe(ctx, containerID, containerConfig, cfg); probed {
3475 return probeErr
3476 }
3477
3478 // 5. Delay fallback
3479 log.Info().Msg("readiness cascade: using delay fallback")
3480 return s.waitForReadyByDelay(ctx, containerID)
3481}
3482
3483// waitForAttachmentReady runs a reduced readiness cascade for attachments:
3484// Docker healthcheck → TCP probe → delay fallback.
3485// HTTP probes are skipped — attachments are typically databases/caches.
3486func (s *Service) waitForAttachmentReady(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig) error {
3487 if err := s.pollContainerRunning(ctx, containerID); err != nil {
3488 return err
3489 }
3490
3491 log := zerowrap.FromCtx(ctx)
3492
3493 s.mu.RLock()
3494 cfg := s.config
3495 s.mu.RUnlock()
3496
3497 // 1. Docker healthcheck (if present)
3498 _, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(ctx, containerID)
3499 if err != nil {
3500 return err
3501 }
3502 if hasHealthcheck {
3503 log.Info().Msg("attachment readiness: using Docker healthcheck")
3504 timeout := cfg.HealthTimeout
3505 if timeout == 0 {
3506 timeout = 90 * time.Second
3507 }
3508 return s.waitForHealthy(ctx, containerID, timeout)
3509 }
3510
3511 // 2. TCP probe (if port info available)
3512 if containerConfig != nil && len(containerConfig.Ports) > 0 {
3513 ip, port, probeErr := s.resolveProbeEndpoint(ctx, containerID, containerConfig)
3514 if probeErr == nil && ip != "" && port > 0 {
3515 addr := fmt.Sprintf("%s:%d", ip, port)
3516 timeout := cfg.AttachmentReadinessTimeout
3517 if timeout == 0 {
3518 timeout = 30 * time.Second
3519 }
3520 log.Info().Str("addr", addr).Dur("timeout", timeout).Msg("attachment readiness: using TCP probe")
3521 return tcpProbe(ctx, addr, timeout)
3522 }
3523 log.Debug().Err(probeErr).Msg("attachment readiness: TCP probe skipped, could not resolve endpoint")
3524 }
3525
3526 // 3. Delay fallback
3527 log.Info().Msg("attachment readiness: using delay fallback")
3528 return s.waitForReadyByDelay(ctx, containerID)
3529}
3530
3531// tryHTTPProbe attempts an HTTP probe if the gordon.health label is set.
3532// Returns (true, err) if the probe was attempted, (false, nil) if skipped.
3533func (s *Service) tryHTTPProbe(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) (bool, error) {
3534 log := zerowrap.FromCtx(ctx)
3535 if containerConfig == nil {
3536 return false, nil
3537 }
3538 healthPath, ok := containerConfig.Labels[domain.LabelHealth]
3539 if !ok || healthPath == "" {
3540 return false, nil
3541 }
3542 ip, port, probeErr := s.resolveProbeEndpoint(ctx, containerID, containerConfig)
3543 if probeErr != nil || ip == "" || port <= 0 {
3544 log.Debug().Err(probeErr).Msg("readiness cascade: HTTP probe skipped, could not resolve container endpoint")
3545 return false, nil
3546 }
3547 url := fmt.Sprintf("http://%s:%d%s", ip, port, healthPath)
3548 timeout := cfg.HTTPProbeTimeout
3549 if timeout == 0 {
3550 timeout = 60 * time.Second
3551 }
3552 log.Info().Str("url", url).Dur("timeout", timeout).Msg("readiness cascade: using HTTP probe")
3553 return true, httpProbe(ctx, url, timeout)
3554}
3555
3556// tryDefaultHTTPProbe attempts an HTTP GET on "/" using the container's
3557// exposed port. This is the default readiness check for web containers —
3558// it verifies the app is actually serving HTTP, not just accepting TCP.
3559// Returns (true, err) if the probe was attempted, (false, nil) if skipped.
3560func (s *Service) tryDefaultHTTPProbe(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) (bool, error) {
3561 log := zerowrap.FromCtx(ctx)
3562 if containerConfig == nil || len(containerConfig.Ports) == 0 {
3563 return false, nil
3564 }
3565 ip, port, probeErr := s.resolveProbeEndpoint(ctx, containerID, containerConfig)
3566 if probeErr != nil || ip == "" || port <= 0 {
3567 log.Debug().Err(probeErr).Msg("readiness cascade: default HTTP probe skipped, could not resolve container endpoint")
3568 return false, nil
3569 }
3570 url := fmt.Sprintf("http://%s:%d/", ip, port)
3571 timeout := cfg.HTTPProbeTimeout
3572 if timeout == 0 {
3573 timeout = 60 * time.Second
3574 }
3575 log.Info().Str("url", url).Dur("timeout", timeout).Msg("readiness cascade: using default HTTP alive probe")
3576 return true, httpAliveProbe(ctx, url, timeout)
3577}
3578
3579// tryTCPProbe attempts a TCP probe if port info is available.
3580// Returns (true, err) if the probe was attempted, (false, nil) if skipped.
3581func (s *Service) tryTCPProbe(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) (bool, error) {
3582 log := zerowrap.FromCtx(ctx)
3583 if containerConfig == nil || len(containerConfig.Ports) == 0 {
3584 return false, nil
3585 }
3586 ip, port, probeErr := s.resolveProbeEndpoint(ctx, containerID, containerConfig)
3587 if probeErr != nil || ip == "" || port <= 0 {
3588 log.Debug().Err(probeErr).Msg("readiness cascade: TCP probe skipped, could not resolve container endpoint")
3589 return false, nil
3590 }
3591 addr := fmt.Sprintf("%s:%d", ip, port)
3592 timeout := cfg.TCPProbeTimeout
3593 if timeout == 0 {
3594 timeout = 30 * time.Second
3595 }
3596 log.Info().Str("addr", addr).Dur("timeout", timeout).Msg("readiness cascade: using TCP probe")
3597 return true, tcpProbe(ctx, addr, timeout)
3598}
3599
3600// resolveContainerEndpoint returns a host-reachable address for probing.
3601// In rootless podman/Docker setups, container internal IPs are not routable
3602// from the host. We use the host port binding (127.0.0.1:<mapped_port>)
3603// which is always reachable. Falls back to the container's internal IP
3604// only if no host port mapping exists (e.g. host-network mode).
3605func (s *Service) resolveProbeEndpoint(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig) (string, int, error) {
3606 log := zerowrap.FromCtx(ctx)
3607
3608 internalPort := 0
3609 if containerConfig != nil {
3610 if labels := containerConfig.Labels; labels != nil {
3611 for _, key := range []string{domain.LabelProxyPort, domain.LabelPort} {
3612 if portStr, ok := labels[key]; ok && portStr != "" {
3613 port, err := strconv.Atoi(portStr)
3614 if err == nil && port > 0 && port <= 65535 {
3615 internalPort = port
3616 break
3617 }
3618 log.Warn().Str("label", key).Str("port_value", portStr).Msg("invalid probe port label value")
3619 }
3620 }
3621 }
3622 }
3623
3624 if internalPort == 0 {
3625 _, resolvedPort, err := s.runtime.GetContainerNetworkInfo(ctx, containerID)
3626 if err != nil {
3627 return "", 0, err
3628 }
3629 internalPort = resolvedPort
3630 }
3631
3632 hostPort, hostErr := s.runtime.GetContainerPort(ctx, containerID, internalPort)
3633 if hostErr == nil && hostPort > 0 {
3634 log.Debug().
3635 Int("internal_port", internalPort).
3636 Int("host_port", hostPort).
3637 Msg("resolved container endpoint via host port binding")
3638 return "127.0.0.1", hostPort, nil
3639 }
3640
3641 ip, _, fallbackErr := s.runtime.GetContainerNetworkInfo(ctx, containerID)
3642 if fallbackErr != nil {
3643 return "", 0, fallbackErr
3644 }
3645 log.Debug().
3646 Str("ip", ip).
3647 Int("port", internalPort).
3648 Msg("resolved container endpoint via internal IP (no host port binding)")
3649 return ip, internalPort, nil
3650}
3651
3652// waitForReadyByDelay waits using the legacy running+delay strategy.
3653func (s *Service) waitForReadyByDelay(ctx context.Context, containerID string) error {
3654 log := zerowrap.FromCtx(ctx)
3655
3656 s.mu.RLock()
3657 cfg := s.config
3658 s.mu.RUnlock()
3659
3660 delay := cfg.ReadinessDelay
3661 if delay == 0 {
3662 delay = 5 * time.Second
3663 }
3664
3665 log.Debug().Dur("delay", delay).Msg("waiting for container readiness")
3666 select {
3667 case <-time.After(delay):
3668 case <-ctx.Done():
3669 return ctx.Err()
3670 }
3671
3672 // Verify still running after delay
3673 running, err := s.runtime.IsContainerRunning(ctx, containerID)
3674 if err != nil {
3675 return err
3676 }
3677 if running {
3678 return nil
3679 }
3680
3681 return s.waitForRecovery(ctx, containerID)
3682}
3683
3684func (s *Service) waitForHealthy(ctx context.Context, containerID string, timeout time.Duration) error {
3685 if timeout == 0 {
3686 timeout = 90 * time.Second
3687 }
3688 waitCtx, cancel := context.WithTimeout(ctx, timeout)
3689 defer cancel()
3690
3691 var lastStatus string
3692 for {
3693 status, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(waitCtx, containerID)
3694 if err != nil {
3695 return err
3696 }
3697 if !hasHealthcheck {
3698 return errors.New("no healthcheck detected")
3699 }
3700 // Treat empty health status as transitional startup. Some runtimes can
3701 // temporarily report an empty status before first probe results.
3702 if status == "" {
3703 status = "starting"
3704 }
3705 if status == "healthy" {
3706 return nil
3707 }
3708 if status == "unhealthy" {
3709 return errors.New("container reported unhealthy")
3710 }
3711 lastStatus = status
3712
3713 select {
3714 case <-time.After(time.Second):
3715 case <-waitCtx.Done():
3716 return fmt.Errorf("container healthcheck timeout after %s (last status: %s)", timeout, lastStatus)
3717 }
3718 }
3719}
3720
3721// pollContainerRunning polls until the container is running (max 30 seconds).
3722func (s *Service) pollContainerRunning(ctx context.Context, containerID string) error {
3723 for i := 0; i < 30; i++ {
3724 running, err := s.runtime.IsContainerRunning(ctx, containerID)
3725 if err != nil {
3726 return err
3727 }
3728 if running {
3729 return nil
3730 }
3731 // Check if the container has already exited (crash) rather than still starting up.
3732 if info, inspectErr := s.runtime.InspectContainer(ctx, containerID); inspectErr == nil && info != nil && info.Status == string(domain.ContainerStatusExited) {
3733 return fmt.Errorf("container exited immediately with code %d: %w", info.ExitCode, domain.ErrContainerExited)
3734 }
3735 if i == 29 {
3736 return fmt.Errorf("container did not start within 30 seconds")
3737 }
3738 select {
3739 case <-time.After(time.Second):
3740 case <-ctx.Done():
3741 return ctx.Err()
3742 }
3743 }
3744 return fmt.Errorf("container did not start within 30 seconds")
3745}
3746
3747// waitForRecovery waits for a container to recover within the recovery window.
3748func (s *Service) waitForRecovery(ctx context.Context, containerID string) error {
3749 log := zerowrap.FromCtx(ctx)
3750 log.Warn().
3751 Dur("recovery_window", readinessRecoveryWindow).
3752 Msg("container not running after readiness delay, waiting for recovery")
3753
3754 deadline := time.Now().Add(readinessRecoveryWindow)
3755 for time.Now().Before(deadline) {
3756 select {
3757 case <-time.After(time.Second):
3758 case <-ctx.Done():
3759 return ctx.Err()
3760 }
3761
3762 running, err := s.runtime.IsContainerRunning(ctx, containerID)
3763 if err != nil {
3764 return err
3765 }
3766 if running {
3767 log.Info().Msg("container recovered during readiness recovery window")
3768 return nil
3769 }
3770 }
3771
3772 return fmt.Errorf("container not running after readiness delay and recovery window")
3773}
3774
3775// publishContainerDeployed publishes a container.deployed event.
3776func (s *Service) publishContainerDeployed(ctx context.Context, domainName, containerID string) {
3777 payload := &domain.ContainerEventPayload{
3778 ContainerID: containerID,
3779 Domain: domainName,
3780 Action: "deployed",
3781 }
3782
3783 if err := s.eventBus.Publish(domain.EventContainerDeployed, payload); err != nil {
3784 log := zerowrap.FromCtx(ctx)
3785 log.Warn().Err(err).Msg("failed to publish container deployed event")
3786 }
3787}
3788
3789// rewriteToRegistryDomain rewrites an image reference to use the configured registry domain.
3790// e.g., "myapp:latest" -> "registry.example.com/myapp:latest"
3791func rewriteToRegistryDomain(imageRef, registryDomain string) string {
3792 if registryDomain == "" {
3793 return imageRef
3794 }
3795
3796 prefix := registryDomain + "/"
3797 if strings.HasPrefix(imageRef, prefix) {
3798 return imageRef
3799 }
3800
3801 return prefix + imageRef
3802}
3803
3804// rewriteToLocalRegistry rewrites Gordon-managed image references to use the
3805// local registry address. Current and legacy Gordon registry domains are
3806// stripped before prefixing localhost:<registryPort>/. Bare refs are prefixed.
3807// External registries are preserved unchanged.
3808func rewriteToLocalRegistry(imageRef, registryDomain string, legacyRegistryDomains []string, registryPort int) string {
3809 if imageRef == "" {
3810 return imageRef
3811 }
3812
3813 localRegistry := fmt.Sprintf("localhost:%d", registryPort)
3814 localPrefix := localRegistry + "/"
3815 if strings.HasPrefix(imageRef, localPrefix) {
3816 return imageRef
3817 }
3818
3819 if domain.IsGordonRegistryImageRef(imageRef, registryDomain, legacyRegistryDomains) {
3820 return localPrefix + domain.StripKnownGordonRegistry(imageRef, registryDomain, legacyRegistryDomains)
3821 }
3822 if hasExplicitRegistry(imageRef) {
3823 return imageRef
3824 }
3825
3826 return localPrefix + imageRef
3827}