1// Package container implements the container management use case.
2package container
3
4import (
5 "bufio"
6 "context"
7 "encoding/binary"
8 "encoding/hex"
9 "errors"
10 "fmt"
11 "io"
12 "maps"
13 "net"
14 "slices"
15 "strconv"
16 "strings"
17 "sync"
18 "time"
19
20 "github.com/bnema/zerowrap"
21 "go.opentelemetry.io/otel"
22 "go.opentelemetry.io/otel/attribute"
23 "go.opentelemetry.io/otel/codes"
24 "go.opentelemetry.io/otel/metric"
25 "go.opentelemetry.io/otel/trace"
26
27 "github.com/bnema/gordon/internal/adapters/out/telemetry"
28 "github.com/bnema/gordon/internal/boundaries/out"
29 "github.com/bnema/gordon/internal/domain"
30)
31
32// Config holds configuration needed by the container service.
33type Config struct {
34 RegistryAuthEnabled bool
35 RegistryDomain string
36 LegacyRegistryDomains []string
37 RegistryPort int
38 ServiceTokenUsername string
39 ServiceToken string
40 InternalRegistryUsername string
41 InternalRegistryPassword string
42 PullPolicy string
43 VolumeAutoCreate bool
44 VolumePrefix string
45 VolumePreserve bool
46 NetworkIsolation bool
47 NetworkPrefix string
48 NetworkGroups map[string][]string
49 NetworkInternal bool
50 Attachments map[string][]string
51 AllowedRegistries []string
52 RequireImageDigest bool
53 SecurityProfile string
54 ReadinessDelay time.Duration // Delay after container starts before considering it ready
55 ReadinessMode string // Readiness strategy: auto, docker-health, delay
56 HealthTimeout time.Duration // Max wait for health-based readiness
57 DrainDelay time.Duration // Grace period after cache invalidation before stopping old container
58 DrainDelayConfigured bool // True when deploy.drain_delay was explicitly configured
59 DrainMode string // Drain strategy: auto, inflight, delay
60 DrainTimeout time.Duration // Max wait for in-flight requests to drain
61 StabilizationDelay time.Duration // Post-switch monitoring window (default 2s)
62 TCPProbeTimeout time.Duration // TCP probe timeout (default 30s)
63 HTTPProbeTimeout time.Duration // HTTP probe timeout (default 60s)
64 AttachmentReadinessTimeout time.Duration // Max wait for attachment readiness (default 30s)
65 DefaultMemoryLimit int64 // Default memory limit in bytes for containers (0 = no limit)
66 DefaultNanoCPUs int64 // Default CPU quota in nanoseconds for containers (0 = no limit)
67 DefaultPidsLimit int64 // Default max PIDs for containers (0 = no limit)
68}
69
70var tracer = otel.Tracer("gordon.container")
71
72const (
73 PullPolicyAlways = "always"
74 PullPolicyIfNotPresent = "if-not-present"
75 PullPolicyIfTagChanged = "if-tag-changed"
76
77 // readinessRecoveryWindow is an additional grace window used when a container
78 // is briefly not running at the end of readiness delay. This avoids false
79 // negatives during short startup flaps.
80 readinessRecoveryWindow = 30 * time.Second
81 failedContainerCleanupTimeout = 30 * time.Second
82 internalPullMaxAttempts = 3
83 deployFailureLogLimit = 20
84)
85
86// Service implements the ContainerService interface.
87type Service struct {
88 runtime out.ContainerRuntime
89 envLoader out.EnvLoader
90 eventBus out.EventPublisher
91 logWriter out.ContainerLogWriter
92 cacheInvalidator out.ProxyCacheInvalidator
93 drainWaiter out.ProxyDrainWaiter
94 config Config
95 configProvider AttachmentConfigProvider // live config reads for attachments/networks (may be nil)
96 metrics *telemetry.Metrics
97 containers map[string]*domain.Container
98 attachments map[string][]string // ownerDomain → []containerIDs
99 managedCount int64 // tracks UpDownCounter value for delta computation
100 mu sync.RWMutex
101 deployMu sync.Map // per-domain deploy locks (domain → *domainDeployLock)
102 cleanupWg sync.WaitGroup // tracks background old-container cleanup goroutines
103 monitor *Monitor
104}
105
106// domainDeployLock is a context-aware mutex using a buffered channel.
107// It allows Lock() to be interrupted by context cancellation.
108type domainDeployLock struct {
109 ch chan struct{}
110}
111
112// newDomainDeployLock creates a new lock that is initially unlocked.
113func newDomainDeployLock() *domainDeployLock {
114 l := &domainDeployLock{ch: make(chan struct{}, 1)}
115 l.ch <- struct{}{}
116 return l
117}
118
119// Lock acquires the lock or returns an error if the context is cancelled.
120func (l *domainDeployLock) Lock(ctx context.Context) error {
121 // Fast path: honor already-canceled contexts before blocking.
122 if err := ctx.Err(); err != nil {
123 return err
124 }
125 select {
126 case <-ctx.Done():
127 return ctx.Err()
128 case <-l.ch:
129 // Re-check context after acquisition: if canceled while we were
130 // in the select, release the token so the lock isn't consumed.
131 if err := ctx.Err(); err != nil {
132 l.ch <- struct{}{}
133 return err
134 }
135 return nil
136 }
137}
138
139// Unlock releases the lock.
140func (l *domainDeployLock) Unlock() {
141 select {
142 case l.ch <- struct{}{}:
143 default:
144 // This should never happen if Lock/Unlock are used correctly.
145 // Panic to detect double-unlocks or other misuse early.
146 panic("domainDeployLock: double unlock detected")
147 }
148}
149
150// NewService creates a new container service.
151func NewService(
152 runtime out.ContainerRuntime,
153 envLoader out.EnvLoader,
154 eventBus out.EventPublisher,
155 logWriter out.ContainerLogWriter,
156 config Config,
157 configProvider AttachmentConfigProvider,
158) *Service {
159 return &Service{
160 runtime: runtime,
161 envLoader: envLoader,
162 eventBus: eventBus,
163 logWriter: logWriter,
164 config: config,
165 configProvider: configProvider,
166 containers: make(map[string]*domain.Container),
167 attachments: make(map[string][]string),
168 }
169}
170
171// SetMetrics sets the telemetry metrics for the container service.
172func (s *Service) SetMetrics(m *telemetry.Metrics) {
173 s.metrics = m
174}
175
176// SetProxyCacheInvalidator sets the proxy cache invalidator for synchronous
177// cache invalidation during zero-downtime deployments. This must be called
178// after construction because the proxy service is created after the container
179// service during application initialization.
180func (s *Service) SetProxyCacheInvalidator(inv out.ProxyCacheInvalidator) {
181 s.mu.Lock()
182 s.cacheInvalidator = inv
183 s.mu.Unlock()
184}
185
186// SetProxyDrainWaiter sets the proxy in-flight drain waiter used during
187// zero-downtime replacement before stopping the old container.
188func (s *Service) SetProxyDrainWaiter(waiter out.ProxyDrainWaiter) {
189 s.mu.Lock()
190 s.drainWaiter = waiter
191 s.mu.Unlock()
192}
193
194// domainDeployMu returns a per-domain lock for serializing deploy operations.
195func (s *Service) domainDeployMu(domain string) *domainDeployLock {
196 v, _ := s.deployMu.LoadOrStore(domain, newDomainDeployLock())
197 // Type assertion is safe: LoadOrStore always stores *domainDeployLock values,
198 // and any concurrent LoadOrStore calls will also store *domainDeployLock.
199 return v.(*domainDeployLock)
200}
201
202// acquireDomainDeployLock acquires the per-domain deploy lock with context cancellation support.
203// Returns an unlock function that must be called to release the lock.
204func (s *Service) acquireDomainDeployLock(ctx context.Context, domain string) (func(), error) {
205 lock := s.domainDeployMu(domain)
206 if err := lock.Lock(ctx); err != nil {
207 return nil, err
208 }
209 return func() { lock.Unlock() }, nil
210}
211
212// buildContainerConfig constructs the container configuration for deployment.
213func (s *Service) deploymentContainerName(containerDomain string, existing *domain.Container) string {
214 canonicalName := managedContainerName(containerDomain)
215 if existing == nil {
216 return canonicalName
217 }
218
219 // Alternate temporary names if the tracked container was left with a temp suffix.
220 // This prevents name collisions after interrupted zero-downtime deploys.
221 newName := canonicalName + "-new"
222 nextName := canonicalName + "-next"
223 switch existing.Name {
224 case newName:
225 return nextName
226 case nextName:
227 return newName
228 default:
229 return newName
230 }
231}
232
233// containerConfigInput groups the parameters for building a container configuration.
234type containerConfigInput struct {
235 Domain string
236 Image string
237 ImageRef string
238 ExposedPorts []int
239 EnvVars []string
240 EnvHash string
241 Volumes map[string]string
242 NetworkName string
243 ImageLabels map[string]string
244 Existing *domain.Container
245}
246
247func (s *Service) buildContainerConfig(in containerConfigInput) *domain.ContainerConfig {
248 s.mu.RLock()
249 cfg := s.config
250 s.mu.RUnlock()
251
252 containerName := s.deploymentContainerName(in.Domain, in.Existing)
253 labels := map[string]string{
254 domain.LabelDomain: in.Domain,
255 domain.LabelEnvHash: in.EnvHash,
256 domain.LabelImage: in.Image,
257 domain.LabelManaged: "true",
258 domain.LabelRoute: in.Domain,
259 }
260
261 // Propagate proxy/port labels from image so readiness probes can find them
262 for _, key := range []string{domain.LabelProxyPort, domain.LabelPort, domain.LabelHealth} {
263 if v, ok := in.ImageLabels[key]; ok && v != "" {
264 labels[key] = v
265 }
266 }
267
268 // Ensure the label-specified proxy port is included in exposed ports
269 // so Docker creates a host port binding for it.
270 ports := in.ExposedPorts
271 if portStr, ok := labels[domain.LabelProxyPort]; ok {
272 if port, err := strconv.Atoi(portStr); err == nil && port > 0 && port <= 65535 {
273 if !slices.Contains(ports, port) {
274 ports = append(ports, port)
275 }
276 }
277 }
278
279 containerConfig := &domain.ContainerConfig{
280 Image: in.ImageRef,
281 Name: containerName,
282 Ports: ports,
283 Env: in.EnvVars,
284 Volumes: in.Volumes,
285 NetworkMode: in.NetworkName,
286 Hostname: in.Domain,
287 Labels: labels,
288 AutoRemove: false,
289 RestartPolicy: domain.RestartPolicyAlways,
290 MemoryLimit: cfg.DefaultMemoryLimit,
291 NanoCPUs: cfg.DefaultNanoCPUs,
292 PidsLimit: cfg.DefaultPidsLimit,
293 }
294 applySecurityProfile(containerConfig, cfg)
295 return containerConfig
296}
297
298// Deploy creates and starts a container for the given route.
299// Implements zero-downtime deployment: new container starts before old one stops.
300func (s *Service) Deploy(ctx context.Context, route domain.Route) (*domain.Container, error) {
301 ctx, span := tracer.Start(ctx, "container.deploy",
302 trace.WithAttributes(
303 attribute.String("domain", route.Domain),
304 attribute.String("image", route.Image),
305 ))
306 defer span.End()
307
308 deployStart := time.Now()
309
310 // Serialize deploys for the same domain to prevent race conditions
311 // (e.g. multiple image.pushed events + explicit deploy call from CLI).
312 unlock, err := s.acquireDomainDeployLock(ctx, route.Domain)
313 if err != nil {
314 return nil, err
315 }
316 defer unlock()
317
318 // Enrich context with use case fields for all downstream logs
319 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
320 zerowrap.FieldLayer: "usecase",
321 zerowrap.FieldUseCase: "Deploy",
322 "domain": route.Domain,
323 })
324 log := zerowrap.FromCtx(ctx)
325
326 // Record deploy metrics and trace status
327 defer s.recordDeployMetrics(ctx, span, route, deployStart, &err)
328
329 existing, hasExisting := s.resolveExistingContainer(ctx, route.Domain)
330
331 resources, err := s.prepareDeployResources(ctx, route, existing)
332 if err != nil {
333 if deployErr := s.wrapPullDeployFailure(route, existing, err); deployErr != nil {
334 err = deployErr
335 return nil, err
336 }
337 return nil, err
338 }
339
340 // Skip redundant deploy: if the existing container is already running
341 // the exact same image (by Docker image ID), return it immediately.
342 // This prevents the double-deploy caused by the event-based deploy
343 // (triggered by image.pushed) racing with the explicit CLI deploy call.
344 if hasExisting {
345 existingForSkip := existing
346 if existingForSkip.ImageID == "" && existingForSkip.Image != "" && normalizeImageRef(existingForSkip.Image) == normalizeImageRef(route.Image) {
347 existingForSkip = s.containerForRedundantCheck(ctx, existingForSkip)
348 }
349 if existingForSkip.ImageID != "" {
350 if skip, container := s.skipRedundantDeploy(ctx, existingForSkip, resources.actualImageRef, resources.envHash); skip {
351 return container, nil
352 }
353 }
354 }
355
356 newContainer, err := s.createStartedContainer(ctx, route, existing, resources)
357 if err != nil {
358 return nil, err
359 }
360
361 invalidated := s.activateDeployedContainer(ctx, route.Domain, newContainer)
362
363 // Post-switch stabilization: verify new container stays running
364 if hasExisting {
365 stable, stabilizeErr := s.stabilizeNewContainer(ctx, route.Domain, newContainer, existing)
366 if stabilizeErr != nil {
367 // Both old and new containers are dead; assign to named return so
368 // the deferred recordDeployMetrics and span see the failure.
369 err = stabilizeErr
370 return nil, err
371 }
372 if !stable {
373 // Rollback performed — old container is restored
374 return existing, nil
375 }
376 }
377
378 // Finalize old container in the background — the new container is already
379 // serving traffic, so there's no reason to block the deploy response while
380 // waiting for the old container to stop (which can take 20s if the app
381 // doesn't handle SIGTERM).
382 s.cleanupWg.Add(1)
383 go func() {
384 defer s.cleanupWg.Done()
385 s.finalizePreviousContainer(context.WithoutCancel(ctx), route.Domain, existing, hasExisting, invalidated, newContainer.ID)
386 }()
387
388 // Start container log collection (non-blocking, errors don't fail deployment)
389 s.startLogCollection(ctx, newContainer.ID, route.Domain)
390
391 log.Info().
392 Str("image", route.Image).
393 Str(zerowrap.FieldEntityID, newContainer.ID).
394 Ints("ports", newContainer.Ports).
395 Str("network", resources.networkName).
396 Bool("zero_downtime", hasExisting).
397 Msg("container deployed successfully")
398
399 return newContainer, nil
400}
401
402// recordDeployMetrics records span error status and deploy metrics at the end of a Deploy call.
403// It is called via defer and receives a pointer to the named return error so it can observe
404// the final error value after all deferred functions have run.
405func (s *Service) recordDeployMetrics(ctx context.Context, span trace.Span, route domain.Route, start time.Time, errPtr *error) {
406 err := *errPtr
407 if err != nil {
408 span.RecordError(err)
409 span.SetStatus(codes.Error, err.Error())
410 }
411 if s.metrics == nil {
412 return
413 }
414 attrs := metric.WithAttributes(
415 attribute.String("domain", route.Domain),
416 attribute.String("image", route.Image),
417 )
418 s.metrics.DeployTotal.Add(ctx, 1, attrs)
419 s.metrics.DeployDuration.Record(ctx, time.Since(start).Seconds(), attrs)
420 if err != nil {
421 s.metrics.DeployErrors.Add(ctx, 1, attrs)
422 }
423}
424
425// skipRedundantDeploy checks whether the existing container is already running
426// the same image (by Docker image ID) as the one we are about to deploy.
427// When a push triggers both an event-based deploy and an explicit CLI deploy,
428// the second one arrives after the first has already completed; this avoids
429// a full redundant create-start-readiness cycle.
430func (s *Service) skipRedundantDeploy(ctx context.Context, existing *domain.Container, actualImageRef, envHash string) (bool, *domain.Container) {
431 log := zerowrap.FromCtx(ctx)
432
433 newImageID, err := s.runtime.GetImageID(ctx, actualImageRef)
434 if err != nil {
435 log.Debug().Err(err).Msg("cannot resolve image ID for redundancy check, proceeding with deploy")
436 return false, nil
437 }
438
439 if existing.ImageID != newImageID {
440 return false, nil
441 }
442
443 // Verify the container is actually still running before skipping.
444 running, err := s.runtime.IsContainerRunning(ctx, existing.ID)
445 if err != nil || !running {
446 return false, nil
447 }
448
449 if existing.Labels != nil {
450 existingEnvHash, hasEnvHash := existing.Labels[domain.LabelEnvHash]
451 if !hasEnvHash || existingEnvHash != envHash {
452 log.Info().
453 Str("container_id", existing.ID).
454 Bool("has_env_hash", hasEnvHash).
455 Msg("env changed, proceeding with deploy despite same image")
456 return false, nil
457 }
458 } else {
459 return false, nil
460 }
461
462 log.Info().
463 Str("container_id", existing.ID).
464 Str("image_id", newImageID).
465 Msg("skipping redundant deploy: container already running this image")
466
467 return true, existing
468}
469
470func (s *Service) containerForRedundantCheck(ctx context.Context, existing *domain.Container) *domain.Container {
471 if existing == nil || existing.ImageID != "" || existing.ID == "" {
472 return existing
473 }
474
475 log := zerowrap.FromCtx(ctx)
476 inspected, err := s.runtime.InspectContainer(ctx, existing.ID)
477 if err != nil {
478 log.Debug().Err(err).Str("container_id", existing.ID).Msg("cannot inspect existing container for redundancy check")
479 return existing
480 }
481 if inspected == nil || inspected.ImageID == "" {
482 return existing
483 }
484
485 existingCopy := *existing
486 existingCopy.ImageID = inspected.ImageID
487 return &existingCopy
488}
489
490type deployResources struct {
491 networkName string
492 actualImageRef string
493 exposedPorts []int
494 imageLabels map[string]string
495 envVars []string
496 envHash string
497 volumes map[string]string
498}
499
500// resolveExistingContainer returns the currently running container for a domain.
501// It first checks the in-memory map (fast path). If not found, it queries the
502// runtime for a running container with matching name and managed label. This
503// handles cases where Gordon restarted and in-memory state is stale.
504//
505// The slow path checks canonical, -new, and -next names because a previous
506// deploy may have been interrupted (e.g. eventbus timeout, Gordon restart)
507// leaving the active container under a temp name. Canonical is preferred when
508// multiple matches exist.
509func (s *Service) resolveExistingContainer(ctx context.Context, domainName string) (*domain.Container, bool) {
510 // Fast path: check in-memory state
511 s.mu.RLock()
512 container, ok := s.containers[domainName]
513 s.mu.RUnlock()
514 if ok {
515 return container, true
516 }
517
518 // Slow path: query runtime for running containers
519 log := zerowrap.FromCtx(ctx)
520 canonicalName := managedContainerName(domainName)
521 candidateNames := map[string]bool{
522 canonicalName: true,
523 canonicalName + "-new": true,
524 canonicalName + "-next": true,
525 }
526
527 running, err := s.runtime.ListContainers(ctx, false)
528 if err != nil {
529 log.Warn().Err(err).Msg("failed to list running containers for existing container resolution")
530 return nil, false
531 }
532
533 // Prefer canonical name; fall back to any running managed temp container.
534 var best *domain.Container
535 for _, c := range running {
536 if !candidateNames[c.Name] || c.Labels[domain.LabelManaged] != "true" {
537 continue
538 }
539 if best == nil || c.Name == canonicalName {
540 best = c
541 }
542 }
543
544 if best != nil {
545 log.Info().
546 Str("container_id", best.ID).
547 Str("container_name", best.Name).
548 Msg("resolved existing container from runtime (in-memory state was stale)")
549
550 // Update in-memory state so subsequent lookups are fast
551 s.mu.Lock()
552 if _, alreadyTracked := s.containers[domainName]; !alreadyTracked {
553 s.managedCount++
554 }
555 s.containers[domainName] = best
556 s.mu.Unlock()
557
558 return best, true
559 }
560
561 return nil, false
562}
563
564func (s *Service) prepareDeployResources(ctx context.Context, route domain.Route, existing *domain.Container) (*deployResources, error) {
565 log := zerowrap.FromCtx(ctx)
566
567 existingID := ""
568 if existing != nil {
569 existingID = existing.ID
570 }
571 if err := s.cleanupOrphanedContainers(ctx, route.Domain, existingID); err != nil {
572 log.WrapErr(err, "failed to cleanup orphaned containers")
573 }
574
575 imageRef, err := s.buildValidatedImageRef(ctx, route.Image)
576 if err != nil {
577 return nil, err
578 }
579 actualImageRef, err := s.ensureImage(ctx, imageRef)
580 if err != nil {
581 return nil, err
582 }
583
584 networkName := s.getNetworkForApp(route.Domain)
585 if err := s.createNetworkIfNeeded(ctx, networkName); err != nil {
586 return nil, log.WrapErr(err, "failed to create network")
587 }
588 if err := s.deployAttachments(ctx, route.Domain, networkName); err != nil {
589 return nil, log.WrapErr(err, "failed to deploy attachments")
590 }
591
592 exposedPorts, err := s.runtime.GetImageExposedPorts(ctx, actualImageRef)
593 if err != nil {
594 log.WrapErr(err, "failed to get exposed ports, using defaults")
595 exposedPorts = []int{80, 8080, 3000}
596 }
597
598 imageLabels, err := s.runtime.GetImageLabels(ctx, actualImageRef)
599 if err != nil {
600 log.Warn().Err(err).Msg("failed to get image labels, skipping label propagation")
601 imageLabels = nil
602 }
603
604 envVars, err := s.loadEnvironment(ctx, route.Env, route.Domain, actualImageRef)
605 if err != nil {
606 return nil, err
607 }
608 envHash := hashEnvironment(envVars)
609
610 volumes, err := s.setupVolumes(ctx, route.Domain, actualImageRef)
611 if err != nil {
612 return nil, err
613 }
614
615 return &deployResources{
616 networkName: networkName,
617 actualImageRef: actualImageRef,
618 exposedPorts: exposedPorts,
619 imageLabels: imageLabels,
620 envVars: envVars,
621 envHash: envHash,
622 volumes: volumes,
623 }, nil
624}
625
626func (s *Service) createStartedContainer(ctx context.Context, route domain.Route, existing *domain.Container, resources *deployResources) (*domain.Container, error) {
627 ctx, span := tracer.Start(ctx, "container.create_and_start")
628 defer span.End()
629
630 log := zerowrap.FromCtx(ctx)
631
632 containerConfig := s.buildContainerConfig(containerConfigInput{
633 Domain: route.Domain,
634 Image: route.Image,
635 ImageRef: resources.actualImageRef,
636 ExposedPorts: resources.exposedPorts,
637 EnvVars: resources.envVars,
638 EnvHash: resources.envHash,
639 Volumes: resources.volumes,
640 NetworkName: resources.networkName,
641 ImageLabels: resources.imageLabels,
642 Existing: existing,
643 })
644
645 newContainer, err := s.runtime.CreateContainer(ctx, containerConfig)
646 if err != nil {
647 return nil, log.WrapErr(err, "failed to create container")
648 }
649 if err := s.runtime.StartContainer(ctx, newContainer.ID); err != nil {
650 s.runtime.RemoveContainer(ctx, newContainer.ID, true)
651 return nil, log.WrapErr(err, "failed to start container")
652 }
653 if !domain.IsSkipReadiness(ctx) {
654 if err := s.waitForReady(ctx, newContainer.ID, containerConfig); err != nil {
655 deployErr := s.newDeployFailure(newContainer, "container failed readiness check", err, s.captureRecentContainerLogs(ctx, newContainer.ID))
656 s.cleanupFailedContainer(ctx, newContainer.ID)
657 return nil, deployErr
658 }
659 }
660
661 inspected, err := s.runtime.InspectContainer(ctx, newContainer.ID)
662 if err != nil {
663 s.cleanupFailedContainer(ctx, newContainer.ID)
664 return nil, log.WrapErr(err, "failed to inspect started container")
665 }
666
667 return inspected, nil
668}
669
670func (s *Service) newDeployFailure(container *domain.Container, cause string, err error, logs []string) error {
671 deployErr := &domain.DeployFailureError{
672 Summary: "failed to deploy",
673 Cause: cause,
674 Hint: inferDeployFailureHint(err),
675 Logs: logs,
676 Err: err,
677 }
678 if container != nil {
679 deployErr.ContainerName = container.Name
680 deployErr.ContainerID = container.ID
681 }
682
683 return deployErr
684}
685
686func (s *Service) wrapPullDeployFailure(route domain.Route, existing *domain.Container, err error) error {
687 if !isPullFailure(err) {
688 return nil
689 }
690
691 return s.newDeployFailure(&domain.Container{Name: s.deploymentContainerName(route.Domain, existing)}, "failed to pull image", err, nil)
692}
693
694func (s *Service) captureRecentContainerLogs(ctx context.Context, containerID string) []string {
695 if ctx.Err() != nil {
696 return nil
697 }
698
699 logStream, err := s.runtime.GetContainerLogs(ctx, containerID, false)
700 if err != nil {
701 return nil
702 }
703 defer func() {
704 _ = logStream.Close()
705 }()
706
707 logs, err := parseDockerLogLines(logStream, deployFailureLogLimit)
708 if err != nil {
709 return nil
710 }
711
712 return logs
713}
714
715func parseDockerLogLines(r io.Reader, maxLines int) ([]string, error) {
716 reader := bufio.NewReader(r)
717 collector := newLogLineCollector(maxLines)
718
719 header, err := reader.Peek(8)
720 if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, bufio.ErrBufferFull) {
721 return nil, err
722 }
723
724 if len(header) >= 8 && looksLikeDockerLogStream(header) {
725 return parseDockerFramedLogLines(reader, collector)
726 }
727
728 return parsePlainLogLines(reader, collector)
729}
730
731func parseDockerFramedLogLines(reader *bufio.Reader, collector *logLineCollector) ([]string, error) {
732 for {
733 frameHeader := make([]byte, 8)
734 if _, err := io.ReadFull(reader, frameHeader); err != nil {
735 if isLogStreamEOF(err) {
736 collector.flush()
737 return collector.lines, nil
738 }
739 return nil, err
740 }
741
742 size := int(binary.BigEndian.Uint32(frameHeader[4:8]))
743 if size <= 0 {
744 continue
745 }
746
747 payload, err := readLogPayload(reader, size)
748 if err != nil {
749 collector.addChunk(string(payload))
750 collector.flush()
751 if isLogStreamEOF(err) {
752 return collector.lines, nil
753 }
754 return nil, err
755 }
756
757 collector.addChunk(string(payload))
758 }
759}
760
761func parsePlainLogLines(reader *bufio.Reader, collector *logLineCollector) ([]string, error) {
762
763 scanner := bufio.NewScanner(reader)
764 for scanner.Scan() {
765 collector.addLine(scanner.Text())
766 }
767 if err := scanner.Err(); err != nil {
768 return nil, err
769 }
770
771 collector.flush()
772 return collector.lines, nil
773}
774
775func readLogPayload(reader *bufio.Reader, size int) ([]byte, error) {
776 payload := make([]byte, size)
777 _, err := io.ReadFull(reader, payload)
778 return payload, err
779}
780
781func isLogStreamEOF(err error) bool {
782 return errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF)
783}
784
785func looksLikeDockerLogStream(data []byte) bool {
786 return len(data) >= 8 && data[1] == 0 && data[2] == 0 && data[3] == 0
787}
788
789type logLineCollector struct {
790 lines []string
791 pending string
792 maxLines int
793}
794
795func newLogLineCollector(maxLines int) *logLineCollector {
796 return &logLineCollector{maxLines: maxLines}
797}
798
799func (c *logLineCollector) addChunk(chunk string) {
800 for _, part := range strings.SplitAfter(chunk, "\n") {
801 c.pending += part
802 if strings.HasSuffix(part, "\n") {
803 c.flush()
804 }
805 }
806}
807
808func (c *logLineCollector) addLine(line string) {
809 line = strings.TrimRight(line, "\r")
810 if line == "" {
811 return
812 }
813 c.lines = append(c.lines, line)
814 if c.maxLines > 0 && len(c.lines) > c.maxLines {
815 c.lines = c.lines[len(c.lines)-c.maxLines:]
816 }
817}
818
819func (c *logLineCollector) flush() {
820 if c.pending == "" {
821 return
822 }
823 c.addLine(strings.TrimRight(c.pending, "\r\n"))
824 c.pending = ""
825}
826
827func inferDeployFailureHint(err error) string {
828 if err == nil {
829 return ""
830 }
831
832 msg := strings.ToLower(err.Error())
833 switch {
834 case strings.Contains(msg, "healthcheck timeout"):
835 return "check startup logs and confirm the health endpoint becomes ready within the configured timeout"
836 case strings.Contains(msg, "unhealthy"):
837 return "inspect recent app logs and verify the healthcheck command reflects actual readiness"
838 case strings.Contains(msg, "no healthcheck detected"):
839 return "add a Docker healthcheck or switch readiness mode"
840 case isPullFailure(err), strings.Contains(msg, "access denied"), strings.Contains(msg, "unauthorized"):
841 return "verify registry auth and confirm the image exists at the requested tag"
842 default:
843 return ""
844 }
845}
846
847func isPullFailure(err error) bool {
848 if err == nil {
849 return false
850 }
851
852 if errors.Is(err, domain.ErrImagePullFailed) {
853 return true
854 }
855
856 msg := strings.ToLower(err.Error())
857 return strings.Contains(msg, "failed to pull image") ||
858 strings.Contains(msg, "pull access denied")
859}
860
861func (s *Service) activateDeployedContainer(ctx context.Context, domainName string, container *domain.Container) bool {
862 s.mu.Lock()
863 _, wasTracked := s.containers[domainName]
864 s.containers[domainName] = container
865 if !wasTracked {
866 s.managedCount++
867 }
868 s.mu.Unlock()
869
870 // Track managed container count (only increment for new domains, not replacements)
871 if !wasTracked && s.metrics != nil {
872 s.metrics.ManagedContainers.Add(ctx, 1)
873 }
874
875 s.publishContainerDeployed(ctx, domainName, container.ID)
876
877 s.mu.RLock()
878 inv := s.cacheInvalidator
879 s.mu.RUnlock()
880 if inv != nil {
881 inv.InvalidateTarget(ctx, domainName)
882 return true
883 }
884
885 return false
886}
887
888// stabilizeNewContainer monitors the new container briefly after traffic switch.
889// If it crashes during this window, rolls back to old container.
890// Returns true if stabilization succeeded, false if rollback was performed.
891// Returns an error only when both new and old containers are dead.
892func (s *Service) stabilizeNewContainer(ctx context.Context, domainName string, newContainer, oldContainer *domain.Container) (bool, error) {
893 log := zerowrap.FromCtx(ctx)
894
895 if oldContainer == nil {
896 return true, nil
897 }
898
899 s.mu.RLock()
900 delay := s.config.StabilizationDelay
901 s.mu.RUnlock()
902 if delay == 0 {
903 delay = 2 * time.Second
904 }
905
906 // Brief stabilization: verify new container is still running after delay
907 select {
908 case <-time.After(delay):
909 case <-ctx.Done():
910 return true, nil
911 }
912
913 running, err := s.runtime.IsContainerRunning(ctx, newContainer.ID)
914 if err != nil || !running {
915 log.Error().
916 Str("new_container", newContainer.ID).
917 Str("old_container", oldContainer.ID).
918 Msg("new container crashed during stabilization, rolling back to old")
919
920 // Verify old container is still running before restoring it.
921 // If both old and new are dead (e.g., OOM), returning a dead
922 // container as "existing" would leave the domain in a broken state.
923 oldRunning, oldErr := s.runtime.IsContainerRunning(ctx, oldContainer.ID)
924 if oldErr != nil || !oldRunning {
925 log.Error().
926 Str("old_container", oldContainer.ID).
927 Msg("old container is also not running, cannot rollback")
928
929 // Cleanup failed new container
930 if stopErr := s.runtime.StopContainer(ctx, newContainer.ID); stopErr != nil {
931 log.WrapErrWithFields(stopErr, "failed to stop failed new container during rollback", map[string]any{zerowrap.FieldEntityID: newContainer.ID})
932 }
933 if removeErr := s.runtime.RemoveContainer(ctx, newContainer.ID, true); removeErr != nil {
934 log.WrapErrWithFields(removeErr, "failed to remove failed new container during rollback", map[string]any{zerowrap.FieldEntityID: newContainer.ID})
935 }
936
937 return false, fmt.Errorf("stabilization failed: new container crashed and old container %s is also not running", oldContainer.ID)
938 }
939
940 // Rollback: restore old container as tracked
941 s.mu.Lock()
942 s.containers[domainName] = oldContainer
943 s.mu.Unlock()
944
945 // Re-invalidate proxy cache to point back to old
946 s.mu.RLock()
947 inv := s.cacheInvalidator
948 s.mu.RUnlock()
949 if inv != nil {
950 inv.InvalidateTarget(ctx, domainName)
951 }
952
953 // Cleanup failed new container
954 if stopErr := s.runtime.StopContainer(ctx, newContainer.ID); stopErr != nil {
955 log.WrapErrWithFields(stopErr, "failed to stop failed new container during rollback", map[string]any{zerowrap.FieldEntityID: newContainer.ID})
956 }
957 if removeErr := s.runtime.RemoveContainer(ctx, newContainer.ID, true); removeErr != nil {
958 log.WrapErrWithFields(removeErr, "failed to remove failed new container during rollback", map[string]any{zerowrap.FieldEntityID: newContainer.ID})
959 }
960
961 return false, nil
962 }
963
964 return true, nil
965}
966
967func (s *Service) finalizePreviousContainer(ctx context.Context, domainName string, existing *domain.Container, hasExisting, invalidated bool, newContainerID string) {
968 if !hasExisting {
969 return
970 }
971
972 if invalidated {
973 s.waitForDrain(ctx, existing.ID)
974 }
975
976 s.cleanupOldContainer(ctx, existing, newContainerID, domainName)
977}
978
979func (s *Service) waitForDrain(ctx context.Context, oldContainerID string) {
980 log := zerowrap.FromCtx(ctx)
981
982 s.mu.RLock()
983 cfg := s.config
984 waiter := s.drainWaiter
985 s.mu.RUnlock()
986
987 drainMode := cfg.DrainMode
988 if drainMode == "" {
989 drainMode = "delay"
990 }
991
992 shouldUseInFlight := (drainMode == "inflight" || drainMode == "auto") && waiter != nil
993 if shouldUseInFlight {
994 timeout := cfg.DrainTimeout
995 if timeout == 0 {
996 timeout = 30 * time.Second
997 }
998 drained := waiter.WaitForNoInFlight(ctx, oldContainerID, timeout)
999 if !drained {
1000 log.Warn().
1001 Str("old_container_id", oldContainerID).
1002 Dur("drain_timeout", timeout).
1003 Msg("drain wait timed out; old container may still have in-flight traffic")
1004 }
1005 return
1006 }
1007
1008 drainDelay := 2 * time.Second
1009 if cfg.DrainDelayConfigured {
1010 drainDelay = cfg.DrainDelay
1011 }
1012 if drainDelay <= 0 {
1013 return
1014 }
1015 select {
1016 case <-time.After(drainDelay):
1017 case <-ctx.Done():
1018 }
1019}
1020
1021// restartContainerWithRecovery restarts a container, reconciling stale state if needed.
1022// It returns the (possibly refreshed) container and updated attachment IDs.
1023func (s *Service) restartContainerWithRecovery(ctx context.Context, domainName string, container *domain.Container, attachmentIDs []string) (*domain.Container, []string, error) {
1024 log := zerowrap.FromCtx(ctx)
1025
1026 if err := s.runtime.RestartContainer(ctx, container.ID); err != nil {
1027 if !isContainerNotFoundError(err) {
1028 return nil, nil, log.WrapErr(err, "failed to restart container")
1029 }
1030
1031 // In-memory container ID can become stale after external runtime changes.
1032 // Re-sync and retry once with the latest tracked container.
1033 log.Warn().
1034 Err(err).
1035 Str(zerowrap.FieldEntityID, container.ID).
1036 Msg("tracked container missing during restart, attempting state reconciliation")
1037
1038 if syncErr := s.SyncContainers(ctx); syncErr != nil {
1039 log.Warn().Err(syncErr).Msg("failed to sync container state during restart recovery")
1040 }
1041
1042 s.mu.RLock()
1043 refreshed, refreshedExists := s.containers[domainName]
1044 attachmentIDs = append([]string{}, s.attachments[domainName]...)
1045 s.mu.RUnlock()
1046
1047 if !refreshedExists || refreshed == nil {
1048 return nil, nil, domain.ErrContainerNotFound
1049 }
1050 if err := s.runtime.RestartContainer(ctx, refreshed.ID); err != nil {
1051 return nil, nil, log.WrapErr(err, "failed to restart container after state reconciliation")
1052 }
1053 return refreshed, attachmentIDs, nil
1054 }
1055
1056 return container, attachmentIDs, nil
1057}
1058
1059// Restart restarts a running container for the given domain.
1060func (s *Service) Restart(ctx context.Context, domainName string, withAttachments bool) error {
1061 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1062 zerowrap.FieldLayer: "usecase",
1063 zerowrap.FieldUseCase: "Restart",
1064 "domain": domainName,
1065 })
1066 log := zerowrap.FromCtx(ctx)
1067
1068 s.mu.RLock()
1069 container, exists := s.containers[domainName]
1070 attachments := s.attachments[domainName]
1071 var attachmentIDs []string
1072 if len(attachments) > 0 {
1073 attachmentIDs = make([]string, len(attachments))
1074 copy(attachmentIDs, attachments)
1075 }
1076 s.mu.RUnlock()
1077
1078 if !exists || container == nil {
1079 return domain.ErrContainerNotFound
1080 }
1081
1082 // When attachments are requested, sync runtime state first to get accurate
1083 // attachment tracking, then check if configured attachments are deployed.
1084 // If fewer attachment containers are tracked than configured, fail early with
1085 // guidance rather than restarting the main container into a broken state
1086 // (e.g., missing database hostname).
1087 if withAttachments {
1088 if syncErr := s.SyncContainers(ctx); syncErr != nil {
1089 log.Warn().Err(syncErr).Msg("failed to sync container state before attachment check")
1090 }
1091
1092 // Re-read attachment IDs after sync
1093 s.mu.RLock()
1094 if freshAttachments := s.attachments[domainName]; len(freshAttachments) > 0 {
1095 attachmentIDs = make([]string, len(freshAttachments))
1096 copy(attachmentIDs, freshAttachments)
1097 } else {
1098 attachmentIDs = nil
1099 }
1100 s.mu.RUnlock()
1101
1102 configuredAttachments := s.resolveAttachmentsForDomain(domainName)
1103 if len(configuredAttachments) > 0 && len(attachmentIDs) < len(configuredAttachments) {
1104 missing := len(configuredAttachments) - len(attachmentIDs)
1105 return fmt.Errorf("%w: domain %q has %d configured attachment(s) but only %d deployed (%d missing)",
1106 domain.ErrAttachmentNotDeployed, domainName, len(configuredAttachments), len(attachmentIDs), missing)
1107 }
1108 }
1109
1110 // Restart the main container (with stale-state recovery)
1111 container, attachmentIDs, err := s.restartContainerWithRecovery(ctx, domainName, container, attachmentIDs)
1112 if err != nil {
1113 return err
1114 }
1115 log.Info().Str(zerowrap.FieldEntityID, container.ID).Msg("container restarted")
1116
1117 // Record restart metric
1118 if s.metrics != nil {
1119 s.metrics.ContainerRestarts.Add(ctx, 1, metric.WithAttributes(
1120 attribute.String("domain", domainName),
1121 attribute.String("source", "api"),
1122 ))
1123 }
1124
1125 // Restart attachments if requested
1126 if withAttachments && len(attachmentIDs) > 0 {
1127 for _, attachID := range attachmentIDs {
1128 if err := s.runtime.RestartContainer(ctx, attachID); err != nil {
1129 log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to restart attachment")
1130 continue // Don't fail the whole operation for one attachment
1131 }
1132 log.Info().Str("attachment_id", attachID).Msg("attachment restarted")
1133 }
1134 }
1135
1136 return nil
1137}
1138
1139// Stop stops a running container.
1140func (s *Service) Stop(ctx context.Context, containerID string) error {
1141 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1142 zerowrap.FieldLayer: "usecase",
1143 zerowrap.FieldUseCase: "Stop",
1144 zerowrap.FieldEntityID: containerID,
1145 })
1146 log := zerowrap.FromCtx(ctx)
1147
1148 // Stop log collection before stopping container
1149 if s.logWriter != nil {
1150 if err := s.logWriter.StopLogging(containerID); err != nil {
1151 log.Warn().Err(err).Msg("failed to stop container log collection")
1152 }
1153 }
1154
1155 if err := s.runtime.StopContainer(ctx, containerID); err != nil {
1156 return log.WrapErr(err, "failed to stop container")
1157 }
1158
1159 log.Info().Msg("container stopped")
1160 return nil
1161}
1162
1163// PreviewRemovedRouteCleanup reports retained runtime state for a route that is
1164// no longer configured, without mutating containers or in-memory tracking.
1165func (s *Service) PreviewRemovedRouteCleanup(ctx context.Context, domainName string) (*domain.CleanupReport, error) {
1166 canonicalDomain, ok := domain.CanonicalRouteDomain(domainName)
1167 if !ok {
1168 return &domain.CleanupReport{Domain: domainName}, domain.ErrRouteDomainInvalid
1169 }
1170 domainName = canonicalDomain
1171
1172 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1173 zerowrap.FieldLayer: "usecase",
1174 zerowrap.FieldUseCase: "PreviewRemovedRouteCleanup",
1175 "domain": domainName,
1176 })
1177 log := zerowrap.FromCtx(ctx)
1178
1179 report := &domain.CleanupReport{Domain: domainName}
1180
1181 unlock, err := s.acquireDomainDeployLock(ctx, domainName)
1182 if err != nil {
1183 return report, err
1184 }
1185 defer unlock()
1186
1187 allContainers, err := s.runtime.ListContainers(ctx, true)
1188 if err != nil {
1189 return report, log.WrapErr(err, "failed to list containers for removed route cleanup preview")
1190 }
1191
1192 routeContainers := routeContainersForDomain(allContainers, domainName)
1193 report.PreservedAttachments = preservedAttachmentsForDomain(allContainers, domainName)
1194 if len(report.PreservedAttachments) > 0 {
1195 report.Hints = append(report.Hints, "attachments preserved; remove or purge attachments explicitly if they are no longer needed")
1196 }
1197 for _, c := range routeContainers {
1198 report.OrphanedEntities = append(report.OrphanedEntities, domain.CleanupOrphanedEntity{
1199 Kind: "route_container",
1200 ID: c.ID,
1201 Name: c.Name,
1202 Status: c.Status,
1203 Reason: "route runtime state still exists after route removal",
1204 })
1205 }
1206
1207 return report, nil
1208}
1209
1210// ReconcileRemovedRoute removes active runtime containers for a route that was
1211// removed from configuration while preserving stateful resources for explicit
1212// follow-up cleanup. It intentionally removes only main route containers, never
1213// attachment containers or volumes.
1214func (s *Service) ReconcileRemovedRoute(ctx context.Context, domainName string) (*domain.CleanupReport, error) {
1215 canonicalDomain, ok := domain.CanonicalRouteDomain(domainName)
1216 if !ok {
1217 return &domain.CleanupReport{Domain: domainName}, domain.ErrRouteDomainInvalid
1218 }
1219 domainName = canonicalDomain
1220
1221 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1222 zerowrap.FieldLayer: "usecase",
1223 zerowrap.FieldUseCase: "ReconcileRemovedRoute",
1224 "domain": domainName,
1225 })
1226 log := zerowrap.FromCtx(ctx)
1227
1228 report := &domain.CleanupReport{Domain: domainName}
1229
1230 unlock, err := s.acquireDomainDeployLock(ctx, domainName)
1231 if err != nil {
1232 return report, err
1233 }
1234 defer unlock()
1235
1236 allContainers, err := s.runtime.ListContainers(ctx, true)
1237 if err != nil {
1238 return report, log.WrapErr(err, "failed to list containers for removed route cleanup")
1239 }
1240
1241 routeContainers := routeContainersForDomain(allContainers, domainName)
1242
1243 // Clear in-memory state before stopping containers so the monitor cannot
1244 // interpret the deliberate removal as a crash that should be restarted.
1245 s.clearRouteContainerTracking(ctx, domainName)
1246 if inv := s.proxyCacheInvalidator(); inv != nil {
1247 inv.InvalidateTarget(ctx, domainName)
1248 }
1249
1250 report.PreservedAttachments = preservedAttachmentsForDomain(allContainers, domainName)
1251 if len(report.PreservedAttachments) > 0 {
1252 report.Hints = append(report.Hints, "attachments preserved; remove or purge attachments explicitly if they are no longer needed")
1253 }
1254
1255 if len(routeContainers) == 0 {
1256 report.Warnings = append(report.Warnings, "no active route container found for removed route")
1257 return report, nil
1258 }
1259
1260 s.removeRouteContainersForCleanup(ctx, log, routeContainers, report)
1261
1262 if len(report.PartialFailures) > 0 {
1263 report.Warnings = append(report.Warnings, "route cleanup completed with partial failures; manual follow-up may be required")
1264 }
1265
1266 return report, nil
1267}
1268
1269func routeContainersForDomain(containers []*domain.Container, domainName string) []*domain.Container {
1270 var routeContainers []*domain.Container
1271 for _, c := range containers {
1272 if isManagedRouteContainerForDomain(c, domainName) {
1273 routeContainers = append(routeContainers, c)
1274 }
1275 }
1276 return routeContainers
1277}
1278
1279func preservedAttachmentsForDomain(containers []*domain.Container, domainName string) []domain.CleanupAttachment {
1280 var attachments []domain.CleanupAttachment
1281 for _, c := range containers {
1282 if isPreservedAttachmentForDomain(c, domainName) {
1283 attachments = append(attachments, domain.CleanupAttachment{
1284 Name: c.Name,
1285 Image: c.Image,
1286 ContainerID: c.ID,
1287 Status: c.Status,
1288 Owner: c.Labels[domain.LabelAttachedTo],
1289 Reason: "attachments are preserved by default; remove or purge them explicitly",
1290 })
1291 }
1292 }
1293 return attachments
1294}
1295
1296func (s *Service) removeRouteContainersForCleanup(ctx context.Context, log zerowrap.Logger, containers []*domain.Container, report *domain.CleanupReport) {
1297 for _, c := range containers {
1298 s.stopRouteLogCollectionForCleanup(log, c, report)
1299 s.stopRouteContainerForCleanup(ctx, log, c, report)
1300 if err := s.runtime.RemoveContainer(ctx, c.ID, true); err != nil {
1301 report.PartialFailures = append(report.PartialFailures, domain.CleanupFailure{Action: "remove", Kind: "container", ID: c.ID, Name: c.Name, Error: err.Error()})
1302 log.Warn().Err(err).Str(zerowrap.FieldEntityID, c.ID).Msg("failed to remove removed route container")
1303 continue
1304 }
1305 report.RemovedContainers = append(report.RemovedContainers, domain.CleanupContainer{ID: c.ID, Name: c.Name, Image: c.Image, Status: c.Status})
1306 }
1307}
1308
1309func (s *Service) stopRouteLogCollectionForCleanup(log zerowrap.Logger, c *domain.Container, report *domain.CleanupReport) {
1310 if s.logWriter == nil {
1311 return
1312 }
1313 if err := s.logWriter.StopLogging(c.ID); err != nil {
1314 report.PartialFailures = append(report.PartialFailures, domain.CleanupFailure{Action: "stop_logging", Kind: "container", ID: c.ID, Name: c.Name, Error: err.Error()})
1315 log.Warn().Err(err).Str(zerowrap.FieldEntityID, c.ID).Msg("failed to stop removed route log collection")
1316 }
1317}
1318
1319func (s *Service) stopRouteContainerForCleanup(ctx context.Context, log zerowrap.Logger, c *domain.Container, report *domain.CleanupReport) {
1320 if c.Status != string(domain.ContainerStatusRunning) {
1321 return
1322 }
1323 if err := s.runtime.StopContainer(ctx, c.ID); err != nil {
1324 report.PartialFailures = append(report.PartialFailures, domain.CleanupFailure{Action: "stop", Kind: "container", ID: c.ID, Name: c.Name, Error: err.Error()})
1325 log.Warn().Err(err).Str(zerowrap.FieldEntityID, c.ID).Msg("failed to stop removed route container")
1326 }
1327}
1328
1329func isManagedRouteContainerForDomain(c *domain.Container, domainName string) bool {
1330 if c == nil || c.Labels == nil || c.Labels[domain.LabelManaged] != "true" {
1331 return false
1332 }
1333 if c.Labels[domain.LabelAttachment] == "true" {
1334 return false
1335 }
1336 return c.Labels[domain.LabelRoute] == domainName || c.Labels[domain.LabelDomain] == domainName || c.Name == managedContainerName(domainName)
1337}
1338
1339func isPreservedAttachmentForDomain(c *domain.Container, domainName string) bool {
1340 if c == nil || c.Labels == nil || c.Labels[domain.LabelManaged] != "true" {
1341 return false
1342 }
1343 return c.Labels[domain.LabelAttachment] == "true" && c.Labels[domain.LabelAttachedTo] == domainName
1344}
1345
1346func (s *Service) proxyCacheInvalidator() out.ProxyCacheInvalidator {
1347 s.mu.RLock()
1348 defer s.mu.RUnlock()
1349 return s.cacheInvalidator
1350}
1351
1352func (s *Service) clearRouteContainerTracking(ctx context.Context, domainName string) {
1353 s.mu.Lock()
1354 removed := false
1355 if _, exists := s.containers[domainName]; exists {
1356 delete(s.containers, domainName)
1357 if s.managedCount > 0 {
1358 s.managedCount--
1359 }
1360 removed = true
1361 }
1362 s.mu.Unlock()
1363
1364 if removed && s.metrics != nil {
1365 s.metrics.ManagedContainers.Add(ctx, -1)
1366 }
1367}
1368
1369// Remove removes a container.
1370func (s *Service) Remove(ctx context.Context, containerID string, force bool) error {
1371 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1372 zerowrap.FieldLayer: "usecase",
1373 zerowrap.FieldUseCase: "Remove",
1374 zerowrap.FieldEntityID: containerID,
1375 })
1376 log := zerowrap.FromCtx(ctx)
1377
1378 // Stop log collection before removing container
1379 if s.logWriter != nil {
1380 if err := s.logWriter.StopLogging(containerID); err != nil {
1381 log.Warn().Err(err).Msg("failed to stop container log collection")
1382 }
1383 }
1384
1385 // Find domain and attachment IDs for cleanup
1386 containerDomain, attachmentIDs, cfg := s.findContainerContext(containerID)
1387
1388 if err := s.runtime.RemoveContainer(ctx, containerID, force); err != nil {
1389 return log.WrapErr(err, "failed to remove container")
1390 }
1391
1392 // Cleanup volumes
1393 if containerDomain != "" && !cfg.VolumePreserve {
1394 if err := s.cleanupVolumesForDomain(ctx, containerDomain); err != nil {
1395 log.WrapErrWithFields(err, "failed to cleanup volumes", map[string]any{"domain": containerDomain})
1396 }
1397 }
1398
1399 // Clean up attachment containers
1400 s.removeAttachments(ctx, attachmentIDs)
1401
1402 // Remove from tracking
1403 s.mu.Lock()
1404 var removedDomain string
1405 for d, c := range s.containers {
1406 if c.ID == containerID {
1407 delete(s.containers, d)
1408 removedDomain = d
1409 s.managedCount--
1410 log.Info().Str("domain", d).Msg("container removed")
1411 break
1412 }
1413 }
1414 if removedDomain != "" {
1415 delete(s.attachments, removedDomain)
1416 }
1417 s.mu.Unlock()
1418
1419 // Decrement managed container count
1420 if removedDomain != "" && s.metrics != nil {
1421 s.metrics.ManagedContainers.Add(ctx, -1)
1422 }
1423
1424 // Note: We intentionally do NOT clean up deployMu entries to avoid a race condition:
1425 // If Remove deletes the mutex entry while a concurrent Deploy is acquiring the lock,
1426 // the Deploy would create a fresh mutex, breaking serialization and allowing concurrent
1427 // deploys to the same domain. The memory footprint is acceptable (one small struct per
1428 // domain ever deployed), and this is the safer choice for correctness.
1429
1430 // Cleanup network
1431 if removedDomain != "" && cfg.NetworkIsolation {
1432 networkName := s.getNetworkForApp(removedDomain)
1433 if err := s.cleanupNetworkIfEmpty(ctx, networkName); err != nil {
1434 log.WrapErrWithFields(err, "failed to cleanup network", map[string]any{"network": networkName})
1435 }
1436 }
1437
1438 return nil
1439}
1440
1441// findContainerContext looks up the domain, attachment IDs, and config snapshot for a container.
1442func (s *Service) findContainerContext(containerID string) (string, []string, Config) {
1443 s.mu.RLock()
1444 defer s.mu.RUnlock()
1445 cfg := s.config
1446 var containerDomain string
1447 for d, c := range s.containers {
1448 if c.ID == containerID {
1449 containerDomain = d
1450 break
1451 }
1452 }
1453 var attachmentIDs []string
1454 if containerDomain != "" {
1455 attachmentIDs = append(attachmentIDs, s.attachments[containerDomain]...)
1456 }
1457 return containerDomain, attachmentIDs, cfg
1458}
1459
1460// removeAttachments stops and removes attachment containers best-effort.
1461func (s *Service) removeAttachments(ctx context.Context, attachmentIDs []string) {
1462 log := zerowrap.FromCtx(ctx)
1463 for _, attachID := range attachmentIDs {
1464 if s.logWriter != nil {
1465 if err := s.logWriter.StopLogging(attachID); err != nil {
1466 log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to stop attachment log collection")
1467 }
1468 }
1469 if err := s.runtime.StopContainer(ctx, attachID); err != nil {
1470 log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to stop attachment container")
1471 }
1472 if err := s.runtime.RemoveContainer(ctx, attachID, true); err != nil {
1473 log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to remove attachment container")
1474 }
1475 }
1476}
1477
1478// Get retrieves a container by domain.
1479func (s *Service) Get(_ context.Context, domainName string) (*domain.Container, bool) {
1480 s.mu.RLock()
1481 defer s.mu.RUnlock()
1482 container, exists := s.containers[domainName]
1483 return container, exists
1484}
1485
1486// List returns all managed containers.
1487func (s *Service) List(_ context.Context) map[string]*domain.Container {
1488 s.mu.RLock()
1489 defer s.mu.RUnlock()
1490
1491 result := make(map[string]*domain.Container, len(s.containers))
1492 maps.Copy(result, s.containers)
1493 return result
1494}
1495
1496// ListRoutesWithDetails returns routes with network and attachment info.
1497// Note: Container map is copied under lock, then external calls are made without lock.
1498// If containers are removed between copy and runtime calls, errors are handled gracefully
1499// (network becomes empty string). This trade-off avoids holding locks during I/O.
1500func (s *Service) ListRoutesWithDetails(ctx context.Context) []domain.RouteInfo {
1501 s.mu.RLock()
1502 containers := make(map[string]*domain.Container, len(s.containers))
1503 maps.Copy(containers, s.containers)
1504 s.mu.RUnlock()
1505
1506 // Fetch all attachments once to avoid N+1 queries
1507 attachmentsByDomain := s.getAllAttachments(ctx)
1508
1509 results := make([]domain.RouteInfo, 0, len(containers))
1510 for domainName, container := range containers {
1511 network := ""
1512 image := ""
1513 containerID := ""
1514 status := ""
1515 if container != nil {
1516 containerID = container.ID
1517 status = container.Status
1518 image = container.Image
1519 if container.Labels != nil {
1520 if labelImage, ok := container.Labels[domain.LabelImage]; ok && labelImage != "" {
1521 image = labelImage
1522 }
1523 }
1524 // Strip registry domain prefix from image for cleaner display
1525 image = s.stripRegistryPrefix(image)
1526 if networkName, err := s.runtime.GetContainerNetwork(ctx, container.ID); err == nil {
1527 network = networkName
1528 }
1529 }
1530
1531 results = append(results, domain.RouteInfo{
1532 Domain: domainName,
1533 Image: image,
1534 ContainerID: containerID,
1535 ContainerStatus: status,
1536 Network: network,
1537 Attachments: attachmentsByDomain[domainName],
1538 })
1539 }
1540
1541 return results
1542}
1543
1544// stripRegistryPrefix removes the configured current or legacy Gordon registry
1545// domain prefix from an image reference. For example,
1546// "reg.example.com/myapp:latest" becomes "myapp:latest" when
1547// "reg.example.com" is a current or legacy Gordon registry domain.
1548func (s *Service) stripRegistryPrefix(image string) string {
1549 s.mu.RLock()
1550 cfg := s.config
1551 s.mu.RUnlock()
1552
1553 return domain.StripKnownGordonRegistry(image, cfg.RegistryDomain, cfg.LegacyRegistryDomains)
1554}
1555
1556// ListAttachments returns attachments for a domain.
1557func (s *Service) ListAttachments(ctx context.Context, domainName string) []domain.Attachment {
1558 return s.getAttachmentsForDomain(ctx, domainName)
1559}
1560
1561// ListOrphanedAttachments returns managed attachment containers no longer configured.
1562func (s *Service) ListOrphanedAttachments(ctx context.Context) ([]domain.CleanupAttachment, error) {
1563 containers, err := s.runtime.ListContainers(ctx, true)
1564 if err != nil {
1565 return nil, fmt.Errorf("failed to list containers with runtime for orphaned attachments: %w", err)
1566 }
1567 return s.orphanedAttachmentsFromContainers(containers), nil
1568}
1569
1570// CleanupOrphanedAttachments optionally stops/removes orphaned attachment containers.
1571// When owner is non-empty, cleanup is scoped to that attachment owner.
1572func (s *Service) CleanupOrphanedAttachments(ctx context.Context, owner string, stop bool) (*domain.CleanupReport, error) {
1573 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1574 zerowrap.FieldLayer: "usecase",
1575 zerowrap.FieldUseCase: "CleanupOrphanedAttachments",
1576 "owner": owner,
1577 })
1578 log := zerowrap.FromCtx(ctx)
1579 report := &domain.CleanupReport{Domain: owner}
1580
1581 containers, err := s.runtime.ListContainers(ctx, true)
1582 if err != nil {
1583 return report, fmt.Errorf("failed to list containers with runtime for orphaned attachment cleanup: %w", err)
1584 }
1585 orphans := filterAttachmentContainersByOwner(s.orphanedAttachmentContainers(containers), owner)
1586 if !stop {
1587 report.PreservedAttachments = cleanupAttachmentsFromContainers(orphans)
1588 if len(report.PreservedAttachments) > 0 {
1589 report.Hints = append(report.Hints, "orphaned attachments preserved; rerun with --stop to stop/remove containers while preserving volumes")
1590 }
1591 return report, nil
1592 }
1593
1594 for _, c := range orphans {
1595 s.stopRouteLogCollectionForCleanup(log, c, report)
1596 s.stopRouteContainerForCleanup(ctx, log, c, report)
1597 if err := s.runtime.RemoveContainer(ctx, c.ID, true); err != nil {
1598 report.PartialFailures = append(report.PartialFailures, domain.CleanupFailure{Action: "remove", Kind: "attachment", ID: c.ID, Name: c.Name, Error: err.Error()})
1599 log.Warn().Err(err).Str(zerowrap.FieldEntityID, c.ID).Msg("failed to remove orphaned attachment")
1600 continue
1601 }
1602 report.RemovedContainers = append(report.RemovedContainers, domain.CleanupContainer{ID: c.ID, Name: c.Name, Image: attachmentImage(c), Status: c.Status})
1603 s.untrackAttachmentContainer(c.Labels[domain.LabelAttachedTo], c.ID)
1604 }
1605 if len(report.RemovedContainers) > 0 {
1606 report.Hints = append(report.Hints, "attachment volumes and data were preserved")
1607 }
1608 if len(report.PartialFailures) > 0 {
1609 report.Warnings = append(report.Warnings, "attachment cleanup completed with partial failures; manual follow-up may be required")
1610 }
1611 return report, nil
1612}
1613
1614func (s *Service) orphanedAttachmentsFromContainers(containers []*domain.Container) []domain.CleanupAttachment {
1615 return cleanupAttachmentsFromContainers(s.orphanedAttachmentContainers(containers))
1616}
1617
1618func (s *Service) orphanedAttachmentContainers(containers []*domain.Container) []*domain.Container {
1619 var out []*domain.Container
1620 for _, c := range containers {
1621 if !isManagedAttachment(c) {
1622 continue
1623 }
1624 owner := c.Labels[domain.LabelAttachedTo]
1625 image := attachmentImage(c)
1626 if owner == "" || image == "" || !s.isAttachmentConfiguredForOwner(owner, image) {
1627 out = append(out, c)
1628 }
1629 }
1630 return out
1631}
1632
1633func filterAttachmentContainersByOwner(containers []*domain.Container, owner string) []*domain.Container {
1634 if owner == "" {
1635 return containers
1636 }
1637 var out []*domain.Container
1638 for _, c := range containers {
1639 if c != nil && c.Labels != nil && strings.EqualFold(c.Labels[domain.LabelAttachedTo], owner) {
1640 out = append(out, c)
1641 }
1642 }
1643 return out
1644}
1645
1646func cleanupAttachmentsFromContainers(containers []*domain.Container) []domain.CleanupAttachment {
1647 attachments := make([]domain.CleanupAttachment, 0, len(containers))
1648 for _, c := range containers {
1649 attachments = append(attachments, domain.CleanupAttachment{
1650 Name: c.Name,
1651 Image: attachmentImage(c),
1652 ContainerID: c.ID,
1653 Status: c.Status,
1654 Owner: c.Labels[domain.LabelAttachedTo],
1655 Reason: "attachment container is no longer configured",
1656 })
1657 }
1658 return attachments
1659}
1660
1661func isManagedAttachment(c *domain.Container) bool {
1662 return c != nil && c.Labels != nil && c.Labels[domain.LabelManaged] == "true" && c.Labels[domain.LabelAttachment] == "true"
1663}
1664
1665func attachmentImage(c *domain.Container) string {
1666 if c == nil {
1667 return ""
1668 }
1669 if c.Labels != nil && c.Labels[domain.LabelImage] != "" {
1670 return c.Labels[domain.LabelImage]
1671 }
1672 return c.Image
1673}
1674
1675func (s *Service) isAttachmentConfiguredForOwner(owner, image string) bool {
1676 return slices.Contains(s.resolveAttachmentsForDomain(owner), image)
1677}
1678
1679func (s *Service) untrackAttachmentContainer(owner, containerID string) {
1680 s.mu.Lock()
1681 defer s.mu.Unlock()
1682 ids := s.attachments[owner]
1683 for i, id := range ids {
1684 if id == containerID {
1685 s.attachments[owner] = slices.Delete(ids, i, i+1)
1686 break
1687 }
1688 }
1689 if len(s.attachments[owner]) == 0 {
1690 delete(s.attachments, owner)
1691 }
1692}
1693
1694// ListNetworks returns Gordon-managed networks.
1695func (s *Service) ListNetworks(ctx context.Context) ([]*domain.NetworkInfo, error) {
1696 s.mu.RLock()
1697 cfg := s.config
1698 s.mu.RUnlock()
1699
1700 networks, err := s.runtime.ListNetworks(ctx)
1701 if err != nil {
1702 return nil, err
1703 }
1704
1705 var filtered []*domain.NetworkInfo
1706 for _, network := range networks {
1707 if strings.HasPrefix(network.Name, cfg.NetworkPrefix+"-") && network.Labels[domain.LabelManaged] == "true" {
1708 filtered = append(filtered, network)
1709 }
1710 }
1711
1712 return filtered, nil
1713}
1714
1715// HealthCheck performs health checks on all containers.
1716func (s *Service) HealthCheck(ctx context.Context) map[string]bool {
1717 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1718 zerowrap.FieldLayer: "usecase",
1719 zerowrap.FieldUseCase: "HealthCheck",
1720 })
1721 log := zerowrap.FromCtx(ctx)
1722
1723 // Copy container IDs under lock, then release before making Docker API calls
1724 s.mu.RLock()
1725 snapshot := make(map[string]string, len(s.containers))
1726 for d, c := range s.containers {
1727 snapshot[d] = c.ID
1728 }
1729 s.mu.RUnlock()
1730
1731 health := make(map[string]bool, len(snapshot))
1732 for d, id := range snapshot {
1733 running, err := s.runtime.IsContainerRunning(ctx, id)
1734 if err != nil {
1735 log.WrapErrWithFields(err, "health check failed", map[string]any{"domain": d, zerowrap.FieldEntityID: id})
1736 health[d] = false
1737 } else {
1738 health[d] = running
1739 }
1740 }
1741 return health
1742}
1743
1744// SyncContainers synchronizes containers with runtime state.
1745func isTrackedManagedContainerStatus(status string) bool {
1746 return status == "" || status == string(domain.ContainerStatusRunning) || status == "restarting"
1747}
1748
1749func (s *Service) SyncContainers(ctx context.Context) error {
1750 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1751 zerowrap.FieldLayer: "usecase",
1752 zerowrap.FieldUseCase: "SyncContainers",
1753 })
1754 log := zerowrap.FromCtx(ctx)
1755
1756 // List containers without holding lock to avoid blocking during the runtime call.
1757 // Use all=true so we can rebuild the running-container maps from a single
1758 // snapshot without a second list call.
1759 allContainers, err := s.runtime.ListContainers(ctx, true)
1760 if err != nil {
1761 return log.WrapErr(err, "failed to list containers")
1762 }
1763
1764 managed := make(map[string]*domain.Container)
1765 attachments := make(map[string][]string)
1766 for _, c := range allContainers {
1767 if !isTrackedManagedContainerStatus(c.Status) {
1768 continue
1769 }
1770 if c.Labels == nil || c.Labels[domain.LabelManaged] != "true" {
1771 continue
1772 }
1773 if c.Labels[domain.LabelAttachment] == "true" {
1774 owner := c.Labels[domain.LabelAttachedTo]
1775 if owner != "" {
1776 attachments[owner] = append(attachments[owner], c.ID)
1777 }
1778 continue
1779 }
1780 if d, ok := c.Labels[domain.LabelDomain]; ok {
1781 managed[d] = c
1782 }
1783 }
1784
1785 s.mu.Lock()
1786 s.containers = managed
1787 s.attachments = attachments
1788 newCount := int64(len(managed))
1789 delta := newCount - s.managedCount
1790 s.managedCount = newCount
1791 s.mu.Unlock()
1792
1793 // Report initial/delta managed container count to OTel UpDownCounter.
1794 if delta != 0 && s.metrics != nil {
1795 s.metrics.ManagedContainers.Add(ctx, delta)
1796 }
1797
1798 log.Info().Int(zerowrap.FieldCount, len(managed)).Msg("container state synchronized")
1799 return nil
1800}
1801
1802// AutoStart starts containers for the provided routes that aren't running.
1803// It skips readiness checks to avoid blocking boot; the background monitor
1804// handles crash recovery.
1805func (s *Service) AutoStart(ctx context.Context, routes []domain.Route) error {
1806 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1807 zerowrap.FieldLayer: "usecase",
1808 zerowrap.FieldUseCase: "AutoStart",
1809 })
1810 log := zerowrap.FromCtx(ctx)
1811
1812 log.Info().Int("route_count", len(routes)).Msg("auto-starting containers for configured routes")
1813
1814 // Filter out routes that already have a running container.
1815 var pending []domain.Route
1816 skipped := 0
1817 for _, route := range routes {
1818 if _, exists := s.Get(ctx, route.Domain); exists {
1819 log.Debug().Str("domain", route.Domain).Msg("container already running, skipping")
1820 skipped++
1821 } else {
1822 pending = append(pending, route)
1823 }
1824 }
1825
1826 if len(pending) == 0 {
1827 log.Info().Int("skipped", skipped).Msg("auto-start completed, all containers already running")
1828 return nil
1829 }
1830
1831 // Deploy all pending routes concurrently with readiness checks skipped.
1832 deployCtx := domain.WithSkipReadiness(ctx)
1833
1834 type result struct {
1835 route domain.Route
1836 err error
1837 }
1838 results := make(chan result, len(pending))
1839 sem := make(chan struct{}, 4) // limit concurrency
1840
1841 for _, route := range pending {
1842 sem <- struct{}{}
1843 go func(r domain.Route) {
1844 defer func() { <-sem }()
1845
1846 log.Info().Str("domain", r.Domain).Str("image", r.Image).Msg("auto-starting container for route")
1847
1848 if _, err := s.Deploy(deployCtx, r); err != nil {
1849 results <- result{route: r, err: err}
1850 return
1851 }
1852 results <- result{route: r}
1853 }(route)
1854 }
1855
1856 var started, errCount int
1857 for i := 0; i < len(pending); i++ {
1858 res := <-results
1859 if res.err != nil {
1860 log.Warn().Err(res.err).Str("domain", res.route.Domain).Msg("failed to auto-start container")
1861 errCount++
1862 } else {
1863 started++
1864 }
1865 }
1866
1867 log.Info().
1868 Int("started", started).
1869 Int("skipped", skipped).
1870 Int("errors", errCount).
1871 Msg("auto-start completed")
1872
1873 if errCount > 0 {
1874 return fmt.Errorf("auto-start completed with %d errors", errCount)
1875 }
1876 return nil
1877}
1878
1879// Shutdown gracefully shuts down all managed containers.
1880func (s *Service) Shutdown(ctx context.Context) error {
1881 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1882 zerowrap.FieldLayer: "usecase",
1883 zerowrap.FieldUseCase: "Shutdown",
1884 })
1885 log := zerowrap.FromCtx(ctx)
1886 log.Info().Msg("shutting down container manager...")
1887
1888 // Wait for any in-flight background cleanup goroutines to finish
1889 s.cleanupWg.Wait()
1890
1891 // Containers are left running across Gordon restarts.
1892 // SyncContainers + AutoStart will pick them back up on next boot.
1893
1894 // Close log writer to stop all log collection
1895 if s.logWriter != nil {
1896 if err := s.logWriter.Close(); err != nil {
1897 log.Warn().Err(err).Msg("failed to close container log writer")
1898 }
1899 }
1900
1901 log.Info().Msg("container manager shutdown complete")
1902 return nil
1903}
1904
1905// WaitForCleanup blocks until all background cleanup goroutines complete.
1906// Intended for use in tests to avoid mock assertion races.
1907func (s *Service) WaitForCleanup() {
1908 s.cleanupWg.Wait()
1909}
1910
1911// StartMonitor begins background monitoring of tracked containers.
1912// The monitor restarts crashed containers and detects crash loops.
1913// Safe to call multiple times; subsequent calls are no-ops.
1914func (s *Service) StartMonitor(ctx context.Context) {
1915 s.mu.Lock()
1916 if s.monitor != nil {
1917 s.mu.Unlock()
1918 return
1919 }
1920 s.monitor = newMonitor(s)
1921 m := s.monitor
1922 m.Start(ctx)
1923 s.mu.Unlock()
1924}
1925
1926// StopMonitor stops the background container monitor.
1927// Safe to call multiple times; subsequent calls are no-ops.
1928func (s *Service) StopMonitor() {
1929 s.mu.Lock()
1930 m := s.monitor
1931 s.monitor = nil
1932 s.mu.Unlock()
1933 if m != nil {
1934 m.Stop()
1935 }
1936}
1937
1938// UpdateConfig updates the service configuration.
1939func (s *Service) UpdateConfig(config Config) {
1940 s.mu.Lock()
1941 s.config = config
1942 s.mu.Unlock()
1943}
1944
1945// UpdateAttachments updates only the attachment configuration in the service.
1946// This is called after a config reload to propagate attachment changes without restart.
1947// The incoming map is deep-copied so external callers cannot mutate service state.
1948func (s *Service) UpdateAttachments(attachments map[string][]string) {
1949 var copied map[string][]string
1950 if attachments != nil {
1951 copied = make(map[string][]string, len(attachments))
1952 for k, v := range attachments {
1953 sl := make([]string, len(v))
1954 copy(sl, v)
1955 copied[k] = sl
1956 }
1957 }
1958 s.mu.Lock()
1959 s.config.Attachments = copied
1960 s.mu.Unlock()
1961}
1962
1963// Helper methods
1964
1965// cleanupFailedContainer stops and removes a container that failed to start properly.
1966func (s *Service) cleanupFailedContainer(ctx context.Context, containerID string) {
1967 cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), failedContainerCleanupTimeout)
1968 defer cancel()
1969
1970 log := zerowrap.FromCtx(cleanupCtx)
1971 if err := s.runtime.StopContainer(cleanupCtx, containerID); err != nil {
1972 log.Warn().Err(err).Str(zerowrap.FieldEntityID, containerID).Msg("failed to stop container after failure")
1973 }
1974 if err := s.runtime.RemoveContainer(cleanupCtx, containerID, true); err != nil {
1975 log.Warn().Err(err).Str(zerowrap.FieldEntityID, containerID).Msg("failed to remove container after failure")
1976 }
1977}
1978
1979// cleanupOldContainer stops and removes an old container after zero-downtime switch.
1980// It also renames the new container to the canonical name.
1981func (s *Service) cleanupOldContainer(ctx context.Context, old *domain.Container, newContainerID, domainName string) {
1982 log := zerowrap.FromCtx(ctx)
1983 log.Info().Str(zerowrap.FieldEntityID, old.ID).Msg("stopping old container after zero-downtime switch")
1984
1985 if s.logWriter != nil {
1986 if err := s.logWriter.StopLogging(old.ID); err != nil {
1987 log.Warn().Err(err).Str(zerowrap.FieldEntityID, old.ID).Msg("failed to stop logging for old container")
1988 }
1989 }
1990
1991 if err := s.runtime.StopContainer(ctx, old.ID); err != nil {
1992 log.Warn().Err(err).Str(zerowrap.FieldEntityID, old.ID).Msg("failed to stop old container")
1993 }
1994
1995 if err := s.runtime.RemoveContainer(ctx, old.ID, true); err != nil {
1996 log.Warn().Err(err).Str(zerowrap.FieldEntityID, old.ID).Msg("failed to remove old container")
1997 }
1998
1999 // Rename new container to canonical name
2000 canonicalName := managedContainerName(domainName)
2001 if err := s.runtime.RenameContainer(ctx, newContainerID, canonicalName); err != nil {
2002 log.Warn().Err(err).Str("canonical_name", canonicalName).Msg("failed to rename container to canonical name")
2003 }
2004}
2005
2006// startLogCollection starts log collection for a container in the background.
2007// Errors are logged but don't fail the calling operation.
2008func (s *Service) startLogCollection(ctx context.Context, containerID, domainName string) {
2009 if s.logWriter == nil {
2010 return
2011 }
2012
2013 log := zerowrap.FromCtx(ctx)
2014
2015 // Use context.WithoutCancel so the log stream outlives the HTTP request.
2016 // Without this, the Docker log stream closes when the deploy response completes.
2017 bgCtx := context.WithoutCancel(ctx)
2018
2019 logStream, err := s.runtime.GetContainerLogs(bgCtx, containerID, true)
2020 if err != nil {
2021 log.Warn().Err(err).Msg("failed to get container logs for collection")
2022 return
2023 }
2024
2025 if err := s.logWriter.StartLogging(bgCtx, containerID, domainName, logStream); err != nil {
2026 log.Warn().Err(err).Msg("failed to start container log collection")
2027 logStream.Close()
2028 }
2029}
2030
2031func (s *Service) buildValidatedImageRef(ctx context.Context, image string) (string, error) {
2032 imageRef := s.buildImageRef(image)
2033 if err := s.validateImageRef(ctx, imageRef); err != nil {
2034 return "", err
2035 }
2036 return imageRef, nil
2037}
2038
2039func (s *Service) buildImageRef(image string) string {
2040 s.mu.RLock()
2041 cfg := s.config
2042 s.mu.RUnlock()
2043
2044 if !cfg.RegistryAuthEnabled || cfg.RegistryDomain == "" {
2045 return image
2046 }
2047
2048 // Normalize registry domain by trimming any trailing slash
2049 reg := strings.TrimSuffix(cfg.RegistryDomain, "/")
2050
2051 // Check if image already has the registry domain prefix
2052 prefix := reg + "/"
2053 if strings.HasPrefix(image, prefix) {
2054 return image
2055 }
2056
2057 // Check if image already has an explicit registry (host:port/ or host.domain/).
2058 // We need to detect patterns like:
2059 // - "docker.io/library/nginx:latest" (has dot and slash)
2060 // - "localhost:5001/myapp:latest" (has host:port/ pattern)
2061 // - "myregistry:5000/app:v1" (has host:port/ pattern)
2062 // - "[fd00::1]:5000/app:v1" (IPv6 with port)
2063 if hasExplicitRegistry(image) {
2064 return image
2065 }
2066
2067 return fmt.Sprintf("%s/%s", reg, image)
2068}
2069
2070// hasExplicitRegistry checks if an image reference already includes an explicit registry.
2071// Returns true for patterns like "docker.io/image", "localhost:5000/image", "localhost/image",
2072// "registry:8080/image", "[fd00::1]/image", "[fd00::1]:5000/image".
2073func hasExplicitRegistry(image string) bool {
2074 // Find the first slash which separates registry from image name
2075 slashIdx := strings.Index(image, "/")
2076 if slashIdx == -1 {
2077 return false // No slash means no registry prefix (e.g., "myapp:latest")
2078 }
2079
2080 // Extract the part before the first slash (potential registry)
2081 registryPart := image[:slashIdx]
2082
2083 // Check for bracketed IPv6 address (e.g., "[fd00::1]" or "[fd00::1]:5000")
2084 if strings.HasPrefix(registryPart, "[") {
2085 // Look for closing bracket
2086 if closeBracket := strings.Index(registryPart, "]"); closeBracket != -1 {
2087 // Valid bracketed IPv6: either ends at ] or has ]:port
2088 return true
2089 }
2090 }
2091
2092 // Check for localhost (with or without port)
2093 if registryPart == "localhost" {
2094 return true
2095 }
2096
2097 // Check for host:port pattern (e.g., "localhost:5000", "registry:8080")
2098 if colonIdx := strings.LastIndex(registryPart, ":"); colonIdx != -1 {
2099 port := registryPart[colonIdx+1:]
2100 // If everything after colon is digits, it's a port
2101 if len(port) > 0 && isNumeric(port) {
2102 return true
2103 }
2104 }
2105
2106 // Check for domain pattern (has a dot, e.g., "docker.io", "gcr.io")
2107 if strings.Contains(registryPart, ".") {
2108 return true
2109 }
2110
2111 return false
2112}
2113
2114func (s *Service) validateImageRef(ctx context.Context, imageRef string) error {
2115 if !hasExplicitRegistry(imageRef) {
2116 return nil
2117 }
2118 return s.validateExternalImageRef(ctx, imageRef, imageRegistry(imageRef))
2119}
2120
2121func (s *Service) validateImagePullRef(ctx context.Context, imageRef string) error {
2122 return s.validateExternalImageRef(ctx, imageRef, imageRegistryForPolicy(imageRef))
2123}
2124
2125func (s *Service) validateExternalImageRef(ctx context.Context, imageRef, registry string) error {
2126 s.mu.RLock()
2127 cfg := s.config
2128 s.mu.RUnlock()
2129
2130 if registry == "" {
2131 return fmt.Errorf("image %q rejected: invalid image reference", imageRef)
2132 }
2133
2134 if domain.IsGordonRegistryImageRef(imageRef, cfg.RegistryDomain, cfg.LegacyRegistryDomains) || sameRegistry(registry, cfg.RegistryDomain) {
2135 return nil
2136 }
2137 dangerous, err := isDangerousRegistryHost(ctx, imageRegistryHost(registry))
2138 if err != nil {
2139 return fmt.Errorf("image %q rejected: registry %q could not be verified: %w", imageRef, registry, err)
2140 }
2141 if dangerous {
2142 return fmt.Errorf("image %q rejected: registry %q is not allowed", imageRef, registry)
2143 }
2144 if len(cfg.AllowedRegistries) == 0 {
2145 return fmt.Errorf("image %q rejected: registry %q is not in images.allowed_registries", imageRef, registry)
2146 }
2147 for _, allowed := range cfg.AllowedRegistries {
2148 if sameRegistry(registry, allowed) {
2149 if cfg.RequireImageDigest && !isValidSha256DigestRef(imageRef) {
2150 return fmt.Errorf("image %q rejected: digest reference required", imageRef)
2151 }
2152 return nil
2153 }
2154 }
2155 return fmt.Errorf("image %q rejected: registry %q is not in images.allowed_registries", imageRef, registry)
2156}
2157
2158func imageRegistry(imageRef string) string {
2159 slashIdx := strings.Index(imageRef, "/")
2160 if slashIdx == -1 {
2161 return ""
2162 }
2163 return normalizeRegistry(imageRef[:slashIdx])
2164}
2165
2166func imageRegistryForPolicy(imageRef string) string {
2167 if hasExplicitRegistry(imageRef) {
2168 return imageRegistry(imageRef)
2169 }
2170 return "docker.io"
2171}
2172
2173// normalizeRegistry normalizes a registry string for comparison purposes only.
2174// This function is used by sameRegistry to determine if two registry strings
2175// refer to the same registry. The output intentionally strips IPv6 brackets
2176// (e.g., "[fd00::1]:5000" -> "fd00::1:5000") which is acceptable for equality
2177// checks but the result MUST NOT be used to construct or emit valid registry URLs.
2178func normalizeRegistry(registry string) string {
2179 registry = strings.ToLower(strings.TrimSuffix(strings.TrimSpace(registry), "."))
2180 registry = strings.TrimSuffix(registry, "/")
2181 if strings.HasPrefix(registry, "[") {
2182 if end := strings.Index(registry, "]"); end != -1 {
2183 host := strings.Trim(registry[:end+1], "[]")
2184 port := registry[end+1:]
2185 return host + port
2186 }
2187 }
2188 return registry
2189}
2190
2191func imageRegistryHost(registry string) string {
2192 if ip := net.ParseIP(registry); ip != nil {
2193 return registry
2194 }
2195 if h, _, err := net.SplitHostPort(registry); err == nil {
2196 return strings.Trim(h, "[]")
2197 }
2198 if strings.HasPrefix(registry, "[") && strings.Contains(registry, "]") {
2199 return strings.TrimPrefix(strings.Split(registry, "]")[0], "[")
2200 }
2201 if colon := strings.LastIndex(registry, ":"); colon != -1 && isNumeric(registry[colon+1:]) {
2202 return registry[:colon]
2203 }
2204 return registry
2205}
2206
2207func sameRegistry(a, b string) bool {
2208 return normalizeRegistry(a) == normalizeRegistry(b)
2209}
2210
2211func isDangerousRegistryHost(ctx context.Context, host string) (bool, error) {
2212 host = strings.ToLower(strings.TrimSuffix(host, "."))
2213 if host == "localhost" || host == "metadata.google.internal" {
2214 return true, nil
2215 }
2216 if ip := net.ParseIP(host); ip != nil {
2217 return isDangerousRegistryIP(ip), nil
2218 }
2219 addrs, err := net.DefaultResolver.LookupIPAddr(ctx, host)
2220 if err != nil {
2221 return false, err
2222 }
2223 for _, addr := range addrs {
2224 if isDangerousRegistryIP(addr.IP) {
2225 return true, nil
2226 }
2227 }
2228 return false, nil
2229}
2230
2231func isDangerousRegistryIP(ip net.IP) bool {
2232 return ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsUnspecified()
2233}
2234
2235func applySecurityProfile(config *domain.ContainerConfig, cfg Config) {
2236 if strings.EqualFold(cfg.SecurityProfile, "strict") {
2237 config.ReadOnlyRootFS = true
2238 config.CapDrop = []string{"ALL"}
2239 config.CapAdd = []string{"NET_BIND_SERVICE"}
2240 }
2241}
2242
2243// isNumeric checks if a string contains only digits.
2244func isNumeric(s string) bool {
2245 for _, c := range s {
2246 if c < '0' || c > '9' {
2247 return false
2248 }
2249 }
2250 return true
2251}
2252
2253func normalizePullPolicy(policy string) string {
2254 switch strings.ToLower(strings.TrimSpace(policy)) {
2255 case PullPolicyAlways:
2256 return PullPolicyAlways
2257 case PullPolicyIfTagChanged:
2258 return PullPolicyIfTagChanged
2259 case PullPolicyIfNotPresent:
2260 return PullPolicyIfNotPresent
2261 default:
2262 return PullPolicyIfNotPresent
2263 }
2264}
2265
2266func isDigestRef(imageRef string) bool {
2267 return isValidSha256DigestRef(imageRef)
2268}
2269
2270func isValidSha256DigestRef(imageRef string) bool {
2271 _, digest, ok := strings.Cut(imageRef, "@sha256:")
2272 if !ok || len(digest) != 64 {
2273 return false
2274 }
2275 _, err := hex.DecodeString(digest)
2276 return err == nil
2277}
2278
2279func (s *Service) pullRefForDeploy(ctx context.Context, imageRef string) (string, bool) {
2280 if !domain.IsInternalDeploy(ctx) {
2281 return imageRef, false
2282 }
2283 s.mu.RLock()
2284 cfg := s.config
2285 s.mu.RUnlock()
2286 return rewriteToLocalRegistry(imageRef, cfg.RegistryDomain, cfg.LegacyRegistryDomains, cfg.RegistryPort), true
2287}
2288
2289// ensureImage ensures the image is available locally, pulling if needed.
2290// Returns the image reference to use for container operations:
2291// - For digest references (@sha256:...), returns the pullRef since Docker can't tag digests
2292// - For tagged images, returns the original imageRef after tagging the pulled image
2293func (s *Service) ensureImage(ctx context.Context, imageRef string) (string, error) {
2294 ctx, span := tracer.Start(ctx, "container.ensure_image",
2295 trace.WithAttributes(attribute.String("image", imageRef)))
2296 defer span.End()
2297
2298 ctx = zerowrap.CtxWithField(ctx, "image", imageRef)
2299 log := zerowrap.FromCtx(ctx)
2300
2301 // Determine if this is an internal deploy and what reference to use for pulls.
2302 pullRef, isInternal := s.pullRefForDeploy(ctx, imageRef)
2303 if err := s.validateImagePullRef(ctx, imageRef); err != nil {
2304 return "", err
2305 }
2306 if pullRef != imageRef {
2307 log.Info().
2308 Str("original_ref", imageRef).
2309 Str("pull_ref", pullRef).
2310 Msg("internal deploy: using localhost registry for pull")
2311 }
2312
2313 found, err := s.ensureLocalImage(ctx, imageRef, pullRef)
2314 if err != nil {
2315 return "", err
2316 }
2317 if found {
2318 return imageRef, nil
2319 }
2320
2321 // Pull image
2322 log.Info().Msg("pulling image from registry")
2323
2324 if err := s.pullImage(ctx, pullRef, isInternal); err != nil {
2325 return "", err
2326 }
2327
2328 // For digest references, we can't create a tag (Docker doesn't allow it).
2329 // In this case, use the pullRef directly since the image is available under that reference.
2330 if strings.Contains(imageRef, "@sha256:") {
2331 log.Info().
2332 Str("pull_ref", pullRef).
2333 Msg("digest reference: using pull reference for container operations")
2334 return pullRef, nil
2335 }
2336
2337 if err := s.tagImageIfNeeded(ctx, pullRef, imageRef); err != nil {
2338 return "", err
2339 }
2340
2341 // Clean up the temporary pull reference tag to avoid duplicate entries
2342 if pullRef != imageRef {
2343 if err := s.runtime.UntagImage(ctx, pullRef); err != nil {
2344 // Log but don't fail - the canonical tag is already applied
2345 log.Debug().Err(err).Str("pull_ref", pullRef).Msg("failed to remove temporary pull tag")
2346 }
2347 }
2348
2349 log.Info().Msg("image pulled successfully")
2350 return imageRef, nil
2351}
2352
2353func (s *Service) ensureLocalImage(ctx context.Context, imageRef, pullRef string) (bool, error) {
2354 log := zerowrap.FromCtx(ctx)
2355
2356 // For internal deploys (image push events), always pull fresh image
2357 // because the same tag may reference new content (e.g., latest tag updated)
2358 if domain.IsInternalDeploy(ctx) {
2359 log.Info().Msg("internal deploy detected, forcing image pull to ensure latest content")
2360 return false, nil
2361 }
2362
2363 s.mu.RLock()
2364 cfg := s.config
2365 s.mu.RUnlock()
2366
2367 pullPolicy := normalizePullPolicy(cfg.PullPolicy)
2368 switch pullPolicy {
2369 case PullPolicyAlways:
2370 log.Info().Str("pull_policy", pullPolicy).Msg("pull policy forces image pull")
2371 return false, nil
2372 case PullPolicyIfTagChanged:
2373 if !isDigestRef(imageRef) {
2374 log.Info().Str("pull_policy", pullPolicy).Msg("tag reference detected, pulling to check for updates")
2375 return false, nil
2376 }
2377 }
2378
2379 localImages, err := s.runtime.ListImages(ctx)
2380 if err != nil {
2381 log.WrapErr(err, "failed to list local images, will attempt pull")
2382 return false, nil
2383 }
2384
2385 normalizedRef := normalizeImageRef(imageRef)
2386 normalizedPullRef := normalizeImageRef(pullRef)
2387 for _, img := range localImages {
2388 normalizedImage := normalizeImageRef(img)
2389 if normalizedImage == normalizedRef {
2390 log.Info().Msg("image found locally, skipping pull")
2391 return true, nil
2392 }
2393 if normalizedImage == normalizedPullRef {
2394 if err := s.tagImageIfNeeded(ctx, pullRef, imageRef); err != nil {
2395 return false, err
2396 }
2397 // Clean up the temporary pull reference tag
2398 if pullRef != imageRef {
2399 if err := s.runtime.UntagImage(ctx, pullRef); err != nil {
2400 log.Debug().Err(err).Str("pull_ref", pullRef).Msg("failed to remove temporary pull tag")
2401 }
2402 }
2403 log.Info().Msg("image found locally, skipping pull")
2404 return true, nil
2405 }
2406 }
2407
2408 return false, nil
2409}
2410
2411func (s *Service) pullImage(ctx context.Context, pullRef string, isInternal bool) error {
2412 s.mu.RLock()
2413 cfg := s.config
2414 s.mu.RUnlock()
2415
2416 log := zerowrap.FromCtx(ctx)
2417
2418 switch {
2419 case isInternal && cfg.RegistryAuthEnabled:
2420 if cfg.InternalRegistryUsername == "" || cfg.InternalRegistryPassword == "" {
2421 return log.WrapErr(fmt.Errorf("internal registry auth not configured"), "failed to pull image for internal deploy")
2422 }
2423 if err := s.pullImageWithRetry(ctx, func(ctx context.Context) error {
2424 return s.runtime.PullImageWithAuth(ctx, pullRef, cfg.InternalRegistryUsername, cfg.InternalRegistryPassword)
2425 }, isInternal); err != nil {
2426 return log.WrapErr(fmt.Errorf("%w: %w", domain.ErrImagePullFailed, err), "failed to pull image with internal auth")
2427 }
2428 case isInternal:
2429 if err := s.pullImageWithRetry(ctx, func(ctx context.Context) error {
2430 return s.runtime.PullImage(ctx, pullRef)
2431 }, isInternal); err != nil {
2432 return log.WrapErr(fmt.Errorf("%w: %w", domain.ErrImagePullFailed, err), "failed to pull image")
2433 }
2434 default:
2435 // Never send Gordon's service credentials to an external registry.
2436 // External pulls are anonymous until per-registry credentials have an
2437 // explicit trust-boundary-aware configuration path.
2438 if err := s.runtime.PullImage(ctx, pullRef); err != nil {
2439 return log.WrapErr(fmt.Errorf("%w: %w", domain.ErrImagePullFailed, err), "failed to pull image")
2440 }
2441 }
2442
2443 return nil
2444}
2445
2446func (s *Service) pullImageWithRetry(ctx context.Context, pullFn func(context.Context) error, isInternal bool) error {
2447 log := zerowrap.FromCtx(ctx)
2448
2449 for attempt := 1; attempt <= internalPullMaxAttempts; attempt++ {
2450 err := pullFn(ctx)
2451 if err == nil {
2452 return nil
2453 }
2454 if !isInternal || !isConnectionRefusedError(err) || attempt == internalPullMaxAttempts {
2455 return fmt.Errorf("internal image pull failed after %d attempts: %w", attempt, err)
2456 }
2457
2458 backoff := time.Duration(attempt) * time.Second
2459 log.Warn().
2460 Err(err).
2461 Int("attempt", attempt).
2462 Dur("backoff", backoff).
2463 Msg("internal image pull failed with connection refused, retrying")
2464
2465 select {
2466 case <-time.After(backoff):
2467 case <-ctx.Done():
2468 return ctx.Err()
2469 }
2470 }
2471 return nil
2472}
2473
2474func (s *Service) tagImageIfNeeded(ctx context.Context, sourceRef, targetRef string) error {
2475 if sourceRef == targetRef {
2476 return nil
2477 }
2478
2479 // Cannot create a tag with a digest reference - Docker/Podman doesn't allow it.
2480 // When using digest references (image@sha256:...), skip tagging as the image
2481 // is already available by its digest.
2482 if strings.Contains(targetRef, "@sha256:") {
2483 return nil
2484 }
2485
2486 log := zerowrap.FromCtx(ctx)
2487 if err := s.runtime.TagImage(ctx, sourceRef, targetRef); err != nil {
2488 return log.WrapErr(err, "failed to tag image from pull reference")
2489 }
2490 return nil
2491}
2492
2493func (s *Service) loadEnvironment(ctx context.Context, preResolved []string, domainName, imageRef string) ([]string, error) {
2494 log := zerowrap.FromCtx(ctx)
2495
2496 var userEnvVars []string
2497 if len(preResolved) > 0 {
2498 userEnvVars = preResolved
2499 } else {
2500 var err error
2501 userEnvVars, err = s.envLoader.LoadEnv(ctx, domainName)
2502 if err != nil {
2503 return nil, log.WrapErr(err, "failed to load environment variables")
2504 }
2505 }
2506
2507 dockerfileEnvVars, err := s.runtime.InspectImageEnv(ctx, imageRef)
2508 if err != nil {
2509 log.WrapErr(err, "failed to inspect image environment")
2510 dockerfileEnvVars = []string{}
2511 }
2512
2513 return mergeEnvironmentVariables(dockerfileEnvVars, userEnvVars), nil
2514}
2515
2516func (s *Service) setupVolumes(ctx context.Context, domainName, imageRef string) (map[string]string, error) {
2517 s.mu.RLock()
2518 cfg := s.config
2519 s.mu.RUnlock()
2520
2521 log := zerowrap.FromCtx(ctx)
2522 volumes := make(map[string]string)
2523
2524 if !cfg.VolumeAutoCreate {
2525 return volumes, nil
2526 }
2527
2528 volumePaths, err := s.runtime.InspectImageVolumes(ctx, imageRef)
2529 if err != nil {
2530 log.WrapErr(err, "failed to inspect image volumes")
2531 return volumes, nil
2532 }
2533
2534 for _, path := range volumePaths {
2535 name := generateVolumeName(cfg.VolumePrefix, domainName, path)
2536 legacyName := legacyVolumeName(cfg.VolumePrefix, domainName, path)
2537 legacyExists, err := s.runtime.VolumeExists(ctx, legacyName)
2538 if err != nil {
2539 log.WrapErrWithFields(err, "failed to check legacy volume", map[string]any{"volume": legacyName})
2540 continue
2541 }
2542 if legacyExists {
2543 owned, ownershipErr := s.legacyVolumeOwnedByDomain(ctx, legacyName, domainName)
2544 if ownershipErr != nil {
2545 log.Warn().Err(ownershipErr).Str("volume", legacyName).Msg("could not verify legacy volume ownership; using stable volume name")
2546 } else if owned {
2547 volumes[path] = legacyName
2548 continue
2549 }
2550 }
2551
2552 exists, err := s.runtime.VolumeExists(ctx, name)
2553 if err != nil {
2554 log.WrapErrWithFields(err, "failed to check volume", map[string]any{"volume": name})
2555 continue
2556 }
2557
2558 if !exists {
2559 if err := s.runtime.CreateVolume(ctx, name); err != nil {
2560 log.WrapErrWithFields(err, "failed to create volume", map[string]any{"volume": name})
2561 continue
2562 }
2563 log.Info().Str("volume", name).Str(zerowrap.FieldPath, path).Msg("created volume")
2564 }
2565
2566 volumes[path] = name
2567 }
2568
2569 return volumes, nil
2570}
2571
2572func (s *Service) getNetworkForApp(domainName string) string {
2573 var networkIsolation bool
2574 var networkGroups map[string][]string
2575
2576 if s.configProvider != nil {
2577 snap := s.configProvider.GetAttachmentConfig()
2578 networkGroups = snap.NetworkGroups
2579 s.mu.RLock()
2580 networkIsolation = s.config.NetworkIsolation
2581 s.mu.RUnlock()
2582 } else {
2583 s.mu.RLock()
2584 networkIsolation = s.config.NetworkIsolation
2585 networkGroups = s.config.NetworkGroups
2586 s.mu.RUnlock()
2587 }
2588
2589 if !networkIsolation {
2590 return "bridge"
2591 }
2592
2593 for groupName, domains := range networkGroups {
2594 if slices.Contains(domains, domainName) {
2595 return s.generateNetworkName(groupName)
2596 }
2597 }
2598
2599 return s.generateNetworkName(domainName)
2600}
2601
2602func (s *Service) generateNetworkName(identifier string) string {
2603 s.mu.RLock()
2604 cfg := s.config
2605 s.mu.RUnlock()
2606
2607 return fmt.Sprintf("%s-%s", cfg.NetworkPrefix, domain.StableResourceName(identifier))
2608}
2609
2610func (s *Service) createNetworkIfNeeded(ctx context.Context, networkName string) error {
2611 if networkName == "bridge" || networkName == "default" {
2612 return nil
2613 }
2614
2615 ctx = zerowrap.CtxWithField(ctx, "network", networkName)
2616 log := zerowrap.FromCtx(ctx)
2617
2618 exists, err := s.runtime.NetworkExists(ctx, networkName)
2619 if err != nil {
2620 return log.WrapErr(err, "failed to check network existence")
2621 }
2622
2623 if !exists {
2624 s.mu.RLock()
2625 internal := s.config.NetworkInternal
2626 s.mu.RUnlock()
2627 if err := s.runtime.CreateNetwork(ctx, networkName, domain.NetworkConfig{
2628 Driver: "bridge",
2629 Internal: internal,
2630 Labels: map[string]string{domain.LabelManaged: "true"},
2631 }); err != nil {
2632 return log.WrapErr(err, "failed to create network")
2633 }
2634 log.Info().Msg("created network for app isolation")
2635 }
2636
2637 return nil
2638}
2639
2640func (s *Service) cleanupOrphanedContainers(ctx context.Context, domainName string, skipContainerID string) error {
2641 log := zerowrap.FromCtx(ctx)
2642 expectedName := managedContainerName(domainName)
2643 expectedNewName := expectedName + "-new"
2644 expectedNextName := expectedName + "-next"
2645
2646 allContainers, err := s.runtime.ListContainers(ctx, true)
2647 if err != nil {
2648 return err
2649 }
2650
2651 for _, c := range allContainers {
2652 if (c.Name == expectedName || c.Name == expectedNewName || c.Name == expectedNextName) && c.ID != skipContainerID {
2653 // Without a selected active container, preserve anything that may still
2654 // be serving traffic. When an active container is selected, other active
2655 // temp-name containers are stale candidates left by interrupted deploys.
2656 isActive := c.Status == "running" || c.Status == "restarting"
2657 if isActive && (skipContainerID == "" || c.Name == expectedName) {
2658 log.Debug().
2659 Str(zerowrap.FieldEntityID, c.ID).
2660 Str("container_name", c.Name).
2661 Str(zerowrap.FieldStatus, c.Status).
2662 Msg("skipping active container during orphan cleanup")
2663 continue
2664 }
2665
2666 log.Info().Str(zerowrap.FieldEntityID, c.ID).Str("container_name", c.Name).Str(zerowrap.FieldStatus, c.Status).Msg("found orphaned container, removing")
2667
2668 if err := s.runtime.StopContainer(ctx, c.ID); err != nil {
2669 log.WrapErrWithFields(err, "failed to stop orphaned container", map[string]any{zerowrap.FieldEntityID: c.ID})
2670 }
2671
2672 if err := s.runtime.RemoveContainer(ctx, c.ID, true); err != nil {
2673 log.WrapErrWithFields(err, "failed to remove orphaned container", map[string]any{zerowrap.FieldEntityID: c.ID})
2674 }
2675 }
2676 }
2677
2678 return nil
2679}
2680
2681func (s *Service) cleanupVolumesForDomain(_ context.Context, _ string) error {
2682 return nil
2683}
2684
2685func (s *Service) cleanupNetworkIfEmpty(ctx context.Context, networkName string) error {
2686 if networkName == "bridge" || networkName == "default" {
2687 return nil
2688 }
2689
2690 ctx = zerowrap.CtxWithField(ctx, "network", networkName)
2691 log := zerowrap.FromCtx(ctx)
2692
2693 networks, err := s.runtime.ListNetworks(ctx)
2694 if err != nil {
2695 return log.WrapErr(err, "failed to list networks")
2696 }
2697
2698 for _, n := range networks {
2699 if n.Name == networkName && len(n.Containers) == 0 && n.Labels[domain.LabelManaged] == "true" {
2700 if err := s.runtime.RemoveNetwork(ctx, networkName); err != nil {
2701 return err
2702 }
2703 log.Info().Msg("cleaned up empty network")
2704 break
2705 }
2706 }
2707
2708 return nil
2709}
2710
2711func (s *Service) deployAttachments(ctx context.Context, domainName, networkName string) error {
2712 // Collect attachments from both domain-specific and network group configs
2713 attachments := s.resolveAttachmentsForDomain(domainName)
2714 if len(attachments) == 0 {
2715 return nil
2716 }
2717
2718 log := zerowrap.FromCtx(ctx)
2719 var deployed []string // track successfully deployed attachment IDs for rollback
2720 for _, svc := range attachments {
2721 if err := s.deployAttachedService(ctx, domainName, svc, networkName); err != nil {
2722 log.WrapErrWithFields(err, "failed to deploy attachment", map[string]any{zerowrap.FieldService: svc, "domain": domainName})
2723
2724 // Rollback: clean up already-deployed attachments
2725 for _, id := range deployed {
2726 if stopErr := s.runtime.StopContainer(ctx, id); stopErr != nil {
2727 log.Warn().Err(stopErr).Str("attachment_id", id).Msg("rollback: failed to stop attachment")
2728 }
2729 if rmErr := s.runtime.RemoveContainer(ctx, id, true); rmErr != nil {
2730 log.Warn().Err(rmErr).Str("attachment_id", id).Msg("rollback: failed to remove attachment")
2731 }
2732 }
2733 // Deregister rolled-back attachments
2734 if len(deployed) > 0 {
2735 s.mu.Lock()
2736 remaining := s.attachments[domainName]
2737 filtered := remaining[:0]
2738 rollbackSet := make(map[string]bool, len(deployed))
2739 for _, id := range deployed {
2740 rollbackSet[id] = true
2741 }
2742 for _, id := range remaining {
2743 if !rollbackSet[id] {
2744 filtered = append(filtered, id)
2745 }
2746 }
2747 s.attachments[domainName] = filtered
2748 s.mu.Unlock()
2749 }
2750
2751 return fmt.Errorf("failed to deploy attachment %q (rolled back %d already-deployed)", svc, len(deployed))
2752 }
2753 // Collect IDs of attachments tracked under this domain after successful deploy
2754 s.mu.RLock()
2755 ids := s.attachments[domainName]
2756 if len(ids) > 0 {
2757 latest := ids[len(ids)-1]
2758 deployed = append(deployed, latest)
2759 }
2760 s.mu.RUnlock()
2761 }
2762 return nil
2763}
2764
2765// resolveAttachmentsForDomain returns attachments for a domain by checking:
2766// 1. Direct domain attachments (attachments[domain])
2767// 2. Network group attachments (attachments[group] where domain is in network_groups[group])
2768func (s *Service) resolveAttachmentsForDomain(domainName string) []string {
2769 var attachments map[string][]string
2770 var networkGroups map[string][]string
2771
2772 if s.configProvider != nil {
2773 snap := s.configProvider.GetAttachmentConfig()
2774 attachments = snap.Attachments
2775 networkGroups = snap.NetworkGroups
2776 } else {
2777 s.mu.RLock()
2778 attachments = s.config.Attachments
2779 networkGroups = s.config.NetworkGroups
2780 s.mu.RUnlock()
2781 }
2782
2783 seen := make(map[string]bool)
2784 var result []string
2785
2786 // First, add domain-specific attachments
2787 if domainAttachments, ok := attachments[domainName]; ok {
2788 for _, img := range domainAttachments {
2789 if !seen[img] {
2790 seen[img] = true
2791 result = append(result, img)
2792 }
2793 }
2794 }
2795
2796 // Then, find which network group this domain belongs to and add group attachments
2797 for groupName, domains := range networkGroups {
2798 if slices.Contains(domains, domainName) {
2799 if groupAttachments, ok := attachments[groupName]; ok {
2800 for _, img := range groupAttachments {
2801 if !seen[img] {
2802 seen[img] = true
2803 result = append(result, img)
2804 }
2805 }
2806 }
2807 break // Domain can only be in one network group
2808 }
2809 }
2810
2811 return result
2812}
2813
2814func (s *Service) getAttachmentsForDomain(ctx context.Context, domainName string) []domain.Attachment {
2815 containers, err := s.runtime.ListContainers(ctx, true)
2816 if err != nil {
2817 return nil
2818 }
2819
2820 return s.filterAttachments(ctx, containers, domainName)
2821}
2822
2823// getAllAttachments fetches all containers once and returns attachments grouped by domain.
2824// This avoids N+1 queries when listing multiple routes.
2825func (s *Service) getAllAttachments(ctx context.Context) map[string][]domain.Attachment {
2826 containers, err := s.runtime.ListContainers(ctx, true)
2827 if err != nil {
2828 return nil
2829 }
2830
2831 result := make(map[string][]domain.Attachment)
2832 for _, container := range containers {
2833 if container.Labels == nil {
2834 continue
2835 }
2836 if container.Labels[domain.LabelAttachment] != "true" {
2837 continue
2838 }
2839 ownerDomain := container.Labels[domain.LabelAttachedTo]
2840 if ownerDomain == "" {
2841 continue
2842 }
2843 image := container.Image
2844 serviceName := container.Name
2845 if labelImage, ok := container.Labels[domain.LabelImage]; ok && labelImage != "" {
2846 image = labelImage
2847 serviceName = extractServiceName(labelImage)
2848 }
2849 // Get the container's network for display in the routes table
2850 network := ""
2851 if networkName, err := s.runtime.GetContainerNetwork(ctx, container.ID); err == nil {
2852 network = networkName
2853 }
2854 attachment := domain.Attachment{
2855 Name: serviceName,
2856 Image: s.stripRegistryPrefix(image),
2857 ContainerID: container.ID,
2858 Status: container.Status,
2859 Network: network,
2860 Ports: append([]int(nil), container.Ports...),
2861 }
2862 result[ownerDomain] = append(result[ownerDomain], attachment)
2863 }
2864
2865 return result
2866}
2867
2868// filterAttachments extracts attachments for a specific domain from a container list.
2869func (s *Service) filterAttachments(ctx context.Context, containers []*domain.Container, domainName string) []domain.Attachment {
2870 attachments := make([]domain.Attachment, 0)
2871 for _, container := range containers {
2872 if container.Labels == nil {
2873 continue
2874 }
2875 if container.Labels[domain.LabelAttachment] != "true" {
2876 continue
2877 }
2878 if container.Labels[domain.LabelAttachedTo] != domainName {
2879 continue
2880 }
2881 image := container.Image
2882 serviceName := container.Name
2883 if labelImage, ok := container.Labels[domain.LabelImage]; ok && labelImage != "" {
2884 image = labelImage
2885 serviceName = extractServiceName(labelImage)
2886 }
2887 // Get the container's network for display in the routes table
2888 network := ""
2889 if networkName, err := s.runtime.GetContainerNetwork(ctx, container.ID); err == nil {
2890 network = networkName
2891 }
2892 attachment := domain.Attachment{
2893 Name: serviceName,
2894 Image: s.stripRegistryPrefix(image),
2895 ContainerID: container.ID,
2896 Status: container.Status,
2897 Network: network,
2898 Ports: append([]int(nil), container.Ports...),
2899 }
2900 attachments = append(attachments, attachment)
2901 }
2902
2903 return attachments
2904}
2905
2906func (s *Service) deployAttachedService(ctx context.Context, ownerDomain, serviceImage, networkName string) error {
2907 log := zerowrap.FromCtx(ctx)
2908
2909 // Parse service name from image (e.g., "my-postgres:latest" → "postgres")
2910 serviceName := extractServiceName(serviceImage)
2911 containerName := fmt.Sprintf("gordon-%s-%s", sanitizeName(ownerDomain), serviceName)
2912
2913 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
2914 "attachment": serviceName,
2915 "container_name": containerName,
2916 "owner_domain": ownerDomain,
2917 })
2918
2919 // Check if already running (idempotent)
2920 // Try new name first, then fall back to legacy name for backwards compatibility
2921 existingContainer, err := s.resolveExistingAttachment(ctx, ownerDomain, serviceName)
2922 if err != nil {
2923 return err
2924 }
2925
2926 if existingContainer != nil {
2927 shouldSkip, err := s.handleRunningAttachment(ctx, existingContainer, containerName, serviceImage)
2928 if err != nil {
2929 return err
2930 }
2931 if shouldSkip {
2932 return nil
2933 }
2934 if existingContainer.Status == string(domain.ContainerStatusRunning) {
2935 existingContainer = nil
2936 }
2937 }
2938
2939 if err := s.removeStoppedAttachment(ctx, existingContainer, containerName); err != nil {
2940 return err
2941 }
2942
2943 log.Info().Str(zerowrap.FieldService, serviceImage).Msg("deploying attached service")
2944
2945 config, err := s.buildAttachmentContainerConfig(ctx, ownerDomain, serviceName, serviceImage, containerName, networkName)
2946 if err != nil {
2947 return err
2948 }
2949
2950 container, err := s.runtime.CreateContainer(ctx, config)
2951 if err != nil {
2952 return log.WrapErr(err, "failed to create attachment container")
2953 }
2954
2955 // Start container
2956 if err := s.runtime.StartContainer(ctx, container.ID); err != nil {
2957 s.runtime.RemoveContainer(ctx, container.ID, true)
2958 return log.WrapErr(err, "failed to start attachment container")
2959 }
2960
2961 // Wait for attachment to be ready before proceeding
2962 if err := s.waitForAttachmentReady(ctx, container.ID, config); err != nil {
2963 log.WrapErr(err, "attachment readiness check failed, cleaning up")
2964 if stopErr := s.runtime.StopContainer(ctx, container.ID); stopErr != nil {
2965 log.Warn().Err(stopErr).Str(zerowrap.FieldEntityID, container.ID).Msg("failed to stop unready attachment")
2966 }
2967 s.runtime.RemoveContainer(ctx, container.ID, true)
2968 return fmt.Errorf("attachment %q not ready: %w", serviceName, err)
2969 }
2970 // Track attachment
2971 s.mu.Lock()
2972 s.attachments[ownerDomain] = append(s.attachments[ownerDomain], container.ID)
2973 s.mu.Unlock()
2974
2975 // Start log collection for attachment
2976 s.startLogCollection(ctx, container.ID, containerName)
2977
2978 log.Info().Str(zerowrap.FieldEntityID, container.ID).Msg("attachment deployed successfully")
2979 return nil
2980}
2981
2982func (s *Service) buildAttachmentContainerConfig(ctx context.Context, ownerDomain, serviceName, serviceImage, containerName, networkName string) (*domain.ContainerConfig, error) {
2983 imageRef, err := s.buildValidatedImageRef(ctx, serviceImage)
2984 if err != nil {
2985 return nil, err
2986 }
2987 actualImageRef, err := s.ensureImage(ctx, imageRef)
2988 if err != nil {
2989 return nil, err
2990 }
2991
2992 exposedPorts := s.attachmentExposedPorts(ctx, actualImageRef)
2993 volumes, err := s.attachmentVolumes(ctx, containerName, actualImageRef)
2994 if err != nil {
2995 return nil, err
2996 }
2997 envVars, err := s.attachmentEnv(ctx, containerName, actualImageRef)
2998 if err != nil {
2999 return nil, err
3000 }
3001
3002 s.mu.RLock()
3003 cfg := s.config
3004 s.mu.RUnlock()
3005
3006 config := &domain.ContainerConfig{
3007 Image: actualImageRef,
3008 Name: containerName,
3009 Hostname: serviceName,
3010 Aliases: []string{serviceName},
3011 Ports: exposedPorts,
3012 Env: envVars,
3013 Volumes: volumes,
3014 NetworkMode: networkName,
3015 Labels: map[string]string{
3016 domain.LabelManaged: "true",
3017 domain.LabelAttachment: "true",
3018 domain.LabelAttachedTo: ownerDomain,
3019 domain.LabelEnvHash: hashEnvironment(envVars),
3020 domain.LabelImage: serviceImage,
3021 },
3022 MemoryLimit: cfg.DefaultMemoryLimit,
3023 NanoCPUs: cfg.DefaultNanoCPUs,
3024 PidsLimit: cfg.DefaultPidsLimit,
3025 RestartPolicy: domain.RestartPolicyAlways,
3026 }
3027 applySecurityProfile(config, cfg)
3028 return config, nil
3029}
3030
3031func (s *Service) attachmentExposedPorts(ctx context.Context, imageRef string) []int {
3032 log := zerowrap.FromCtx(ctx)
3033 exposedPorts, err := s.runtime.GetImageExposedPorts(ctx, imageRef)
3034 if err != nil {
3035 log.WrapErr(err, "failed to get exposed ports for attachment, using defaults")
3036 return []int{}
3037 }
3038 return exposedPorts
3039}
3040
3041func (s *Service) attachmentVolumes(ctx context.Context, containerName, imageRef string) (map[string]string, error) {
3042 volumes, err := s.setupVolumes(ctx, containerName, imageRef)
3043 if err != nil {
3044 return nil, fmt.Errorf("failed to setup volumes for attachment %s with image %s: %w", containerName, imageRef, err)
3045 }
3046 return volumes, nil
3047}
3048
3049func (s *Service) attachmentEnv(ctx context.Context, containerName, imageRef string) ([]string, error) {
3050 envVars, err := s.loadEnvironment(ctx, nil, containerName, imageRef)
3051 if err != nil {
3052 return nil, fmt.Errorf("failed to load environment for attachment %s with image %s: %w", containerName, imageRef, err)
3053 }
3054 return envVars, nil
3055}
3056
3057func (s *Service) attachmentEnvDrifted(ctx context.Context, existing *domain.Container, containerName, serviceImage string) (bool, error) {
3058 imageRef, err := s.buildValidatedImageRef(ctx, serviceImage)
3059 if err != nil {
3060 return false, err
3061 }
3062 currentEnv, err := s.loadEnvironment(ctx, nil, containerName, imageRef)
3063 if err != nil {
3064 return false, err
3065 }
3066
3067 currentHash := hashEnvironment(currentEnv)
3068 existingHash, ok := existing.Labels[domain.LabelEnvHash]
3069 if ok && existingHash == currentHash {
3070 return false, nil
3071 }
3072
3073 return true, nil
3074}
3075
3076func (s *Service) handleRunningAttachment(ctx context.Context, existing *domain.Container, containerName, serviceImage string) (bool, error) {
3077 if existing.Status != string(domain.ContainerStatusRunning) {
3078 return false, nil
3079 }
3080
3081 log := zerowrap.FromCtx(ctx)
3082
3083 // Check image drift
3084 existingImage := existing.Labels[domain.LabelImage]
3085 imageDrifted := existingImage != "" && existingImage != serviceImage
3086
3087 // Check env drift
3088 envDrifted, err := s.attachmentEnvDrifted(ctx, existing, containerName, serviceImage)
3089 if err != nil {
3090 return false, log.WrapErr(err, "failed to check attachment env drift")
3091 }
3092
3093 if !envDrifted && !imageDrifted {
3094 log.Debug().Str("container_name", containerName).Msg("attachment already running with current env and image, skipping")
3095 return true, nil
3096 }
3097
3098 if imageDrifted {
3099 log.Info().Str("container_name", containerName).Str("existing_image", existingImage).Str("desired_image", serviceImage).Msg("attachment image changed, recreating")
3100 }
3101 if envDrifted {
3102 log.Info().Str("container_name", containerName).Msg("attachment env changed, recreating")
3103 }
3104
3105 if err := s.runtime.StopContainer(ctx, existing.ID); err != nil {
3106 return false, log.WrapErr(err, "failed to stop attachment for drift update")
3107 }
3108 if err := s.runtime.RemoveContainer(ctx, existing.ID, true); err != nil {
3109 return false, log.WrapErr(err, "failed to remove attachment for drift update")
3110 }
3111
3112 return false, nil
3113}
3114
3115func (s *Service) removeStoppedAttachment(ctx context.Context, existing *domain.Container, containerName string) error {
3116 if existing == nil {
3117 return nil
3118 }
3119
3120 log := zerowrap.FromCtx(ctx)
3121 log.Info().Str("container_name", containerName).Msg("removing stopped attachment container")
3122 if err := s.runtime.RemoveContainer(ctx, existing.ID, true); err != nil {
3123 return log.WrapErr(err, "failed to remove existing attachment container")
3124 }
3125
3126 return nil
3127}
3128
3129func (s *Service) resolveExistingAttachment(ctx context.Context, ownerDomain, serviceName string) (*domain.Container, error) {
3130 log := zerowrap.FromCtx(ctx)
3131 containerName := fmt.Sprintf("gordon-%s-%s", sanitizeName(ownerDomain), serviceName)
3132 existingContainer := s.findContainerByName(ctx, containerName)
3133 if existingContainer != nil {
3134 if err := validateAttachmentOwnership(existingContainer, ownerDomain, containerName); err != nil {
3135 return nil, err
3136 }
3137 return existingContainer, nil
3138 }
3139
3140 containerNameLegacy := fmt.Sprintf("gordon-%s-%s", sanitizeNameLegacy(ownerDomain), serviceName)
3141 existingContainer = s.findContainerByName(ctx, containerNameLegacy)
3142 if existingContainer == nil {
3143 return nil, nil
3144 }
3145
3146 if err := validateAttachmentOwnership(existingContainer, ownerDomain, containerNameLegacy); err != nil {
3147 return nil, err
3148 }
3149 log.Info().Str("container_name", containerNameLegacy).Msg("found attachment with legacy naming, will be replaced")
3150 if err := s.runtime.StopContainer(ctx, existingContainer.ID); err != nil {
3151 return nil, log.WrapErr(err, "failed to stop legacy attachment container")
3152 }
3153 if err := s.runtime.RemoveContainer(ctx, existingContainer.ID, true); err != nil {
3154 return nil, log.WrapErr(err, "failed to remove legacy attachment container")
3155 }
3156
3157 return nil, nil
3158}
3159
3160func validateAttachmentOwnership(container *domain.Container, ownerDomain, containerName string) error {
3161 if container.Labels[domain.LabelManaged] != "true" ||
3162 container.Labels[domain.LabelAttachment] != "true" ||
3163 container.Labels[domain.LabelAttachedTo] != ownerDomain {
3164 return fmt.Errorf("%w: attachment name %q is owned by another workload", domain.ErrAttachmentOwnershipMismatch, containerName)
3165 }
3166 return nil
3167}
3168
3169// Utility functions
3170
3171func managedContainerName(domainName string) string {
3172 return "gordon-" + domainName
3173}
3174
3175func normalizeImageRef(image string) string {
3176 if isDigestRef(image) {
3177 return image
3178 }
3179
3180 // Extract repository from image reference, handling port numbers correctly.
3181 // Examples:
3182 // nginx:latest -> docker.io/library/nginx
3183 // user/repo:tag -> docker.io/user/repo
3184 // localhost:5000/image:tag -> localhost:5000/image
3185 // registry.example.com/image:tag -> registry.example.com/image
3186 //
3187 // Note: This function assumes valid Docker image references. Edge cases like
3188 // "registry.com:5000" (registry with port but no image name) are not valid
3189 // image references per Docker's naming conventions, which require at least
3190 // one path component after the registry (e.g., "registry.com:5000/image").
3191
3192 // Find the tag separator: the last colon that isn't part of a port number.
3193 // A colon is part of a port if there's no slash after it until the next colon.
3194 repo := image
3195 lastColon := strings.LastIndex(image, ":")
3196 if lastColon != -1 {
3197 afterColon := image[lastColon+1:]
3198 // If there's no slash after the colon, it's the tag separator
3199 if !strings.Contains(afterColon, "/") {
3200 repo = image[:lastColon]
3201 }
3202 }
3203
3204 if !strings.Contains(repo, "/") {
3205 return "docker.io/library/" + repo
3206 }
3207
3208 if strings.Count(repo, "/") == 1 && !strings.Contains(strings.Split(repo, "/")[0], ".") && !strings.Contains(strings.Split(repo, "/")[0], ":") {
3209 return "docker.io/" + repo
3210 }
3211
3212 return repo
3213}
3214
3215func generateVolumeName(prefix, domainName, volumePath string) string {
3216 return fmt.Sprintf("%s-%s-%s",
3217 prefix,
3218 domain.StableResourceName(domainName),
3219 domain.StableResourceName(strings.Trim(volumePath, "/")))
3220}
3221
3222func legacyVolumeName(prefix, domainName, volumePath string) string {
3223 return fmt.Sprintf("%s-%s-%s",
3224 prefix,
3225 strings.ReplaceAll(domainName, ".", "-"),
3226 strings.ReplaceAll(strings.Trim(volumePath, "/"), "/", "-"))
3227}
3228
3229func (s *Service) legacyVolumeOwnedByDomain(ctx context.Context, volumeName, domainName string) (bool, error) {
3230 containers, err := s.runtime.ListContainers(ctx, true)
3231 if err != nil {
3232 return false, fmt.Errorf("list containers for legacy volume ownership: %w", err)
3233 }
3234 for _, container := range containers {
3235 if container.Labels[domain.LabelManaged] != "true" ||
3236 (container.Labels[domain.LabelRoute] != domainName && container.Labels[domain.LabelDomain] != domainName) {
3237 continue
3238 }
3239 for _, mount := range container.VolumeMounts {
3240 if mount.Name == volumeName || mount.Source == volumeName {
3241 return true, nil
3242 }
3243 }
3244 }
3245 return false, nil
3246}
3247
3248func mergeEnvironmentVariables(dockerfileEnv, userEnv []string) []string {
3249 envMap := make(map[string]string)
3250
3251 for _, env := range dockerfileEnv {
3252 if k, v, ok := strings.Cut(env, "="); ok {
3253 envMap[k] = v
3254 }
3255 }
3256
3257 for _, env := range userEnv {
3258 if k, v, ok := strings.Cut(env, "="); ok {
3259 envMap[k] = v
3260 }
3261 }
3262
3263 keys := make([]string, 0, len(envMap))
3264 for k := range envMap {
3265 keys = append(keys, k)
3266 }
3267 slices.Sort(keys)
3268
3269 result := make([]string, 0, len(envMap))
3270 for _, k := range keys {
3271 result = append(result, k+"="+envMap[k])
3272 }
3273
3274 return result
3275}
3276
3277// extractServiceName gets service name from image reference.
3278// "my-postgres:latest" → "postgres", "redis:7" → "redis"
3279func extractServiceName(image string) string {
3280 // Remove tag
3281 parts := strings.Split(image, ":")
3282 name := parts[0]
3283
3284 // Remove registry prefix if present
3285 if strings.Contains(name, "/") {
3286 nameParts := strings.Split(name, "/")
3287 name = nameParts[len(nameParts)-1]
3288 }
3289
3290 // Remove common prefixes like "my-"
3291 name = strings.TrimPrefix(name, "my-")
3292
3293 return name
3294}
3295
3296// sanitizeName is a convenience wrapper around domain.SanitizeDomainForContainer.
3297func sanitizeName(d string) string {
3298 return domain.SanitizeDomainForContainer(d)
3299}
3300
3301// sanitizeNameLegacy is a convenience wrapper around domain.SanitizeDomainForContainerLegacy.
3302func sanitizeNameLegacy(d string) string {
3303 return domain.SanitizeDomainForContainerLegacy(d)
3304}
3305
3306func isContainerNotFoundError(err error) bool {
3307 if err == nil {
3308 return false
3309 }
3310 if errors.Is(err, domain.ErrContainerNotFound) {
3311 return true
3312 }
3313 msg := strings.ToLower(err.Error())
3314 return strings.Contains(msg, "no such container") ||
3315 strings.Contains(msg, "no container with name or id") ||
3316 strings.Contains(msg, "container not found")
3317}
3318
3319func isConnectionRefusedError(err error) bool {
3320 if err == nil {
3321 return false
3322 }
3323 return strings.Contains(strings.ToLower(err.Error()), "connection refused")
3324}
3325
3326// findContainerByName finds a container by its name.
3327func (s *Service) findContainerByName(ctx context.Context, name string) *domain.Container {
3328 containers, err := s.runtime.ListContainers(ctx, true)
3329 if err != nil {
3330 return nil
3331 }
3332
3333 for _, c := range containers {
3334 if c.Name == name {
3335 return c
3336 }
3337 }
3338 return nil
3339}
3340
3341func (s *Service) waitForReady(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig) error {
3342 if err := s.pollContainerRunning(ctx, containerID); err != nil {
3343 return err
3344 }
3345
3346 s.mu.RLock()
3347 cfg := s.config
3348 s.mu.RUnlock()
3349
3350 readinessMode := cfg.ReadinessMode
3351 if readinessMode == "" {
3352 readinessMode = "delay"
3353 }
3354
3355 switch readinessMode {
3356 case "delay":
3357 return s.waitForReadyByDelay(ctx, containerID)
3358 case "docker-health":
3359 _, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(ctx, containerID)
3360 if err != nil {
3361 return err
3362 }
3363 if !hasHealthcheck {
3364 return errors.New("no healthcheck detected")
3365 }
3366 return s.waitForHealthy(ctx, containerID, cfg.HealthTimeout)
3367 default: // auto
3368 return s.readinessCascade(ctx, containerID, containerConfig, cfg)
3369 }
3370}
3371
3372// readinessCascade auto-detects the strongest available readiness signal:
3373// 1. Docker healthcheck (if present) → wait for healthy status
3374// 2. HTTP probe (if gordon.health label set) → GET until 2xx/3xx
3375// 3. Default HTTP probe (GET / on exposed port) → wait for 2xx/3xx
3376// 4. TCP probe (if port info available) → connect until accepted
3377// 5. Delay fallback (last resort) → waitForReadyByDelay
3378func (s *Service) readinessCascade(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) error {
3379 log := zerowrap.FromCtx(ctx)
3380
3381 // 1. Docker healthcheck
3382 _, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(ctx, containerID)
3383 if err != nil {
3384 return err
3385 }
3386 if hasHealthcheck {
3387 log.Info().Msg("readiness cascade: using Docker healthcheck")
3388 return s.waitForHealthy(ctx, containerID, cfg.HealthTimeout)
3389 }
3390
3391 // 2. HTTP probe via gordon.health label
3392 if probed, probeErr := s.tryHTTPProbe(ctx, containerID, containerConfig, cfg); probed {
3393 return probeErr
3394 }
3395
3396 // 3. Default HTTP probe on root path (if port info available)
3397 if probed, probeErr := s.tryDefaultHTTPProbe(ctx, containerID, containerConfig, cfg); probed {
3398 return probeErr
3399 }
3400
3401 // 4. TCP probe (if port info available)
3402 if probed, probeErr := s.tryTCPProbe(ctx, containerID, containerConfig, cfg); probed {
3403 return probeErr
3404 }
3405
3406 // 5. Delay fallback
3407 log.Info().Msg("readiness cascade: using delay fallback")
3408 return s.waitForReadyByDelay(ctx, containerID)
3409}
3410
3411// waitForAttachmentReady runs a reduced readiness cascade for attachments:
3412// Docker healthcheck → TCP probe → delay fallback.
3413// HTTP probes are skipped — attachments are typically databases/caches.
3414func (s *Service) waitForAttachmentReady(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig) error {
3415 if err := s.pollContainerRunning(ctx, containerID); err != nil {
3416 return err
3417 }
3418
3419 log := zerowrap.FromCtx(ctx)
3420
3421 s.mu.RLock()
3422 cfg := s.config
3423 s.mu.RUnlock()
3424
3425 // 1. Docker healthcheck (if present)
3426 _, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(ctx, containerID)
3427 if err != nil {
3428 return err
3429 }
3430 if hasHealthcheck {
3431 log.Info().Msg("attachment readiness: using Docker healthcheck")
3432 timeout := cfg.HealthTimeout
3433 if timeout == 0 {
3434 timeout = 90 * time.Second
3435 }
3436 return s.waitForHealthy(ctx, containerID, timeout)
3437 }
3438
3439 // 2. TCP probe (if port info available)
3440 if containerConfig != nil && len(containerConfig.Ports) > 0 {
3441 ip, port, probeErr := s.resolveProbeEndpoint(ctx, containerID, containerConfig)
3442 if probeErr == nil && ip != "" && port > 0 {
3443 addr := fmt.Sprintf("%s:%d", ip, port)
3444 timeout := cfg.AttachmentReadinessTimeout
3445 if timeout == 0 {
3446 timeout = 30 * time.Second
3447 }
3448 log.Info().Str("addr", addr).Dur("timeout", timeout).Msg("attachment readiness: using TCP probe")
3449 return tcpProbe(ctx, addr, timeout)
3450 }
3451 log.Debug().Err(probeErr).Msg("attachment readiness: TCP probe skipped, could not resolve endpoint")
3452 }
3453
3454 // 3. Delay fallback
3455 log.Info().Msg("attachment readiness: using delay fallback")
3456 return s.waitForReadyByDelay(ctx, containerID)
3457}
3458
3459// tryHTTPProbe attempts an HTTP probe if the gordon.health label is set.
3460// Returns (true, err) if the probe was attempted, (false, nil) if skipped.
3461func (s *Service) tryHTTPProbe(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) (bool, error) {
3462 log := zerowrap.FromCtx(ctx)
3463 if containerConfig == nil {
3464 return false, nil
3465 }
3466 healthPath, ok := containerConfig.Labels[domain.LabelHealth]
3467 if !ok || healthPath == "" {
3468 return false, nil
3469 }
3470 ip, port, probeErr := s.resolveProbeEndpoint(ctx, containerID, containerConfig)
3471 if probeErr != nil || ip == "" || port <= 0 {
3472 log.Debug().Err(probeErr).Msg("readiness cascade: HTTP probe skipped, could not resolve container endpoint")
3473 return false, nil
3474 }
3475 url := fmt.Sprintf("http://%s:%d%s", ip, port, healthPath)
3476 timeout := cfg.HTTPProbeTimeout
3477 if timeout == 0 {
3478 timeout = 60 * time.Second
3479 }
3480 log.Info().Str("url", url).Dur("timeout", timeout).Msg("readiness cascade: using HTTP probe")
3481 return true, httpProbe(ctx, url, timeout)
3482}
3483
3484// tryDefaultHTTPProbe attempts an HTTP GET on "/" using the container's
3485// exposed port. This is the default readiness check for web containers —
3486// it verifies the app is actually serving HTTP, not just accepting TCP.
3487// Returns (true, err) if the probe was attempted, (false, nil) if skipped.
3488func (s *Service) tryDefaultHTTPProbe(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) (bool, error) {
3489 log := zerowrap.FromCtx(ctx)
3490 if containerConfig == nil || len(containerConfig.Ports) == 0 {
3491 return false, nil
3492 }
3493 ip, port, probeErr := s.resolveProbeEndpoint(ctx, containerID, containerConfig)
3494 if probeErr != nil || ip == "" || port <= 0 {
3495 log.Debug().Err(probeErr).Msg("readiness cascade: default HTTP probe skipped, could not resolve container endpoint")
3496 return false, nil
3497 }
3498 url := fmt.Sprintf("http://%s:%d/", ip, port)
3499 timeout := cfg.HTTPProbeTimeout
3500 if timeout == 0 {
3501 timeout = 60 * time.Second
3502 }
3503 log.Info().Str("url", url).Dur("timeout", timeout).Msg("readiness cascade: using default HTTP alive probe")
3504 return true, httpAliveProbe(ctx, url, timeout)
3505}
3506
3507// tryTCPProbe attempts a TCP probe if port info is available.
3508// Returns (true, err) if the probe was attempted, (false, nil) if skipped.
3509func (s *Service) tryTCPProbe(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) (bool, error) {
3510 log := zerowrap.FromCtx(ctx)
3511 if containerConfig == nil || len(containerConfig.Ports) == 0 {
3512 return false, nil
3513 }
3514 ip, port, probeErr := s.resolveProbeEndpoint(ctx, containerID, containerConfig)
3515 if probeErr != nil || ip == "" || port <= 0 {
3516 log.Debug().Err(probeErr).Msg("readiness cascade: TCP probe skipped, could not resolve container endpoint")
3517 return false, nil
3518 }
3519 addr := fmt.Sprintf("%s:%d", ip, port)
3520 timeout := cfg.TCPProbeTimeout
3521 if timeout == 0 {
3522 timeout = 30 * time.Second
3523 }
3524 log.Info().Str("addr", addr).Dur("timeout", timeout).Msg("readiness cascade: using TCP probe")
3525 return true, tcpProbe(ctx, addr, timeout)
3526}
3527
3528// resolveContainerEndpoint returns a host-reachable address for probing.
3529// In rootless podman/Docker setups, container internal IPs are not routable
3530// from the host. We use the host port binding (127.0.0.1:<mapped_port>)
3531// which is always reachable. Falls back to the container's internal IP
3532// only if no host port mapping exists (e.g. host-network mode).
3533func (s *Service) resolveProbeEndpoint(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig) (string, int, error) {
3534 log := zerowrap.FromCtx(ctx)
3535
3536 internalPort := 0
3537 if containerConfig != nil {
3538 if labels := containerConfig.Labels; labels != nil {
3539 for _, key := range []string{domain.LabelProxyPort, domain.LabelPort} {
3540 if portStr, ok := labels[key]; ok && portStr != "" {
3541 port, err := strconv.Atoi(portStr)
3542 if err == nil && port > 0 && port <= 65535 {
3543 internalPort = port
3544 break
3545 }
3546 log.Warn().Str("label", key).Str("port_value", portStr).Msg("invalid probe port label value")
3547 }
3548 }
3549 }
3550 }
3551
3552 if internalPort == 0 {
3553 _, resolvedPort, err := s.runtime.GetContainerNetworkInfo(ctx, containerID)
3554 if err != nil {
3555 return "", 0, err
3556 }
3557 internalPort = resolvedPort
3558 }
3559
3560 hostPort, hostErr := s.runtime.GetContainerPort(ctx, containerID, internalPort)
3561 if hostErr == nil && hostPort > 0 {
3562 log.Debug().
3563 Int("internal_port", internalPort).
3564 Int("host_port", hostPort).
3565 Msg("resolved container endpoint via host port binding")
3566 return "127.0.0.1", hostPort, nil
3567 }
3568
3569 ip, _, fallbackErr := s.runtime.GetContainerNetworkInfo(ctx, containerID)
3570 if fallbackErr != nil {
3571 return "", 0, fallbackErr
3572 }
3573 log.Debug().
3574 Str("ip", ip).
3575 Int("port", internalPort).
3576 Msg("resolved container endpoint via internal IP (no host port binding)")
3577 return ip, internalPort, nil
3578}
3579
3580// waitForReadyByDelay waits using the legacy running+delay strategy.
3581func (s *Service) waitForReadyByDelay(ctx context.Context, containerID string) error {
3582 log := zerowrap.FromCtx(ctx)
3583
3584 s.mu.RLock()
3585 cfg := s.config
3586 s.mu.RUnlock()
3587
3588 delay := cfg.ReadinessDelay
3589 if delay == 0 {
3590 delay = 5 * time.Second
3591 }
3592
3593 log.Debug().Dur("delay", delay).Msg("waiting for container readiness")
3594 select {
3595 case <-time.After(delay):
3596 case <-ctx.Done():
3597 return ctx.Err()
3598 }
3599
3600 // Verify still running after delay
3601 running, err := s.runtime.IsContainerRunning(ctx, containerID)
3602 if err != nil {
3603 return err
3604 }
3605 if running {
3606 return nil
3607 }
3608
3609 return s.waitForRecovery(ctx, containerID)
3610}
3611
3612func (s *Service) waitForHealthy(ctx context.Context, containerID string, timeout time.Duration) error {
3613 if timeout == 0 {
3614 timeout = 90 * time.Second
3615 }
3616 waitCtx, cancel := context.WithTimeout(ctx, timeout)
3617 defer cancel()
3618
3619 var lastStatus string
3620 for {
3621 status, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(waitCtx, containerID)
3622 if err != nil {
3623 return err
3624 }
3625 if !hasHealthcheck {
3626 return errors.New("no healthcheck detected")
3627 }
3628 // Treat empty health status as transitional startup. Some runtimes can
3629 // temporarily report an empty status before first probe results.
3630 if status == "" {
3631 status = "starting"
3632 }
3633 if status == "healthy" {
3634 return nil
3635 }
3636 if status == "unhealthy" {
3637 return errors.New("container reported unhealthy")
3638 }
3639 lastStatus = status
3640
3641 select {
3642 case <-time.After(time.Second):
3643 case <-waitCtx.Done():
3644 return fmt.Errorf("container healthcheck timeout after %s (last status: %s)", timeout, lastStatus)
3645 }
3646 }
3647}
3648
3649// pollContainerRunning polls until the container is running (max 30 seconds).
3650func (s *Service) pollContainerRunning(ctx context.Context, containerID string) error {
3651 for i := 0; i < 30; i++ {
3652 running, err := s.runtime.IsContainerRunning(ctx, containerID)
3653 if err != nil {
3654 return err
3655 }
3656 if running {
3657 return nil
3658 }
3659 // Check if the container has already exited (crash) rather than still starting up.
3660 if info, inspectErr := s.runtime.InspectContainer(ctx, containerID); inspectErr == nil && info != nil && info.Status == string(domain.ContainerStatusExited) {
3661 return fmt.Errorf("container exited immediately with code %d: %w", info.ExitCode, domain.ErrContainerExited)
3662 }
3663 if i == 29 {
3664 return fmt.Errorf("container did not start within 30 seconds")
3665 }
3666 select {
3667 case <-time.After(time.Second):
3668 case <-ctx.Done():
3669 return ctx.Err()
3670 }
3671 }
3672 return fmt.Errorf("container did not start within 30 seconds")
3673}
3674
3675// waitForRecovery waits for a container to recover within the recovery window.
3676func (s *Service) waitForRecovery(ctx context.Context, containerID string) error {
3677 log := zerowrap.FromCtx(ctx)
3678 log.Warn().
3679 Dur("recovery_window", readinessRecoveryWindow).
3680 Msg("container not running after readiness delay, waiting for recovery")
3681
3682 deadline := time.Now().Add(readinessRecoveryWindow)
3683 for time.Now().Before(deadline) {
3684 select {
3685 case <-time.After(time.Second):
3686 case <-ctx.Done():
3687 return ctx.Err()
3688 }
3689
3690 running, err := s.runtime.IsContainerRunning(ctx, containerID)
3691 if err != nil {
3692 return err
3693 }
3694 if running {
3695 log.Info().Msg("container recovered during readiness recovery window")
3696 return nil
3697 }
3698 }
3699
3700 return fmt.Errorf("container not running after readiness delay and recovery window")
3701}
3702
3703// publishContainerDeployed publishes a container.deployed event.
3704func (s *Service) publishContainerDeployed(ctx context.Context, domainName, containerID string) {
3705 payload := &domain.ContainerEventPayload{
3706 ContainerID: containerID,
3707 Domain: domainName,
3708 Action: "deployed",
3709 }
3710
3711 if err := s.eventBus.Publish(domain.EventContainerDeployed, payload); err != nil {
3712 log := zerowrap.FromCtx(ctx)
3713 log.Warn().Err(err).Msg("failed to publish container deployed event")
3714 }
3715}
3716
3717// rewriteToRegistryDomain rewrites an image reference to use the configured registry domain.
3718// e.g., "myapp:latest" -> "registry.example.com/myapp:latest"
3719func rewriteToRegistryDomain(imageRef, registryDomain string) string {
3720 if registryDomain == "" {
3721 return imageRef
3722 }
3723
3724 prefix := registryDomain + "/"
3725 if strings.HasPrefix(imageRef, prefix) {
3726 return imageRef
3727 }
3728
3729 return prefix + imageRef
3730}
3731
3732// rewriteToLocalRegistry rewrites Gordon-managed image references to use the
3733// local registry address. Current and legacy Gordon registry domains are
3734// stripped before prefixing localhost:<registryPort>/. Bare refs are prefixed.
3735// External registries are preserved unchanged.
3736func rewriteToLocalRegistry(imageRef, registryDomain string, legacyRegistryDomains []string, registryPort int) string {
3737 if imageRef == "" {
3738 return imageRef
3739 }
3740
3741 localRegistry := fmt.Sprintf("localhost:%d", registryPort)
3742 localPrefix := localRegistry + "/"
3743 if strings.HasPrefix(imageRef, localPrefix) {
3744 return imageRef
3745 }
3746
3747 if domain.IsGordonRegistryImageRef(imageRef, registryDomain, legacyRegistryDomains) {
3748 return localPrefix + domain.StripKnownGordonRegistry(imageRef, registryDomain, legacyRegistryDomains)
3749 }
3750 if hasExplicitRegistry(imageRef) {
3751 return imageRef
3752 }
3753
3754 return localPrefix + imageRef
3755}