1// Package container implements the container management use case.
2package container
3
4import (
5 "bufio"
6 "context"
7 "encoding/binary"
8 "encoding/hex"
9 "errors"
10 "fmt"
11 "io"
12 "maps"
13 "net"
14 "slices"
15 "strconv"
16 "strings"
17 "sync"
18 "time"
19
20 "github.com/bnema/zerowrap"
21 "go.opentelemetry.io/otel"
22 "go.opentelemetry.io/otel/attribute"
23 "go.opentelemetry.io/otel/codes"
24 "go.opentelemetry.io/otel/metric"
25 "go.opentelemetry.io/otel/trace"
26
27 "github.com/bnema/gordon/internal/adapters/out/telemetry"
28 "github.com/bnema/gordon/internal/boundaries/out"
29 "github.com/bnema/gordon/internal/domain"
30)
31
32// Config holds configuration needed by the container service.
33type Config struct {
34 RegistryAuthEnabled bool
35 RegistryDomain string
36 LegacyRegistryDomains []string
37 RegistryPort int
38 ServiceTokenUsername string
39 ServiceToken string
40 InternalRegistryUsername string
41 InternalRegistryPassword string
42 PullPolicy string
43 VolumeAutoCreate bool
44 VolumePrefix string
45 VolumePreserve bool
46 NetworkIsolation bool
47 NetworkPrefix string
48 NetworkGroups map[string][]string
49 NetworkInternal bool
50 Attachments map[string][]string
51 AllowedRegistries []string
52 RequireImageDigest bool
53 SecurityProfile string
54 ReadinessDelay time.Duration // Delay after container starts before considering it ready
55 ReadinessMode string // Readiness strategy: auto, docker-health, delay
56 HealthTimeout time.Duration // Max wait for health-based readiness
57 DrainDelay time.Duration // Grace period after cache invalidation before stopping old container
58 DrainDelayConfigured bool // True when deploy.drain_delay was explicitly configured
59 DrainMode string // Drain strategy: auto, inflight, delay
60 DrainTimeout time.Duration // Max wait for in-flight requests to drain
61 StabilizationDelay time.Duration // Post-switch monitoring window (default 2s)
62 TCPProbeTimeout time.Duration // TCP probe timeout (default 30s)
63 HTTPProbeTimeout time.Duration // HTTP probe timeout (default 60s)
64 AttachmentReadinessTimeout time.Duration // Max wait for attachment readiness (default 30s)
65 DefaultMemoryLimit int64 // Default memory limit in bytes for containers (0 = no limit)
66 DefaultNanoCPUs int64 // Default CPU quota in nanoseconds for containers (0 = no limit)
67 DefaultPidsLimit int64 // Default max PIDs for containers (0 = no limit)
68}
69
70var tracer = otel.Tracer("gordon.container")
71
72const (
73 PullPolicyAlways = "always"
74 PullPolicyIfNotPresent = "if-not-present"
75 PullPolicyIfTagChanged = "if-tag-changed"
76
77 // readinessRecoveryWindow is an additional grace window used when a container
78 // is briefly not running at the end of readiness delay. This avoids false
79 // negatives during short startup flaps.
80 readinessRecoveryWindow = 30 * time.Second
81 internalPullMaxAttempts = 3
82 deployFailureLogLimit = 20
83)
84
85// Service implements the ContainerService interface.
86type Service struct {
87 runtime out.ContainerRuntime
88 envLoader out.EnvLoader
89 eventBus out.EventPublisher
90 logWriter out.ContainerLogWriter
91 cacheInvalidator out.ProxyCacheInvalidator
92 drainWaiter out.ProxyDrainWaiter
93 config Config
94 configProvider AttachmentConfigProvider // live config reads for attachments/networks (may be nil)
95 metrics *telemetry.Metrics
96 containers map[string]*domain.Container
97 attachments map[string][]string // ownerDomain → []containerIDs
98 managedCount int64 // tracks UpDownCounter value for delta computation
99 mu sync.RWMutex
100 deployMu sync.Map // per-domain deploy locks (domain → *domainDeployLock)
101 cleanupWg sync.WaitGroup // tracks background old-container cleanup goroutines
102 monitor *Monitor
103}
104
105// domainDeployLock is a context-aware mutex using a buffered channel.
106// It allows Lock() to be interrupted by context cancellation.
107type domainDeployLock struct {
108 ch chan struct{}
109}
110
111// newDomainDeployLock creates a new lock that is initially unlocked.
112func newDomainDeployLock() *domainDeployLock {
113 l := &domainDeployLock{ch: make(chan struct{}, 1)}
114 l.ch <- struct{}{}
115 return l
116}
117
118// Lock acquires the lock or returns an error if the context is cancelled.
119func (l *domainDeployLock) Lock(ctx context.Context) error {
120 // Fast path: honor already-canceled contexts before blocking.
121 if err := ctx.Err(); err != nil {
122 return err
123 }
124 select {
125 case <-ctx.Done():
126 return ctx.Err()
127 case <-l.ch:
128 // Re-check context after acquisition: if canceled while we were
129 // in the select, release the token so the lock isn't consumed.
130 if err := ctx.Err(); err != nil {
131 l.ch <- struct{}{}
132 return err
133 }
134 return nil
135 }
136}
137
138// Unlock releases the lock.
139func (l *domainDeployLock) Unlock() {
140 select {
141 case l.ch <- struct{}{}:
142 default:
143 // This should never happen if Lock/Unlock are used correctly.
144 // Panic to detect double-unlocks or other misuse early.
145 panic("domainDeployLock: double unlock detected")
146 }
147}
148
149// NewService creates a new container service.
150func NewService(
151 runtime out.ContainerRuntime,
152 envLoader out.EnvLoader,
153 eventBus out.EventPublisher,
154 logWriter out.ContainerLogWriter,
155 config Config,
156 configProvider AttachmentConfigProvider,
157) *Service {
158 return &Service{
159 runtime: runtime,
160 envLoader: envLoader,
161 eventBus: eventBus,
162 logWriter: logWriter,
163 config: config,
164 configProvider: configProvider,
165 containers: make(map[string]*domain.Container),
166 attachments: make(map[string][]string),
167 }
168}
169
170// SetMetrics sets the telemetry metrics for the container service.
171func (s *Service) SetMetrics(m *telemetry.Metrics) {
172 s.metrics = m
173}
174
175// SetProxyCacheInvalidator sets the proxy cache invalidator for synchronous
176// cache invalidation during zero-downtime deployments. This must be called
177// after construction because the proxy service is created after the container
178// service during application initialization.
179func (s *Service) SetProxyCacheInvalidator(inv out.ProxyCacheInvalidator) {
180 s.mu.Lock()
181 s.cacheInvalidator = inv
182 s.mu.Unlock()
183}
184
185// SetProxyDrainWaiter sets the proxy in-flight drain waiter used during
186// zero-downtime replacement before stopping the old container.
187func (s *Service) SetProxyDrainWaiter(waiter out.ProxyDrainWaiter) {
188 s.mu.Lock()
189 s.drainWaiter = waiter
190 s.mu.Unlock()
191}
192
193// domainDeployMu returns a per-domain lock for serializing deploy operations.
194func (s *Service) domainDeployMu(domain string) *domainDeployLock {
195 v, _ := s.deployMu.LoadOrStore(domain, newDomainDeployLock())
196 // Type assertion is safe: LoadOrStore always stores *domainDeployLock values,
197 // and any concurrent LoadOrStore calls will also store *domainDeployLock.
198 return v.(*domainDeployLock)
199}
200
201// acquireDomainDeployLock acquires the per-domain deploy lock with context cancellation support.
202// Returns an unlock function that must be called to release the lock.
203func (s *Service) acquireDomainDeployLock(ctx context.Context, domain string) (func(), error) {
204 lock := s.domainDeployMu(domain)
205 if err := lock.Lock(ctx); err != nil {
206 return nil, err
207 }
208 return func() { lock.Unlock() }, nil
209}
210
211// buildContainerConfig constructs the container configuration for deployment.
212func (s *Service) deploymentContainerName(containerDomain string, existing *domain.Container) string {
213 canonicalName := fmt.Sprintf("gordon-%s", containerDomain)
214 if existing == nil {
215 return canonicalName
216 }
217
218 // Alternate temporary names if the tracked container was left with a temp suffix.
219 // This prevents name collisions after interrupted zero-downtime deploys.
220 newName := canonicalName + "-new"
221 nextName := canonicalName + "-next"
222 switch existing.Name {
223 case newName:
224 return nextName
225 case nextName:
226 return newName
227 default:
228 return newName
229 }
230}
231
232// containerConfigInput groups the parameters for building a container configuration.
233type containerConfigInput struct {
234 Domain string
235 Image string
236 ImageRef string
237 ExposedPorts []int
238 EnvVars []string
239 EnvHash string
240 Volumes map[string]string
241 NetworkName string
242 ImageLabels map[string]string
243 Existing *domain.Container
244}
245
246func (s *Service) buildContainerConfig(in containerConfigInput) *domain.ContainerConfig {
247 s.mu.RLock()
248 cfg := s.config
249 s.mu.RUnlock()
250
251 containerName := s.deploymentContainerName(in.Domain, in.Existing)
252 labels := map[string]string{
253 domain.LabelDomain: in.Domain,
254 domain.LabelEnvHash: in.EnvHash,
255 domain.LabelImage: in.Image,
256 domain.LabelManaged: "true",
257 domain.LabelRoute: in.Domain,
258 }
259
260 // Propagate proxy/port labels from image so readiness probes can find them
261 for _, key := range []string{domain.LabelProxyPort, domain.LabelPort, domain.LabelHealth} {
262 if v, ok := in.ImageLabels[key]; ok && v != "" {
263 labels[key] = v
264 }
265 }
266
267 // Ensure the label-specified proxy port is included in exposed ports
268 // so Docker creates a host port binding for it.
269 ports := in.ExposedPorts
270 if portStr, ok := labels[domain.LabelProxyPort]; ok {
271 if port, err := strconv.Atoi(portStr); err == nil && port > 0 && port <= 65535 {
272 if !slices.Contains(ports, port) {
273 ports = append(ports, port)
274 }
275 }
276 }
277
278 containerConfig := &domain.ContainerConfig{
279 Image: in.ImageRef,
280 Name: containerName,
281 Ports: ports,
282 Env: in.EnvVars,
283 Volumes: in.Volumes,
284 NetworkMode: in.NetworkName,
285 Hostname: in.Domain,
286 Labels: labels,
287 AutoRemove: false,
288 RestartPolicy: domain.RestartPolicyAlways,
289 MemoryLimit: cfg.DefaultMemoryLimit,
290 NanoCPUs: cfg.DefaultNanoCPUs,
291 PidsLimit: cfg.DefaultPidsLimit,
292 }
293 applySecurityProfile(containerConfig, cfg)
294 return containerConfig
295}
296
297// Deploy creates and starts a container for the given route.
298// Implements zero-downtime deployment: new container starts before old one stops.
299func (s *Service) Deploy(ctx context.Context, route domain.Route) (*domain.Container, error) {
300 ctx, span := tracer.Start(ctx, "container.deploy",
301 trace.WithAttributes(
302 attribute.String("domain", route.Domain),
303 attribute.String("image", route.Image),
304 ))
305 defer span.End()
306
307 deployStart := time.Now()
308
309 // Serialize deploys for the same domain to prevent race conditions
310 // (e.g. multiple image.pushed events + explicit deploy call from CLI).
311 unlock, err := s.acquireDomainDeployLock(ctx, route.Domain)
312 if err != nil {
313 return nil, err
314 }
315 defer unlock()
316
317 // Enrich context with use case fields for all downstream logs
318 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
319 zerowrap.FieldLayer: "usecase",
320 zerowrap.FieldUseCase: "Deploy",
321 "domain": route.Domain,
322 })
323 log := zerowrap.FromCtx(ctx)
324
325 // Record deploy metrics and trace status
326 defer s.recordDeployMetrics(ctx, span, route, deployStart, &err)
327
328 existing, hasExisting := s.resolveExistingContainer(ctx, route.Domain)
329
330 resources, err := s.prepareDeployResources(ctx, route, existing)
331 if err != nil {
332 if deployErr := s.wrapPullDeployFailure(route, existing, err); deployErr != nil {
333 err = deployErr
334 return nil, err
335 }
336 return nil, err
337 }
338
339 // Skip redundant deploy: if the existing container is already running
340 // the exact same image (by Docker image ID), return it immediately.
341 // This prevents the double-deploy caused by the event-based deploy
342 // (triggered by image.pushed) racing with the explicit CLI deploy call.
343 if hasExisting {
344 existingForSkip := existing
345 if existingForSkip.ImageID == "" && existingForSkip.Image != "" && normalizeImageRef(existingForSkip.Image) == normalizeImageRef(route.Image) {
346 existingForSkip = s.containerForRedundantCheck(ctx, existingForSkip)
347 }
348 if existingForSkip.ImageID != "" {
349 if skip, container := s.skipRedundantDeploy(ctx, existingForSkip, resources.actualImageRef, resources.envHash); skip {
350 return container, nil
351 }
352 }
353 }
354
355 newContainer, err := s.createStartedContainer(ctx, route, existing, resources)
356 if err != nil {
357 return nil, err
358 }
359
360 invalidated := s.activateDeployedContainer(ctx, route.Domain, newContainer)
361
362 // Post-switch stabilization: verify new container stays running
363 if hasExisting {
364 stable, stabilizeErr := s.stabilizeNewContainer(ctx, route.Domain, newContainer, existing)
365 if stabilizeErr != nil {
366 // Both old and new containers are dead; assign to named return so
367 // the deferred recordDeployMetrics and span see the failure.
368 err = stabilizeErr
369 return nil, err
370 }
371 if !stable {
372 // Rollback performed — old container is restored
373 return existing, nil
374 }
375 }
376
377 // Finalize old container in the background — the new container is already
378 // serving traffic, so there's no reason to block the deploy response while
379 // waiting for the old container to stop (which can take 20s if the app
380 // doesn't handle SIGTERM).
381 s.cleanupWg.Add(1)
382 go func() {
383 defer s.cleanupWg.Done()
384 s.finalizePreviousContainer(context.WithoutCancel(ctx), route.Domain, existing, hasExisting, invalidated, newContainer.ID)
385 }()
386
387 // Start container log collection (non-blocking, errors don't fail deployment)
388 s.startLogCollection(ctx, newContainer.ID, route.Domain)
389
390 log.Info().
391 Str("image", route.Image).
392 Str(zerowrap.FieldEntityID, newContainer.ID).
393 Ints("ports", newContainer.Ports).
394 Str("network", resources.networkName).
395 Bool("zero_downtime", hasExisting).
396 Msg("container deployed successfully")
397
398 return newContainer, nil
399}
400
401// recordDeployMetrics records span error status and deploy metrics at the end of a Deploy call.
402// It is called via defer and receives a pointer to the named return error so it can observe
403// the final error value after all deferred functions have run.
404func (s *Service) recordDeployMetrics(ctx context.Context, span trace.Span, route domain.Route, start time.Time, errPtr *error) {
405 err := *errPtr
406 if err != nil {
407 span.RecordError(err)
408 span.SetStatus(codes.Error, err.Error())
409 }
410 if s.metrics == nil {
411 return
412 }
413 attrs := metric.WithAttributes(
414 attribute.String("domain", route.Domain),
415 attribute.String("image", route.Image),
416 )
417 s.metrics.DeployTotal.Add(ctx, 1, attrs)
418 s.metrics.DeployDuration.Record(ctx, time.Since(start).Seconds(), attrs)
419 if err != nil {
420 s.metrics.DeployErrors.Add(ctx, 1, attrs)
421 }
422}
423
424// skipRedundantDeploy checks whether the existing container is already running
425// the same image (by Docker image ID) as the one we are about to deploy.
426// When a push triggers both an event-based deploy and an explicit CLI deploy,
427// the second one arrives after the first has already completed; this avoids
428// a full redundant create-start-readiness cycle.
429func (s *Service) skipRedundantDeploy(ctx context.Context, existing *domain.Container, actualImageRef, envHash string) (bool, *domain.Container) {
430 log := zerowrap.FromCtx(ctx)
431
432 newImageID, err := s.runtime.GetImageID(ctx, actualImageRef)
433 if err != nil {
434 log.Debug().Err(err).Msg("cannot resolve image ID for redundancy check, proceeding with deploy")
435 return false, nil
436 }
437
438 if existing.ImageID != newImageID {
439 return false, nil
440 }
441
442 // Verify the container is actually still running before skipping.
443 running, err := s.runtime.IsContainerRunning(ctx, existing.ID)
444 if err != nil || !running {
445 return false, nil
446 }
447
448 if existing.Labels != nil {
449 existingEnvHash, hasEnvHash := existing.Labels[domain.LabelEnvHash]
450 if !hasEnvHash || existingEnvHash != envHash {
451 log.Info().
452 Str("container_id", existing.ID).
453 Bool("has_env_hash", hasEnvHash).
454 Msg("env changed, proceeding with deploy despite same image")
455 return false, nil
456 }
457 } else {
458 return false, nil
459 }
460
461 log.Info().
462 Str("container_id", existing.ID).
463 Str("image_id", newImageID).
464 Msg("skipping redundant deploy: container already running this image")
465
466 return true, existing
467}
468
469func (s *Service) containerForRedundantCheck(ctx context.Context, existing *domain.Container) *domain.Container {
470 if existing == nil || existing.ImageID != "" || existing.ID == "" {
471 return existing
472 }
473
474 log := zerowrap.FromCtx(ctx)
475 inspected, err := s.runtime.InspectContainer(ctx, existing.ID)
476 if err != nil {
477 log.Debug().Err(err).Str("container_id", existing.ID).Msg("cannot inspect existing container for redundancy check")
478 return existing
479 }
480 if inspected == nil || inspected.ImageID == "" {
481 return existing
482 }
483
484 existingCopy := *existing
485 existingCopy.ImageID = inspected.ImageID
486 return &existingCopy
487}
488
489type deployResources struct {
490 networkName string
491 actualImageRef string
492 exposedPorts []int
493 imageLabels map[string]string
494 envVars []string
495 envHash string
496 volumes map[string]string
497}
498
499// resolveExistingContainer returns the currently running container for a domain.
500// It first checks the in-memory map (fast path). If not found, it queries the
501// runtime for a running container with matching name and managed label. This
502// handles cases where Gordon restarted and in-memory state is stale.
503//
504// The slow path checks canonical, -new, and -next names because a previous
505// deploy may have been interrupted (e.g. eventbus timeout, Gordon restart)
506// leaving the active container under a temp name. Canonical is preferred when
507// multiple matches exist.
508func (s *Service) resolveExistingContainer(ctx context.Context, domainName string) (*domain.Container, bool) {
509 // Fast path: check in-memory state
510 s.mu.RLock()
511 container, ok := s.containers[domainName]
512 s.mu.RUnlock()
513 if ok {
514 return container, true
515 }
516
517 // Slow path: query runtime for running containers
518 log := zerowrap.FromCtx(ctx)
519 canonicalName := fmt.Sprintf("gordon-%s", domainName)
520 candidateNames := map[string]bool{
521 canonicalName: true,
522 canonicalName + "-new": true,
523 canonicalName + "-next": true,
524 }
525
526 running, err := s.runtime.ListContainers(ctx, false)
527 if err != nil {
528 log.Warn().Err(err).Msg("failed to list running containers for existing container resolution")
529 return nil, false
530 }
531
532 // Prefer canonical name; fall back to any running managed temp container.
533 var best *domain.Container
534 for _, c := range running {
535 if !candidateNames[c.Name] || c.Labels[domain.LabelManaged] != "true" {
536 continue
537 }
538 if best == nil || c.Name == canonicalName {
539 best = c
540 }
541 }
542
543 if best != nil {
544 log.Info().
545 Str("container_id", best.ID).
546 Str("container_name", best.Name).
547 Msg("resolved existing container from runtime (in-memory state was stale)")
548
549 // Update in-memory state so subsequent lookups are fast
550 s.mu.Lock()
551 if _, alreadyTracked := s.containers[domainName]; !alreadyTracked {
552 s.managedCount++
553 }
554 s.containers[domainName] = best
555 s.mu.Unlock()
556
557 return best, true
558 }
559
560 return nil, false
561}
562
563func (s *Service) prepareDeployResources(ctx context.Context, route domain.Route, existing *domain.Container) (*deployResources, error) {
564 log := zerowrap.FromCtx(ctx)
565
566 existingID := ""
567 if existing != nil {
568 existingID = existing.ID
569 }
570 if err := s.cleanupOrphanedContainers(ctx, route.Domain, existingID); err != nil {
571 log.WrapErr(err, "failed to cleanup orphaned containers")
572 }
573
574 imageRef, err := s.buildValidatedImageRef(ctx, route.Image)
575 if err != nil {
576 return nil, err
577 }
578 actualImageRef, err := s.ensureImage(ctx, imageRef)
579 if err != nil {
580 return nil, err
581 }
582
583 networkName := s.getNetworkForApp(route.Domain)
584 if err := s.createNetworkIfNeeded(ctx, networkName); err != nil {
585 return nil, log.WrapErr(err, "failed to create network")
586 }
587 if err := s.deployAttachments(ctx, route.Domain, networkName); err != nil {
588 return nil, log.WrapErr(err, "failed to deploy attachments")
589 }
590
591 exposedPorts, err := s.runtime.GetImageExposedPorts(ctx, actualImageRef)
592 if err != nil {
593 log.WrapErr(err, "failed to get exposed ports, using defaults")
594 exposedPorts = []int{80, 8080, 3000}
595 }
596
597 imageLabels, err := s.runtime.GetImageLabels(ctx, actualImageRef)
598 if err != nil {
599 log.Warn().Err(err).Msg("failed to get image labels, skipping label propagation")
600 imageLabels = nil
601 }
602
603 envVars, err := s.loadEnvironment(ctx, route.Env, route.Domain, actualImageRef)
604 if err != nil {
605 return nil, err
606 }
607 envHash := hashEnvironment(envVars)
608
609 volumes, err := s.setupVolumes(ctx, route.Domain, actualImageRef)
610 if err != nil {
611 return nil, err
612 }
613
614 return &deployResources{
615 networkName: networkName,
616 actualImageRef: actualImageRef,
617 exposedPorts: exposedPorts,
618 imageLabels: imageLabels,
619 envVars: envVars,
620 envHash: envHash,
621 volumes: volumes,
622 }, nil
623}
624
625func (s *Service) createStartedContainer(ctx context.Context, route domain.Route, existing *domain.Container, resources *deployResources) (*domain.Container, error) {
626 ctx, span := tracer.Start(ctx, "container.create_and_start")
627 defer span.End()
628
629 log := zerowrap.FromCtx(ctx)
630
631 containerConfig := s.buildContainerConfig(containerConfigInput{
632 Domain: route.Domain,
633 Image: route.Image,
634 ImageRef: resources.actualImageRef,
635 ExposedPorts: resources.exposedPorts,
636 EnvVars: resources.envVars,
637 EnvHash: resources.envHash,
638 Volumes: resources.volumes,
639 NetworkName: resources.networkName,
640 ImageLabels: resources.imageLabels,
641 Existing: existing,
642 })
643
644 newContainer, err := s.runtime.CreateContainer(ctx, containerConfig)
645 if err != nil {
646 return nil, log.WrapErr(err, "failed to create container")
647 }
648 if err := s.runtime.StartContainer(ctx, newContainer.ID); err != nil {
649 s.runtime.RemoveContainer(ctx, newContainer.ID, true)
650 return nil, log.WrapErr(err, "failed to start container")
651 }
652 if !domain.IsSkipReadiness(ctx) {
653 if err := s.waitForReady(ctx, newContainer.ID, containerConfig); err != nil {
654 deployErr := s.newDeployFailure(newContainer, "container failed readiness check", err, s.captureRecentContainerLogs(ctx, newContainer.ID))
655 s.cleanupFailedContainer(ctx, newContainer.ID)
656 return nil, deployErr
657 }
658 }
659
660 inspected, err := s.runtime.InspectContainer(ctx, newContainer.ID)
661 if err != nil {
662 s.cleanupFailedContainer(ctx, newContainer.ID)
663 return nil, log.WrapErr(err, "failed to inspect started container")
664 }
665
666 return inspected, nil
667}
668
669func (s *Service) newDeployFailure(container *domain.Container, cause string, err error, logs []string) error {
670 deployErr := &domain.DeployFailureError{
671 Summary: "failed to deploy",
672 Cause: cause,
673 Hint: inferDeployFailureHint(err),
674 Logs: logs,
675 Err: err,
676 }
677 if container != nil {
678 deployErr.ContainerName = container.Name
679 deployErr.ContainerID = container.ID
680 }
681
682 return deployErr
683}
684
685func (s *Service) wrapPullDeployFailure(route domain.Route, existing *domain.Container, err error) error {
686 if !isPullFailure(err) {
687 return nil
688 }
689
690 return s.newDeployFailure(&domain.Container{Name: s.deploymentContainerName(route.Domain, existing)}, "failed to pull image", err, nil)
691}
692
693func (s *Service) captureRecentContainerLogs(ctx context.Context, containerID string) []string {
694 if ctx.Err() != nil {
695 return nil
696 }
697
698 logStream, err := s.runtime.GetContainerLogs(ctx, containerID, false)
699 if err != nil {
700 return nil
701 }
702 defer func() {
703 _ = logStream.Close()
704 }()
705
706 logs, err := parseDockerLogLines(logStream, deployFailureLogLimit)
707 if err != nil {
708 return nil
709 }
710
711 return logs
712}
713
714func parseDockerLogLines(r io.Reader, maxLines int) ([]string, error) {
715 reader := bufio.NewReader(r)
716 collector := newLogLineCollector(maxLines)
717
718 header, err := reader.Peek(8)
719 if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, bufio.ErrBufferFull) {
720 return nil, err
721 }
722
723 if len(header) >= 8 && looksLikeDockerLogStream(header) {
724 return parseDockerFramedLogLines(reader, collector)
725 }
726
727 return parsePlainLogLines(reader, collector)
728}
729
730func parseDockerFramedLogLines(reader *bufio.Reader, collector *logLineCollector) ([]string, error) {
731 for {
732 frameHeader := make([]byte, 8)
733 if _, err := io.ReadFull(reader, frameHeader); err != nil {
734 if isLogStreamEOF(err) {
735 collector.flush()
736 return collector.lines, nil
737 }
738 return nil, err
739 }
740
741 size := int(binary.BigEndian.Uint32(frameHeader[4:8]))
742 if size <= 0 {
743 continue
744 }
745
746 payload, err := readLogPayload(reader, size)
747 if err != nil {
748 collector.addChunk(string(payload))
749 collector.flush()
750 if isLogStreamEOF(err) {
751 return collector.lines, nil
752 }
753 return nil, err
754 }
755
756 collector.addChunk(string(payload))
757 }
758}
759
760func parsePlainLogLines(reader *bufio.Reader, collector *logLineCollector) ([]string, error) {
761
762 scanner := bufio.NewScanner(reader)
763 for scanner.Scan() {
764 collector.addLine(scanner.Text())
765 }
766 if err := scanner.Err(); err != nil {
767 return nil, err
768 }
769
770 collector.flush()
771 return collector.lines, nil
772}
773
774func readLogPayload(reader *bufio.Reader, size int) ([]byte, error) {
775 payload := make([]byte, size)
776 _, err := io.ReadFull(reader, payload)
777 return payload, err
778}
779
780func isLogStreamEOF(err error) bool {
781 return errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF)
782}
783
784func looksLikeDockerLogStream(data []byte) bool {
785 return len(data) >= 8 && data[1] == 0 && data[2] == 0 && data[3] == 0
786}
787
788type logLineCollector struct {
789 lines []string
790 pending string
791 maxLines int
792}
793
794func newLogLineCollector(maxLines int) *logLineCollector {
795 return &logLineCollector{maxLines: maxLines}
796}
797
798func (c *logLineCollector) addChunk(chunk string) {
799 for _, part := range strings.SplitAfter(chunk, "\n") {
800 c.pending += part
801 if strings.HasSuffix(part, "\n") {
802 c.flush()
803 }
804 }
805}
806
807func (c *logLineCollector) addLine(line string) {
808 line = strings.TrimRight(line, "\r")
809 if line == "" {
810 return
811 }
812 c.lines = append(c.lines, line)
813 if c.maxLines > 0 && len(c.lines) > c.maxLines {
814 c.lines = c.lines[len(c.lines)-c.maxLines:]
815 }
816}
817
818func (c *logLineCollector) flush() {
819 if c.pending == "" {
820 return
821 }
822 c.addLine(strings.TrimRight(c.pending, "\r\n"))
823 c.pending = ""
824}
825
826func inferDeployFailureHint(err error) string {
827 if err == nil {
828 return ""
829 }
830
831 msg := strings.ToLower(err.Error())
832 switch {
833 case strings.Contains(msg, "healthcheck timeout"):
834 return "check startup logs and confirm the health endpoint becomes ready within the configured timeout"
835 case strings.Contains(msg, "unhealthy"):
836 return "inspect recent app logs and verify the healthcheck command reflects actual readiness"
837 case strings.Contains(msg, "no healthcheck detected"):
838 return "add a Docker healthcheck or switch readiness mode"
839 case isPullFailure(err), strings.Contains(msg, "access denied"), strings.Contains(msg, "unauthorized"):
840 return "verify registry auth and confirm the image exists at the requested tag"
841 default:
842 return ""
843 }
844}
845
846func isPullFailure(err error) bool {
847 if err == nil {
848 return false
849 }
850
851 if errors.Is(err, domain.ErrImagePullFailed) {
852 return true
853 }
854
855 msg := strings.ToLower(err.Error())
856 return strings.Contains(msg, "failed to pull image") ||
857 strings.Contains(msg, "pull access denied")
858}
859
860func (s *Service) activateDeployedContainer(ctx context.Context, domainName string, container *domain.Container) bool {
861 s.mu.Lock()
862 _, wasTracked := s.containers[domainName]
863 s.containers[domainName] = container
864 if !wasTracked {
865 s.managedCount++
866 }
867 s.mu.Unlock()
868
869 // Track managed container count (only increment for new domains, not replacements)
870 if !wasTracked && s.metrics != nil {
871 s.metrics.ManagedContainers.Add(ctx, 1)
872 }
873
874 s.publishContainerDeployed(ctx, domainName, container.ID)
875
876 s.mu.RLock()
877 inv := s.cacheInvalidator
878 s.mu.RUnlock()
879 if inv != nil {
880 inv.InvalidateTarget(ctx, domainName)
881 return true
882 }
883
884 return false
885}
886
887// stabilizeNewContainer monitors the new container briefly after traffic switch.
888// If it crashes during this window, rolls back to old container.
889// Returns true if stabilization succeeded, false if rollback was performed.
890// Returns an error only when both new and old containers are dead.
891func (s *Service) stabilizeNewContainer(ctx context.Context, domainName string, newContainer, oldContainer *domain.Container) (bool, error) {
892 log := zerowrap.FromCtx(ctx)
893
894 if oldContainer == nil {
895 return true, nil
896 }
897
898 s.mu.RLock()
899 delay := s.config.StabilizationDelay
900 s.mu.RUnlock()
901 if delay == 0 {
902 delay = 2 * time.Second
903 }
904
905 // Brief stabilization: verify new container is still running after delay
906 select {
907 case <-time.After(delay):
908 case <-ctx.Done():
909 return true, nil
910 }
911
912 running, err := s.runtime.IsContainerRunning(ctx, newContainer.ID)
913 if err != nil || !running {
914 log.Error().
915 Str("new_container", newContainer.ID).
916 Str("old_container", oldContainer.ID).
917 Msg("new container crashed during stabilization, rolling back to old")
918
919 // Verify old container is still running before restoring it.
920 // If both old and new are dead (e.g., OOM), returning a dead
921 // container as "existing" would leave the domain in a broken state.
922 oldRunning, oldErr := s.runtime.IsContainerRunning(ctx, oldContainer.ID)
923 if oldErr != nil || !oldRunning {
924 log.Error().
925 Str("old_container", oldContainer.ID).
926 Msg("old container is also not running, cannot rollback")
927
928 // Cleanup failed new container
929 if stopErr := s.runtime.StopContainer(ctx, newContainer.ID); stopErr != nil {
930 log.WrapErrWithFields(stopErr, "failed to stop failed new container during rollback", map[string]any{zerowrap.FieldEntityID: newContainer.ID})
931 }
932 if removeErr := s.runtime.RemoveContainer(ctx, newContainer.ID, true); removeErr != nil {
933 log.WrapErrWithFields(removeErr, "failed to remove failed new container during rollback", map[string]any{zerowrap.FieldEntityID: newContainer.ID})
934 }
935
936 return false, fmt.Errorf("stabilization failed: new container crashed and old container %s is also not running", oldContainer.ID)
937 }
938
939 // Rollback: restore old container as tracked
940 s.mu.Lock()
941 s.containers[domainName] = oldContainer
942 s.mu.Unlock()
943
944 // Re-invalidate proxy cache to point back to old
945 s.mu.RLock()
946 inv := s.cacheInvalidator
947 s.mu.RUnlock()
948 if inv != nil {
949 inv.InvalidateTarget(ctx, domainName)
950 }
951
952 // Cleanup failed new container
953 if stopErr := s.runtime.StopContainer(ctx, newContainer.ID); stopErr != nil {
954 log.WrapErrWithFields(stopErr, "failed to stop failed new container during rollback", map[string]any{zerowrap.FieldEntityID: newContainer.ID})
955 }
956 if removeErr := s.runtime.RemoveContainer(ctx, newContainer.ID, true); removeErr != nil {
957 log.WrapErrWithFields(removeErr, "failed to remove failed new container during rollback", map[string]any{zerowrap.FieldEntityID: newContainer.ID})
958 }
959
960 return false, nil
961 }
962
963 return true, nil
964}
965
966func (s *Service) finalizePreviousContainer(ctx context.Context, domainName string, existing *domain.Container, hasExisting, invalidated bool, newContainerID string) {
967 if !hasExisting {
968 return
969 }
970
971 if invalidated {
972 s.waitForDrain(ctx, existing.ID)
973 }
974
975 s.cleanupOldContainer(ctx, existing, newContainerID, domainName)
976}
977
978func (s *Service) waitForDrain(ctx context.Context, oldContainerID string) {
979 log := zerowrap.FromCtx(ctx)
980
981 s.mu.RLock()
982 cfg := s.config
983 waiter := s.drainWaiter
984 s.mu.RUnlock()
985
986 drainMode := cfg.DrainMode
987 if drainMode == "" {
988 drainMode = "delay"
989 }
990
991 shouldUseInFlight := (drainMode == "inflight" || drainMode == "auto") && waiter != nil
992 if shouldUseInFlight {
993 timeout := cfg.DrainTimeout
994 if timeout == 0 {
995 timeout = 30 * time.Second
996 }
997 drained := waiter.WaitForNoInFlight(ctx, oldContainerID, timeout)
998 if !drained {
999 log.Warn().
1000 Str("old_container_id", oldContainerID).
1001 Dur("drain_timeout", timeout).
1002 Msg("drain wait timed out; old container may still have in-flight traffic")
1003 }
1004 return
1005 }
1006
1007 drainDelay := 2 * time.Second
1008 if cfg.DrainDelayConfigured {
1009 drainDelay = cfg.DrainDelay
1010 }
1011 if drainDelay <= 0 {
1012 return
1013 }
1014 select {
1015 case <-time.After(drainDelay):
1016 case <-ctx.Done():
1017 }
1018}
1019
1020// restartContainerWithRecovery restarts a container, reconciling stale state if needed.
1021// It returns the (possibly refreshed) container and updated attachment IDs.
1022func (s *Service) restartContainerWithRecovery(ctx context.Context, domainName string, container *domain.Container, attachmentIDs []string) (*domain.Container, []string, error) {
1023 log := zerowrap.FromCtx(ctx)
1024
1025 if err := s.runtime.RestartContainer(ctx, container.ID); err != nil {
1026 if !isContainerNotFoundError(err) {
1027 return nil, nil, log.WrapErr(err, "failed to restart container")
1028 }
1029
1030 // In-memory container ID can become stale after external runtime changes.
1031 // Re-sync and retry once with the latest tracked container.
1032 log.Warn().
1033 Err(err).
1034 Str(zerowrap.FieldEntityID, container.ID).
1035 Msg("tracked container missing during restart, attempting state reconciliation")
1036
1037 if syncErr := s.SyncContainers(ctx); syncErr != nil {
1038 log.Warn().Err(syncErr).Msg("failed to sync container state during restart recovery")
1039 }
1040
1041 s.mu.RLock()
1042 refreshed, refreshedExists := s.containers[domainName]
1043 attachmentIDs = append([]string{}, s.attachments[domainName]...)
1044 s.mu.RUnlock()
1045
1046 if !refreshedExists || refreshed == nil {
1047 return nil, nil, domain.ErrContainerNotFound
1048 }
1049 if err := s.runtime.RestartContainer(ctx, refreshed.ID); err != nil {
1050 return nil, nil, log.WrapErr(err, "failed to restart container after state reconciliation")
1051 }
1052 return refreshed, attachmentIDs, nil
1053 }
1054
1055 return container, attachmentIDs, nil
1056}
1057
1058// Restart restarts a running container for the given domain.
1059func (s *Service) Restart(ctx context.Context, domainName string, withAttachments bool) error {
1060 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1061 zerowrap.FieldLayer: "usecase",
1062 zerowrap.FieldUseCase: "Restart",
1063 "domain": domainName,
1064 })
1065 log := zerowrap.FromCtx(ctx)
1066
1067 s.mu.RLock()
1068 container, exists := s.containers[domainName]
1069 attachments := s.attachments[domainName]
1070 var attachmentIDs []string
1071 if len(attachments) > 0 {
1072 attachmentIDs = make([]string, len(attachments))
1073 copy(attachmentIDs, attachments)
1074 }
1075 s.mu.RUnlock()
1076
1077 if !exists || container == nil {
1078 return domain.ErrContainerNotFound
1079 }
1080
1081 // When attachments are requested, sync runtime state first to get accurate
1082 // attachment tracking, then check if configured attachments are deployed.
1083 // If fewer attachment containers are tracked than configured, fail early with
1084 // guidance rather than restarting the main container into a broken state
1085 // (e.g., missing database hostname).
1086 if withAttachments {
1087 if syncErr := s.SyncContainers(ctx); syncErr != nil {
1088 log.Warn().Err(syncErr).Msg("failed to sync container state before attachment check")
1089 }
1090
1091 // Re-read attachment IDs after sync
1092 s.mu.RLock()
1093 if freshAttachments := s.attachments[domainName]; len(freshAttachments) > 0 {
1094 attachmentIDs = make([]string, len(freshAttachments))
1095 copy(attachmentIDs, freshAttachments)
1096 } else {
1097 attachmentIDs = nil
1098 }
1099 s.mu.RUnlock()
1100
1101 configuredAttachments := s.resolveAttachmentsForDomain(domainName)
1102 if len(configuredAttachments) > 0 && len(attachmentIDs) < len(configuredAttachments) {
1103 missing := len(configuredAttachments) - len(attachmentIDs)
1104 return fmt.Errorf("%w: domain %q has %d configured attachment(s) but only %d deployed (%d missing)",
1105 domain.ErrAttachmentNotDeployed, domainName, len(configuredAttachments), len(attachmentIDs), missing)
1106 }
1107 }
1108
1109 // Restart the main container (with stale-state recovery)
1110 container, attachmentIDs, err := s.restartContainerWithRecovery(ctx, domainName, container, attachmentIDs)
1111 if err != nil {
1112 return err
1113 }
1114 log.Info().Str(zerowrap.FieldEntityID, container.ID).Msg("container restarted")
1115
1116 // Record restart metric
1117 if s.metrics != nil {
1118 s.metrics.ContainerRestarts.Add(ctx, 1, metric.WithAttributes(
1119 attribute.String("domain", domainName),
1120 attribute.String("source", "api"),
1121 ))
1122 }
1123
1124 // Restart attachments if requested
1125 if withAttachments && len(attachmentIDs) > 0 {
1126 for _, attachID := range attachmentIDs {
1127 if err := s.runtime.RestartContainer(ctx, attachID); err != nil {
1128 log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to restart attachment")
1129 continue // Don't fail the whole operation for one attachment
1130 }
1131 log.Info().Str("attachment_id", attachID).Msg("attachment restarted")
1132 }
1133 }
1134
1135 return nil
1136}
1137
1138// Stop stops a running container.
1139func (s *Service) Stop(ctx context.Context, containerID string) error {
1140 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1141 zerowrap.FieldLayer: "usecase",
1142 zerowrap.FieldUseCase: "Stop",
1143 zerowrap.FieldEntityID: containerID,
1144 })
1145 log := zerowrap.FromCtx(ctx)
1146
1147 // Stop log collection before stopping container
1148 if s.logWriter != nil {
1149 if err := s.logWriter.StopLogging(containerID); err != nil {
1150 log.Warn().Err(err).Msg("failed to stop container log collection")
1151 }
1152 }
1153
1154 if err := s.runtime.StopContainer(ctx, containerID); err != nil {
1155 return log.WrapErr(err, "failed to stop container")
1156 }
1157
1158 log.Info().Msg("container stopped")
1159 return nil
1160}
1161
1162// ReconcileRemovedRoute removes active runtime containers for a route that was
1163// removed from configuration while preserving stateful resources for explicit
1164// follow-up cleanup. It intentionally removes only main route containers, never
1165// attachment containers or volumes.
1166func (s *Service) ReconcileRemovedRoute(ctx context.Context, domainName string) (*domain.CleanupReport, error) {
1167 canonicalDomain, ok := domain.CanonicalRouteDomain(domainName)
1168 if !ok {
1169 return &domain.CleanupReport{Domain: domainName}, domain.ErrRouteDomainInvalid
1170 }
1171 domainName = canonicalDomain
1172
1173 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1174 zerowrap.FieldLayer: "usecase",
1175 zerowrap.FieldUseCase: "ReconcileRemovedRoute",
1176 "domain": domainName,
1177 })
1178 log := zerowrap.FromCtx(ctx)
1179
1180 report := &domain.CleanupReport{Domain: domainName}
1181
1182 unlock, err := s.acquireDomainDeployLock(ctx, domainName)
1183 if err != nil {
1184 return report, err
1185 }
1186 defer unlock()
1187
1188 allContainers, err := s.runtime.ListContainers(ctx, true)
1189 if err != nil {
1190 return report, log.WrapErr(err, "failed to list containers for removed route cleanup")
1191 }
1192
1193 routeContainers := routeContainersForDomain(allContainers, domainName)
1194
1195 // Clear in-memory state before stopping containers so the monitor cannot
1196 // interpret the deliberate removal as a crash that should be restarted.
1197 s.clearRouteContainerTracking(ctx, domainName)
1198 if inv := s.proxyCacheInvalidator(); inv != nil {
1199 inv.InvalidateTarget(ctx, domainName)
1200 }
1201
1202 report.PreservedAttachments = preservedAttachmentsForDomain(allContainers, domainName)
1203 if len(report.PreservedAttachments) > 0 {
1204 report.Hints = append(report.Hints, "attachments preserved; remove or purge attachments explicitly if they are no longer needed")
1205 }
1206
1207 if len(routeContainers) == 0 {
1208 report.Warnings = append(report.Warnings, "no active route container found for removed route")
1209 return report, nil
1210 }
1211
1212 s.removeRouteContainersForCleanup(ctx, log, routeContainers, report)
1213
1214 if len(report.PartialFailures) > 0 {
1215 report.Warnings = append(report.Warnings, "route cleanup completed with partial failures; manual follow-up may be required")
1216 }
1217
1218 return report, nil
1219}
1220
1221func routeContainersForDomain(containers []*domain.Container, domainName string) []*domain.Container {
1222 var routeContainers []*domain.Container
1223 for _, c := range containers {
1224 if isManagedRouteContainerForDomain(c, domainName) {
1225 routeContainers = append(routeContainers, c)
1226 }
1227 }
1228 return routeContainers
1229}
1230
1231func preservedAttachmentsForDomain(containers []*domain.Container, domainName string) []domain.CleanupAttachment {
1232 var attachments []domain.CleanupAttachment
1233 for _, c := range containers {
1234 if isPreservedAttachmentForDomain(c, domainName) {
1235 attachments = append(attachments, domain.CleanupAttachment{
1236 Name: c.Name,
1237 Image: c.Image,
1238 ContainerID: c.ID,
1239 Status: c.Status,
1240 Owner: c.Labels[domain.LabelAttachedTo],
1241 Reason: "attachments are preserved by default; remove or purge them explicitly",
1242 })
1243 }
1244 }
1245 return attachments
1246}
1247
1248func (s *Service) removeRouteContainersForCleanup(ctx context.Context, log zerowrap.Logger, containers []*domain.Container, report *domain.CleanupReport) {
1249 for _, c := range containers {
1250 s.stopRouteLogCollectionForCleanup(log, c, report)
1251 s.stopRouteContainerForCleanup(ctx, log, c, report)
1252 if err := s.runtime.RemoveContainer(ctx, c.ID, true); err != nil {
1253 report.PartialFailures = append(report.PartialFailures, domain.CleanupFailure{Action: "remove", Kind: "container", ID: c.ID, Name: c.Name, Error: err.Error()})
1254 log.Warn().Err(err).Str(zerowrap.FieldEntityID, c.ID).Msg("failed to remove removed route container")
1255 continue
1256 }
1257 report.RemovedContainers = append(report.RemovedContainers, domain.CleanupContainer{ID: c.ID, Name: c.Name, Image: c.Image, Status: c.Status})
1258 }
1259}
1260
1261func (s *Service) stopRouteLogCollectionForCleanup(log zerowrap.Logger, c *domain.Container, report *domain.CleanupReport) {
1262 if s.logWriter == nil {
1263 return
1264 }
1265 if err := s.logWriter.StopLogging(c.ID); err != nil {
1266 report.PartialFailures = append(report.PartialFailures, domain.CleanupFailure{Action: "stop_logging", Kind: "container", ID: c.ID, Name: c.Name, Error: err.Error()})
1267 log.Warn().Err(err).Str(zerowrap.FieldEntityID, c.ID).Msg("failed to stop removed route log collection")
1268 }
1269}
1270
1271func (s *Service) stopRouteContainerForCleanup(ctx context.Context, log zerowrap.Logger, c *domain.Container, report *domain.CleanupReport) {
1272 if c.Status != string(domain.ContainerStatusRunning) {
1273 return
1274 }
1275 if err := s.runtime.StopContainer(ctx, c.ID); err != nil {
1276 report.PartialFailures = append(report.PartialFailures, domain.CleanupFailure{Action: "stop", Kind: "container", ID: c.ID, Name: c.Name, Error: err.Error()})
1277 log.Warn().Err(err).Str(zerowrap.FieldEntityID, c.ID).Msg("failed to stop removed route container")
1278 }
1279}
1280
1281func isManagedRouteContainerForDomain(c *domain.Container, domainName string) bool {
1282 if c == nil || c.Labels == nil || c.Labels[domain.LabelManaged] != "true" {
1283 return false
1284 }
1285 if c.Labels[domain.LabelAttachment] == "true" {
1286 return false
1287 }
1288 return c.Labels[domain.LabelRoute] == domainName || c.Labels[domain.LabelDomain] == domainName || c.Name == fmt.Sprintf("gordon-%s", domainName)
1289}
1290
1291func isPreservedAttachmentForDomain(c *domain.Container, domainName string) bool {
1292 if c == nil || c.Labels == nil || c.Labels[domain.LabelManaged] != "true" {
1293 return false
1294 }
1295 return c.Labels[domain.LabelAttachment] == "true" && c.Labels[domain.LabelAttachedTo] == domainName
1296}
1297
1298func (s *Service) proxyCacheInvalidator() out.ProxyCacheInvalidator {
1299 s.mu.RLock()
1300 defer s.mu.RUnlock()
1301 return s.cacheInvalidator
1302}
1303
1304func (s *Service) clearRouteContainerTracking(ctx context.Context, domainName string) {
1305 s.mu.Lock()
1306 removed := false
1307 if _, exists := s.containers[domainName]; exists {
1308 delete(s.containers, domainName)
1309 if s.managedCount > 0 {
1310 s.managedCount--
1311 }
1312 removed = true
1313 }
1314 s.mu.Unlock()
1315
1316 if removed && s.metrics != nil {
1317 s.metrics.ManagedContainers.Add(ctx, -1)
1318 }
1319}
1320
1321// Remove removes a container.
1322func (s *Service) Remove(ctx context.Context, containerID string, force bool) error {
1323 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1324 zerowrap.FieldLayer: "usecase",
1325 zerowrap.FieldUseCase: "Remove",
1326 zerowrap.FieldEntityID: containerID,
1327 })
1328 log := zerowrap.FromCtx(ctx)
1329
1330 // Stop log collection before removing container
1331 if s.logWriter != nil {
1332 if err := s.logWriter.StopLogging(containerID); err != nil {
1333 log.Warn().Err(err).Msg("failed to stop container log collection")
1334 }
1335 }
1336
1337 // Find domain and attachment IDs for cleanup
1338 containerDomain, attachmentIDs, cfg := s.findContainerContext(containerID)
1339
1340 if err := s.runtime.RemoveContainer(ctx, containerID, force); err != nil {
1341 return log.WrapErr(err, "failed to remove container")
1342 }
1343
1344 // Cleanup volumes
1345 if containerDomain != "" && !cfg.VolumePreserve {
1346 if err := s.cleanupVolumesForDomain(ctx, containerDomain); err != nil {
1347 log.WrapErrWithFields(err, "failed to cleanup volumes", map[string]any{"domain": containerDomain})
1348 }
1349 }
1350
1351 // Clean up attachment containers
1352 s.removeAttachments(ctx, attachmentIDs)
1353
1354 // Remove from tracking
1355 s.mu.Lock()
1356 var removedDomain string
1357 for d, c := range s.containers {
1358 if c.ID == containerID {
1359 delete(s.containers, d)
1360 removedDomain = d
1361 s.managedCount--
1362 log.Info().Str("domain", d).Msg("container removed")
1363 break
1364 }
1365 }
1366 if removedDomain != "" {
1367 delete(s.attachments, removedDomain)
1368 }
1369 s.mu.Unlock()
1370
1371 // Decrement managed container count
1372 if removedDomain != "" && s.metrics != nil {
1373 s.metrics.ManagedContainers.Add(ctx, -1)
1374 }
1375
1376 // Note: We intentionally do NOT clean up deployMu entries to avoid a race condition:
1377 // If Remove deletes the mutex entry while a concurrent Deploy is acquiring the lock,
1378 // the Deploy would create a fresh mutex, breaking serialization and allowing concurrent
1379 // deploys to the same domain. The memory footprint is acceptable (one small struct per
1380 // domain ever deployed), and this is the safer choice for correctness.
1381
1382 // Cleanup network
1383 if removedDomain != "" && cfg.NetworkIsolation {
1384 networkName := s.getNetworkForApp(removedDomain)
1385 if err := s.cleanupNetworkIfEmpty(ctx, networkName); err != nil {
1386 log.WrapErrWithFields(err, "failed to cleanup network", map[string]any{"network": networkName})
1387 }
1388 }
1389
1390 return nil
1391}
1392
1393// findContainerContext looks up the domain, attachment IDs, and config snapshot for a container.
1394func (s *Service) findContainerContext(containerID string) (string, []string, Config) {
1395 s.mu.RLock()
1396 defer s.mu.RUnlock()
1397 cfg := s.config
1398 var containerDomain string
1399 for d, c := range s.containers {
1400 if c.ID == containerID {
1401 containerDomain = d
1402 break
1403 }
1404 }
1405 var attachmentIDs []string
1406 if containerDomain != "" {
1407 attachmentIDs = append(attachmentIDs, s.attachments[containerDomain]...)
1408 }
1409 return containerDomain, attachmentIDs, cfg
1410}
1411
1412// removeAttachments stops and removes attachment containers best-effort.
1413func (s *Service) removeAttachments(ctx context.Context, attachmentIDs []string) {
1414 log := zerowrap.FromCtx(ctx)
1415 for _, attachID := range attachmentIDs {
1416 if s.logWriter != nil {
1417 if err := s.logWriter.StopLogging(attachID); err != nil {
1418 log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to stop attachment log collection")
1419 }
1420 }
1421 if err := s.runtime.StopContainer(ctx, attachID); err != nil {
1422 log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to stop attachment container")
1423 }
1424 if err := s.runtime.RemoveContainer(ctx, attachID, true); err != nil {
1425 log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to remove attachment container")
1426 }
1427 }
1428}
1429
1430// Get retrieves a container by domain.
1431func (s *Service) Get(_ context.Context, domainName string) (*domain.Container, bool) {
1432 s.mu.RLock()
1433 defer s.mu.RUnlock()
1434 container, exists := s.containers[domainName]
1435 return container, exists
1436}
1437
1438// List returns all managed containers.
1439func (s *Service) List(_ context.Context) map[string]*domain.Container {
1440 s.mu.RLock()
1441 defer s.mu.RUnlock()
1442
1443 result := make(map[string]*domain.Container, len(s.containers))
1444 maps.Copy(result, s.containers)
1445 return result
1446}
1447
1448// ListRoutesWithDetails returns routes with network and attachment info.
1449// Note: Container map is copied under lock, then external calls are made without lock.
1450// If containers are removed between copy and runtime calls, errors are handled gracefully
1451// (network becomes empty string). This trade-off avoids holding locks during I/O.
1452func (s *Service) ListRoutesWithDetails(ctx context.Context) []domain.RouteInfo {
1453 s.mu.RLock()
1454 containers := make(map[string]*domain.Container, len(s.containers))
1455 maps.Copy(containers, s.containers)
1456 s.mu.RUnlock()
1457
1458 // Fetch all attachments once to avoid N+1 queries
1459 attachmentsByDomain := s.getAllAttachments(ctx)
1460
1461 results := make([]domain.RouteInfo, 0, len(containers))
1462 for domainName, container := range containers {
1463 network := ""
1464 image := ""
1465 containerID := ""
1466 status := ""
1467 if container != nil {
1468 containerID = container.ID
1469 status = container.Status
1470 image = container.Image
1471 if container.Labels != nil {
1472 if labelImage, ok := container.Labels[domain.LabelImage]; ok && labelImage != "" {
1473 image = labelImage
1474 }
1475 }
1476 // Strip registry domain prefix from image for cleaner display
1477 image = s.stripRegistryPrefix(image)
1478 if networkName, err := s.runtime.GetContainerNetwork(ctx, container.ID); err == nil {
1479 network = networkName
1480 }
1481 }
1482
1483 results = append(results, domain.RouteInfo{
1484 Domain: domainName,
1485 Image: image,
1486 ContainerID: containerID,
1487 ContainerStatus: status,
1488 Network: network,
1489 Attachments: attachmentsByDomain[domainName],
1490 })
1491 }
1492
1493 return results
1494}
1495
1496// stripRegistryPrefix removes the configured current or legacy Gordon registry
1497// domain prefix from an image reference. For example,
1498// "reg.example.com/myapp:latest" becomes "myapp:latest" when
1499// "reg.example.com" is a current or legacy Gordon registry domain.
1500func (s *Service) stripRegistryPrefix(image string) string {
1501 s.mu.RLock()
1502 cfg := s.config
1503 s.mu.RUnlock()
1504
1505 return domain.StripKnownGordonRegistry(image, cfg.RegistryDomain, cfg.LegacyRegistryDomains)
1506}
1507
1508// ListAttachments returns attachments for a domain.
1509func (s *Service) ListAttachments(ctx context.Context, domainName string) []domain.Attachment {
1510 return s.getAttachmentsForDomain(ctx, domainName)
1511}
1512
1513// ListOrphanedAttachments returns running attachment containers no longer configured.
1514func (s *Service) ListOrphanedAttachments(ctx context.Context) ([]domain.CleanupAttachment, error) {
1515 containers, err := s.runtime.ListContainers(ctx, true)
1516 if err != nil {
1517 return nil, fmt.Errorf("failed to list containers with runtime for orphaned attachments: %w", err)
1518 }
1519 return s.orphanedAttachmentsFromContainers(containers), nil
1520}
1521
1522// CleanupOrphanedAttachments optionally stops/removes orphaned attachment containers.
1523// When owner is non-empty, cleanup is scoped to that attachment owner.
1524func (s *Service) CleanupOrphanedAttachments(ctx context.Context, owner string, stop bool) (*domain.CleanupReport, error) {
1525 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1526 zerowrap.FieldLayer: "usecase",
1527 zerowrap.FieldUseCase: "CleanupOrphanedAttachments",
1528 "owner": owner,
1529 })
1530 log := zerowrap.FromCtx(ctx)
1531 report := &domain.CleanupReport{Domain: owner}
1532
1533 containers, err := s.runtime.ListContainers(ctx, true)
1534 if err != nil {
1535 return report, fmt.Errorf("failed to list containers with runtime for orphaned attachment cleanup: %w", err)
1536 }
1537 orphans := filterAttachmentContainersByOwner(s.orphanedAttachmentContainers(containers), owner)
1538 if !stop {
1539 report.PreservedAttachments = cleanupAttachmentsFromContainers(orphans)
1540 if len(report.PreservedAttachments) > 0 {
1541 report.Hints = append(report.Hints, "orphaned attachments preserved; rerun with --stop to stop/remove containers while preserving volumes")
1542 }
1543 return report, nil
1544 }
1545
1546 for _, c := range orphans {
1547 s.stopRouteLogCollectionForCleanup(log, c, report)
1548 s.stopRouteContainerForCleanup(ctx, log, c, report)
1549 if err := s.runtime.RemoveContainer(ctx, c.ID, true); err != nil {
1550 report.PartialFailures = append(report.PartialFailures, domain.CleanupFailure{Action: "remove", Kind: "attachment", ID: c.ID, Name: c.Name, Error: err.Error()})
1551 log.Warn().Err(err).Str(zerowrap.FieldEntityID, c.ID).Msg("failed to remove orphaned attachment")
1552 continue
1553 }
1554 report.RemovedContainers = append(report.RemovedContainers, domain.CleanupContainer{ID: c.ID, Name: c.Name, Image: attachmentImage(c), Status: c.Status})
1555 s.untrackAttachmentContainer(c.Labels[domain.LabelAttachedTo], c.ID)
1556 }
1557 if len(report.RemovedContainers) > 0 {
1558 report.Hints = append(report.Hints, "attachment volumes and data were preserved")
1559 }
1560 if len(report.PartialFailures) > 0 {
1561 report.Warnings = append(report.Warnings, "attachment cleanup completed with partial failures; manual follow-up may be required")
1562 }
1563 return report, nil
1564}
1565
1566func (s *Service) orphanedAttachmentsFromContainers(containers []*domain.Container) []domain.CleanupAttachment {
1567 return cleanupAttachmentsFromContainers(s.orphanedAttachmentContainers(containers))
1568}
1569
1570func (s *Service) orphanedAttachmentContainers(containers []*domain.Container) []*domain.Container {
1571 var out []*domain.Container
1572 for _, c := range containers {
1573 if !isManagedAttachment(c) {
1574 continue
1575 }
1576 owner := c.Labels[domain.LabelAttachedTo]
1577 image := attachmentImage(c)
1578 if owner == "" || image == "" || !s.isAttachmentConfiguredForOwner(owner, image) {
1579 out = append(out, c)
1580 }
1581 }
1582 return out
1583}
1584
1585func filterAttachmentContainersByOwner(containers []*domain.Container, owner string) []*domain.Container {
1586 if owner == "" {
1587 return containers
1588 }
1589 var out []*domain.Container
1590 for _, c := range containers {
1591 if c != nil && c.Labels != nil && strings.EqualFold(c.Labels[domain.LabelAttachedTo], owner) {
1592 out = append(out, c)
1593 }
1594 }
1595 return out
1596}
1597
1598func cleanupAttachmentsFromContainers(containers []*domain.Container) []domain.CleanupAttachment {
1599 attachments := make([]domain.CleanupAttachment, 0, len(containers))
1600 for _, c := range containers {
1601 attachments = append(attachments, domain.CleanupAttachment{
1602 Name: c.Name,
1603 Image: attachmentImage(c),
1604 ContainerID: c.ID,
1605 Status: c.Status,
1606 Owner: c.Labels[domain.LabelAttachedTo],
1607 Reason: "attachment container is no longer configured",
1608 })
1609 }
1610 return attachments
1611}
1612
1613func isManagedAttachment(c *domain.Container) bool {
1614 return c != nil && c.Status == string(domain.ContainerStatusRunning) && c.Labels != nil && c.Labels[domain.LabelManaged] == "true" && c.Labels[domain.LabelAttachment] == "true"
1615}
1616
1617func attachmentImage(c *domain.Container) string {
1618 if c == nil {
1619 return ""
1620 }
1621 if c.Labels != nil && c.Labels[domain.LabelImage] != "" {
1622 return c.Labels[domain.LabelImage]
1623 }
1624 return c.Image
1625}
1626
1627func (s *Service) isAttachmentConfiguredForOwner(owner, image string) bool {
1628 return slices.Contains(s.resolveAttachmentsForDomain(owner), image)
1629}
1630
1631func (s *Service) untrackAttachmentContainer(owner, containerID string) {
1632 s.mu.Lock()
1633 defer s.mu.Unlock()
1634 ids := s.attachments[owner]
1635 for i, id := range ids {
1636 if id == containerID {
1637 s.attachments[owner] = slices.Delete(ids, i, i+1)
1638 break
1639 }
1640 }
1641 if len(s.attachments[owner]) == 0 {
1642 delete(s.attachments, owner)
1643 }
1644}
1645
1646// ListNetworks returns Gordon-managed networks.
1647func (s *Service) ListNetworks(ctx context.Context) ([]*domain.NetworkInfo, error) {
1648 s.mu.RLock()
1649 cfg := s.config
1650 s.mu.RUnlock()
1651
1652 networks, err := s.runtime.ListNetworks(ctx)
1653 if err != nil {
1654 return nil, err
1655 }
1656
1657 var filtered []*domain.NetworkInfo
1658 for _, network := range networks {
1659 if strings.HasPrefix(network.Name, cfg.NetworkPrefix+"-") && network.Labels[domain.LabelManaged] == "true" {
1660 filtered = append(filtered, network)
1661 }
1662 }
1663
1664 return filtered, nil
1665}
1666
1667// HealthCheck performs health checks on all containers.
1668func (s *Service) HealthCheck(ctx context.Context) map[string]bool {
1669 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1670 zerowrap.FieldLayer: "usecase",
1671 zerowrap.FieldUseCase: "HealthCheck",
1672 })
1673 log := zerowrap.FromCtx(ctx)
1674
1675 // Copy container IDs under lock, then release before making Docker API calls
1676 s.mu.RLock()
1677 snapshot := make(map[string]string, len(s.containers))
1678 for d, c := range s.containers {
1679 snapshot[d] = c.ID
1680 }
1681 s.mu.RUnlock()
1682
1683 health := make(map[string]bool, len(snapshot))
1684 for d, id := range snapshot {
1685 running, err := s.runtime.IsContainerRunning(ctx, id)
1686 if err != nil {
1687 log.WrapErrWithFields(err, "health check failed", map[string]any{"domain": d, zerowrap.FieldEntityID: id})
1688 health[d] = false
1689 } else {
1690 health[d] = running
1691 }
1692 }
1693 return health
1694}
1695
1696// SyncContainers synchronizes containers with runtime state.
1697func (s *Service) SyncContainers(ctx context.Context) error {
1698 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1699 zerowrap.FieldLayer: "usecase",
1700 zerowrap.FieldUseCase: "SyncContainers",
1701 })
1702 log := zerowrap.FromCtx(ctx)
1703
1704 // List containers without holding lock to avoid blocking during the runtime call.
1705 // Use all=true so we can rebuild the running-container maps from a single
1706 // snapshot without a second list call.
1707 allContainers, err := s.runtime.ListContainers(ctx, true)
1708 if err != nil {
1709 return log.WrapErr(err, "failed to list containers")
1710 }
1711
1712 managed := make(map[string]*domain.Container)
1713 attachments := make(map[string][]string)
1714 for _, c := range allContainers {
1715 if c.Status != "" && c.Status != string(domain.ContainerStatusRunning) {
1716 continue
1717 }
1718 if c.Labels == nil || c.Labels[domain.LabelManaged] != "true" {
1719 continue
1720 }
1721 if c.Labels[domain.LabelAttachment] == "true" {
1722 owner := c.Labels[domain.LabelAttachedTo]
1723 if owner != "" {
1724 attachments[owner] = append(attachments[owner], c.ID)
1725 }
1726 continue
1727 }
1728 if d, ok := c.Labels[domain.LabelDomain]; ok {
1729 managed[d] = c
1730 }
1731 }
1732
1733 s.mu.Lock()
1734 s.containers = managed
1735 s.attachments = attachments
1736 newCount := int64(len(managed))
1737 delta := newCount - s.managedCount
1738 s.managedCount = newCount
1739 s.mu.Unlock()
1740
1741 // Report initial/delta managed container count to OTel UpDownCounter.
1742 if delta != 0 && s.metrics != nil {
1743 s.metrics.ManagedContainers.Add(ctx, delta)
1744 }
1745
1746 log.Info().Int(zerowrap.FieldCount, len(managed)).Msg("container state synchronized")
1747 return nil
1748}
1749
1750// AutoStart starts containers for the provided routes that aren't running.
1751// It skips readiness checks to avoid blocking boot; the background monitor
1752// handles crash recovery.
1753func (s *Service) AutoStart(ctx context.Context, routes []domain.Route) error {
1754 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1755 zerowrap.FieldLayer: "usecase",
1756 zerowrap.FieldUseCase: "AutoStart",
1757 })
1758 log := zerowrap.FromCtx(ctx)
1759
1760 log.Info().Int("route_count", len(routes)).Msg("auto-starting containers for configured routes")
1761
1762 // Filter out routes that already have a running container.
1763 var pending []domain.Route
1764 skipped := 0
1765 for _, route := range routes {
1766 if _, exists := s.Get(ctx, route.Domain); exists {
1767 log.Debug().Str("domain", route.Domain).Msg("container already running, skipping")
1768 skipped++
1769 } else {
1770 pending = append(pending, route)
1771 }
1772 }
1773
1774 if len(pending) == 0 {
1775 log.Info().Int("skipped", skipped).Msg("auto-start completed, all containers already running")
1776 return nil
1777 }
1778
1779 // Deploy all pending routes concurrently with readiness checks skipped.
1780 deployCtx := domain.WithSkipReadiness(ctx)
1781
1782 type result struct {
1783 route domain.Route
1784 err error
1785 }
1786 results := make(chan result, len(pending))
1787 sem := make(chan struct{}, 4) // limit concurrency
1788
1789 for _, route := range pending {
1790 sem <- struct{}{}
1791 go func(r domain.Route) {
1792 defer func() { <-sem }()
1793
1794 log.Info().Str("domain", r.Domain).Str("image", r.Image).Msg("auto-starting container for route")
1795
1796 if _, err := s.Deploy(deployCtx, r); err != nil {
1797 results <- result{route: r, err: err}
1798 return
1799 }
1800 results <- result{route: r}
1801 }(route)
1802 }
1803
1804 var started, errCount int
1805 for i := 0; i < len(pending); i++ {
1806 res := <-results
1807 if res.err != nil {
1808 log.Warn().Err(res.err).Str("domain", res.route.Domain).Msg("failed to auto-start container")
1809 errCount++
1810 } else {
1811 started++
1812 }
1813 }
1814
1815 log.Info().
1816 Int("started", started).
1817 Int("skipped", skipped).
1818 Int("errors", errCount).
1819 Msg("auto-start completed")
1820
1821 if errCount > 0 {
1822 return fmt.Errorf("auto-start completed with %d errors", errCount)
1823 }
1824 return nil
1825}
1826
1827// Shutdown gracefully shuts down all managed containers.
1828func (s *Service) Shutdown(ctx context.Context) error {
1829 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1830 zerowrap.FieldLayer: "usecase",
1831 zerowrap.FieldUseCase: "Shutdown",
1832 })
1833 log := zerowrap.FromCtx(ctx)
1834 log.Info().Msg("shutting down container manager...")
1835
1836 // Wait for any in-flight background cleanup goroutines to finish
1837 s.cleanupWg.Wait()
1838
1839 // Containers are left running across Gordon restarts.
1840 // SyncContainers + AutoStart will pick them back up on next boot.
1841
1842 // Close log writer to stop all log collection
1843 if s.logWriter != nil {
1844 if err := s.logWriter.Close(); err != nil {
1845 log.Warn().Err(err).Msg("failed to close container log writer")
1846 }
1847 }
1848
1849 log.Info().Msg("container manager shutdown complete")
1850 return nil
1851}
1852
1853// WaitForCleanup blocks until all background cleanup goroutines complete.
1854// Intended for use in tests to avoid mock assertion races.
1855func (s *Service) WaitForCleanup() {
1856 s.cleanupWg.Wait()
1857}
1858
1859// StartMonitor begins background monitoring of tracked containers.
1860// The monitor restarts crashed containers and detects crash loops.
1861// Safe to call multiple times; subsequent calls are no-ops.
1862func (s *Service) StartMonitor(ctx context.Context) {
1863 s.mu.Lock()
1864 if s.monitor != nil {
1865 s.mu.Unlock()
1866 return
1867 }
1868 s.monitor = newMonitor(s)
1869 m := s.monitor
1870 m.Start(ctx)
1871 s.mu.Unlock()
1872}
1873
1874// StopMonitor stops the background container monitor.
1875// Safe to call multiple times; subsequent calls are no-ops.
1876func (s *Service) StopMonitor() {
1877 s.mu.Lock()
1878 m := s.monitor
1879 s.monitor = nil
1880 s.mu.Unlock()
1881 if m != nil {
1882 m.Stop()
1883 }
1884}
1885
1886// UpdateConfig updates the service configuration.
1887func (s *Service) UpdateConfig(config Config) {
1888 s.mu.Lock()
1889 s.config = config
1890 s.mu.Unlock()
1891}
1892
1893// UpdateAttachments updates only the attachment configuration in the service.
1894// This is called after a config reload to propagate attachment changes without restart.
1895// The incoming map is deep-copied so external callers cannot mutate service state.
1896func (s *Service) UpdateAttachments(attachments map[string][]string) {
1897 var copied map[string][]string
1898 if attachments != nil {
1899 copied = make(map[string][]string, len(attachments))
1900 for k, v := range attachments {
1901 sl := make([]string, len(v))
1902 copy(sl, v)
1903 copied[k] = sl
1904 }
1905 }
1906 s.mu.Lock()
1907 s.config.Attachments = copied
1908 s.mu.Unlock()
1909}
1910
1911// Helper methods
1912
1913// cleanupFailedContainer stops and removes a container that failed to start properly.
1914func (s *Service) cleanupFailedContainer(ctx context.Context, containerID string) {
1915 log := zerowrap.FromCtx(ctx)
1916 if err := s.runtime.StopContainer(ctx, containerID); err != nil {
1917 log.Warn().Err(err).Str(zerowrap.FieldEntityID, containerID).Msg("failed to stop container after failure")
1918 }
1919 if err := s.runtime.RemoveContainer(ctx, containerID, true); err != nil {
1920 log.Warn().Err(err).Str(zerowrap.FieldEntityID, containerID).Msg("failed to remove container after failure")
1921 }
1922}
1923
1924// cleanupOldContainer stops and removes an old container after zero-downtime switch.
1925// It also renames the new container to the canonical name.
1926func (s *Service) cleanupOldContainer(ctx context.Context, old *domain.Container, newContainerID, domainName string) {
1927 log := zerowrap.FromCtx(ctx)
1928 log.Info().Str(zerowrap.FieldEntityID, old.ID).Msg("stopping old container after zero-downtime switch")
1929
1930 if s.logWriter != nil {
1931 if err := s.logWriter.StopLogging(old.ID); err != nil {
1932 log.Warn().Err(err).Str(zerowrap.FieldEntityID, old.ID).Msg("failed to stop logging for old container")
1933 }
1934 }
1935
1936 if err := s.runtime.StopContainer(ctx, old.ID); err != nil {
1937 log.Warn().Err(err).Str(zerowrap.FieldEntityID, old.ID).Msg("failed to stop old container")
1938 }
1939
1940 if err := s.runtime.RemoveContainer(ctx, old.ID, true); err != nil {
1941 log.Warn().Err(err).Str(zerowrap.FieldEntityID, old.ID).Msg("failed to remove old container")
1942 }
1943
1944 // Rename new container to canonical name
1945 canonicalName := fmt.Sprintf("gordon-%s", domainName)
1946 if err := s.runtime.RenameContainer(ctx, newContainerID, canonicalName); err != nil {
1947 log.Warn().Err(err).Str("canonical_name", canonicalName).Msg("failed to rename container to canonical name")
1948 }
1949}
1950
1951// startLogCollection starts log collection for a container in the background.
1952// Errors are logged but don't fail the calling operation.
1953func (s *Service) startLogCollection(ctx context.Context, containerID, domainName string) {
1954 if s.logWriter == nil {
1955 return
1956 }
1957
1958 log := zerowrap.FromCtx(ctx)
1959
1960 // Use context.WithoutCancel so the log stream outlives the HTTP request.
1961 // Without this, the Docker log stream closes when the deploy response completes.
1962 bgCtx := context.WithoutCancel(ctx)
1963
1964 logStream, err := s.runtime.GetContainerLogs(bgCtx, containerID, true)
1965 if err != nil {
1966 log.Warn().Err(err).Msg("failed to get container logs for collection")
1967 return
1968 }
1969
1970 if err := s.logWriter.StartLogging(bgCtx, containerID, domainName, logStream); err != nil {
1971 log.Warn().Err(err).Msg("failed to start container log collection")
1972 logStream.Close()
1973 }
1974}
1975
1976func (s *Service) buildValidatedImageRef(ctx context.Context, image string) (string, error) {
1977 imageRef := s.buildImageRef(image)
1978 if err := s.validateImageRef(ctx, imageRef); err != nil {
1979 return "", err
1980 }
1981 return imageRef, nil
1982}
1983
1984func (s *Service) buildImageRef(image string) string {
1985 s.mu.RLock()
1986 cfg := s.config
1987 s.mu.RUnlock()
1988
1989 if !cfg.RegistryAuthEnabled || cfg.RegistryDomain == "" {
1990 return image
1991 }
1992
1993 // Normalize registry domain by trimming any trailing slash
1994 reg := strings.TrimSuffix(cfg.RegistryDomain, "/")
1995
1996 // Check if image already has the registry domain prefix
1997 prefix := reg + "/"
1998 if strings.HasPrefix(image, prefix) {
1999 return image
2000 }
2001
2002 // Check if image already has an explicit registry (host:port/ or host.domain/).
2003 // We need to detect patterns like:
2004 // - "docker.io/library/nginx:latest" (has dot and slash)
2005 // - "localhost:5001/myapp:latest" (has host:port/ pattern)
2006 // - "myregistry:5000/app:v1" (has host:port/ pattern)
2007 // - "[fd00::1]:5000/app:v1" (IPv6 with port)
2008 if hasExplicitRegistry(image) {
2009 return image
2010 }
2011
2012 return fmt.Sprintf("%s/%s", reg, image)
2013}
2014
2015// hasExplicitRegistry checks if an image reference already includes an explicit registry.
2016// Returns true for patterns like "docker.io/image", "localhost:5000/image", "localhost/image",
2017// "registry:8080/image", "[fd00::1]/image", "[fd00::1]:5000/image".
2018func hasExplicitRegistry(image string) bool {
2019 // Find the first slash which separates registry from image name
2020 slashIdx := strings.Index(image, "/")
2021 if slashIdx == -1 {
2022 return false // No slash means no registry prefix (e.g., "myapp:latest")
2023 }
2024
2025 // Extract the part before the first slash (potential registry)
2026 registryPart := image[:slashIdx]
2027
2028 // Check for bracketed IPv6 address (e.g., "[fd00::1]" or "[fd00::1]:5000")
2029 if strings.HasPrefix(registryPart, "[") {
2030 // Look for closing bracket
2031 if closeBracket := strings.Index(registryPart, "]"); closeBracket != -1 {
2032 // Valid bracketed IPv6: either ends at ] or has ]:port
2033 return true
2034 }
2035 }
2036
2037 // Check for localhost (with or without port)
2038 if registryPart == "localhost" {
2039 return true
2040 }
2041
2042 // Check for host:port pattern (e.g., "localhost:5000", "registry:8080")
2043 if colonIdx := strings.LastIndex(registryPart, ":"); colonIdx != -1 {
2044 port := registryPart[colonIdx+1:]
2045 // If everything after colon is digits, it's a port
2046 if len(port) > 0 && isNumeric(port) {
2047 return true
2048 }
2049 }
2050
2051 // Check for domain pattern (has a dot, e.g., "docker.io", "gcr.io")
2052 if strings.Contains(registryPart, ".") {
2053 return true
2054 }
2055
2056 return false
2057}
2058
2059func (s *Service) validateImageRef(ctx context.Context, imageRef string) error {
2060 if !hasExplicitRegistry(imageRef) {
2061 return nil
2062 }
2063 return s.validateExternalImageRef(ctx, imageRef, imageRegistry(imageRef))
2064}
2065
2066func (s *Service) validateImagePullRef(ctx context.Context, imageRef string) error {
2067 return s.validateExternalImageRef(ctx, imageRef, imageRegistryForPolicy(imageRef))
2068}
2069
2070func (s *Service) validateExternalImageRef(ctx context.Context, imageRef, registry string) error {
2071 s.mu.RLock()
2072 cfg := s.config
2073 s.mu.RUnlock()
2074
2075 if registry == "" {
2076 return fmt.Errorf("image %q rejected: invalid image reference", imageRef)
2077 }
2078
2079 if domain.IsGordonRegistryImageRef(imageRef, cfg.RegistryDomain, cfg.LegacyRegistryDomains) || sameRegistry(registry, cfg.RegistryDomain) {
2080 return nil
2081 }
2082 dangerous, err := isDangerousRegistryHost(ctx, imageRegistryHost(registry))
2083 if err != nil {
2084 return fmt.Errorf("image %q rejected: registry %q could not be verified: %w", imageRef, registry, err)
2085 }
2086 if dangerous {
2087 return fmt.Errorf("image %q rejected: registry %q is not allowed", imageRef, registry)
2088 }
2089 if len(cfg.AllowedRegistries) == 0 {
2090 return fmt.Errorf("image %q rejected: registry %q is not in images.allowed_registries", imageRef, registry)
2091 }
2092 for _, allowed := range cfg.AllowedRegistries {
2093 if sameRegistry(registry, allowed) {
2094 if cfg.RequireImageDigest && !isValidSha256DigestRef(imageRef) {
2095 return fmt.Errorf("image %q rejected: digest reference required", imageRef)
2096 }
2097 return nil
2098 }
2099 }
2100 return fmt.Errorf("image %q rejected: registry %q is not in images.allowed_registries", imageRef, registry)
2101}
2102
2103func imageRegistry(imageRef string) string {
2104 slashIdx := strings.Index(imageRef, "/")
2105 if slashIdx == -1 {
2106 return ""
2107 }
2108 return normalizeRegistry(imageRef[:slashIdx])
2109}
2110
2111func imageRegistryForPolicy(imageRef string) string {
2112 if hasExplicitRegistry(imageRef) {
2113 return imageRegistry(imageRef)
2114 }
2115 return "docker.io"
2116}
2117
2118// normalizeRegistry normalizes a registry string for comparison purposes only.
2119// This function is used by sameRegistry to determine if two registry strings
2120// refer to the same registry. The output intentionally strips IPv6 brackets
2121// (e.g., "[fd00::1]:5000" -> "fd00::1:5000") which is acceptable for equality
2122// checks but the result MUST NOT be used to construct or emit valid registry URLs.
2123func normalizeRegistry(registry string) string {
2124 registry = strings.ToLower(strings.TrimSuffix(strings.TrimSpace(registry), "."))
2125 registry = strings.TrimSuffix(registry, "/")
2126 if strings.HasPrefix(registry, "[") {
2127 if end := strings.Index(registry, "]"); end != -1 {
2128 host := strings.Trim(registry[:end+1], "[]")
2129 port := registry[end+1:]
2130 return host + port
2131 }
2132 }
2133 return registry
2134}
2135
2136func imageRegistryHost(registry string) string {
2137 if ip := net.ParseIP(registry); ip != nil {
2138 return registry
2139 }
2140 if h, _, err := net.SplitHostPort(registry); err == nil {
2141 return strings.Trim(h, "[]")
2142 }
2143 if strings.HasPrefix(registry, "[") && strings.Contains(registry, "]") {
2144 return strings.TrimPrefix(strings.Split(registry, "]")[0], "[")
2145 }
2146 if colon := strings.LastIndex(registry, ":"); colon != -1 && isNumeric(registry[colon+1:]) {
2147 return registry[:colon]
2148 }
2149 return registry
2150}
2151
2152func sameRegistry(a, b string) bool {
2153 return normalizeRegistry(a) == normalizeRegistry(b)
2154}
2155
2156func isDangerousRegistryHost(ctx context.Context, host string) (bool, error) {
2157 host = strings.ToLower(strings.TrimSuffix(host, "."))
2158 if host == "localhost" || host == "metadata.google.internal" {
2159 return true, nil
2160 }
2161 if ip := net.ParseIP(host); ip != nil {
2162 return isDangerousRegistryIP(ip), nil
2163 }
2164 addrs, err := net.DefaultResolver.LookupIPAddr(ctx, host)
2165 if err != nil {
2166 return false, err
2167 }
2168 for _, addr := range addrs {
2169 if isDangerousRegistryIP(addr.IP) {
2170 return true, nil
2171 }
2172 }
2173 return false, nil
2174}
2175
2176func isDangerousRegistryIP(ip net.IP) bool {
2177 return ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsUnspecified()
2178}
2179
2180func applySecurityProfile(config *domain.ContainerConfig, cfg Config) {
2181 if strings.EqualFold(cfg.SecurityProfile, "strict") {
2182 config.ReadOnlyRootFS = true
2183 config.CapDrop = []string{"ALL"}
2184 config.CapAdd = []string{"NET_BIND_SERVICE"}
2185 }
2186}
2187
2188// isNumeric checks if a string contains only digits.
2189func isNumeric(s string) bool {
2190 for _, c := range s {
2191 if c < '0' || c > '9' {
2192 return false
2193 }
2194 }
2195 return true
2196}
2197
2198func normalizePullPolicy(policy string) string {
2199 switch strings.ToLower(strings.TrimSpace(policy)) {
2200 case PullPolicyAlways:
2201 return PullPolicyAlways
2202 case PullPolicyIfTagChanged:
2203 return PullPolicyIfTagChanged
2204 case PullPolicyIfNotPresent:
2205 return PullPolicyIfNotPresent
2206 default:
2207 return PullPolicyIfNotPresent
2208 }
2209}
2210
2211func isDigestRef(imageRef string) bool {
2212 return isValidSha256DigestRef(imageRef)
2213}
2214
2215func isValidSha256DigestRef(imageRef string) bool {
2216 _, digest, ok := strings.Cut(imageRef, "@sha256:")
2217 if !ok || len(digest) != 64 {
2218 return false
2219 }
2220 _, err := hex.DecodeString(digest)
2221 return err == nil
2222}
2223
2224func (s *Service) pullRefForDeploy(ctx context.Context, imageRef string) (string, bool) {
2225 if !domain.IsInternalDeploy(ctx) {
2226 return imageRef, false
2227 }
2228 s.mu.RLock()
2229 cfg := s.config
2230 s.mu.RUnlock()
2231 return rewriteToLocalRegistry(imageRef, cfg.RegistryDomain, cfg.LegacyRegistryDomains, cfg.RegistryPort), true
2232}
2233
2234// ensureImage ensures the image is available locally, pulling if needed.
2235// Returns the image reference to use for container operations:
2236// - For digest references (@sha256:...), returns the pullRef since Docker can't tag digests
2237// - For tagged images, returns the original imageRef after tagging the pulled image
2238func (s *Service) ensureImage(ctx context.Context, imageRef string) (string, error) {
2239 ctx, span := tracer.Start(ctx, "container.ensure_image",
2240 trace.WithAttributes(attribute.String("image", imageRef)))
2241 defer span.End()
2242
2243 ctx = zerowrap.CtxWithField(ctx, "image", imageRef)
2244 log := zerowrap.FromCtx(ctx)
2245
2246 // Determine if this is an internal deploy and what reference to use for pulls.
2247 pullRef, isInternal := s.pullRefForDeploy(ctx, imageRef)
2248 if err := s.validateImagePullRef(ctx, imageRef); err != nil {
2249 return "", err
2250 }
2251 if pullRef != imageRef {
2252 log.Info().
2253 Str("original_ref", imageRef).
2254 Str("pull_ref", pullRef).
2255 Msg("internal deploy: using localhost registry for pull")
2256 }
2257
2258 found, err := s.ensureLocalImage(ctx, imageRef, pullRef)
2259 if err != nil {
2260 return "", err
2261 }
2262 if found {
2263 return imageRef, nil
2264 }
2265
2266 // Pull image
2267 log.Info().Msg("pulling image from registry")
2268
2269 if err := s.pullImage(ctx, pullRef, isInternal); err != nil {
2270 return "", err
2271 }
2272
2273 // For digest references, we can't create a tag (Docker doesn't allow it).
2274 // In this case, use the pullRef directly since the image is available under that reference.
2275 if strings.Contains(imageRef, "@sha256:") {
2276 log.Info().
2277 Str("pull_ref", pullRef).
2278 Msg("digest reference: using pull reference for container operations")
2279 return pullRef, nil
2280 }
2281
2282 if err := s.tagImageIfNeeded(ctx, pullRef, imageRef); err != nil {
2283 return "", err
2284 }
2285
2286 // Clean up the temporary pull reference tag to avoid duplicate entries
2287 if pullRef != imageRef {
2288 if err := s.runtime.UntagImage(ctx, pullRef); err != nil {
2289 // Log but don't fail - the canonical tag is already applied
2290 log.Debug().Err(err).Str("pull_ref", pullRef).Msg("failed to remove temporary pull tag")
2291 }
2292 }
2293
2294 log.Info().Msg("image pulled successfully")
2295 return imageRef, nil
2296}
2297
2298func (s *Service) ensureLocalImage(ctx context.Context, imageRef, pullRef string) (bool, error) {
2299 log := zerowrap.FromCtx(ctx)
2300
2301 // For internal deploys (image push events), always pull fresh image
2302 // because the same tag may reference new content (e.g., latest tag updated)
2303 if domain.IsInternalDeploy(ctx) {
2304 log.Info().Msg("internal deploy detected, forcing image pull to ensure latest content")
2305 return false, nil
2306 }
2307
2308 s.mu.RLock()
2309 cfg := s.config
2310 s.mu.RUnlock()
2311
2312 pullPolicy := normalizePullPolicy(cfg.PullPolicy)
2313 switch pullPolicy {
2314 case PullPolicyAlways:
2315 log.Info().Str("pull_policy", pullPolicy).Msg("pull policy forces image pull")
2316 return false, nil
2317 case PullPolicyIfTagChanged:
2318 if !isDigestRef(imageRef) {
2319 log.Info().Str("pull_policy", pullPolicy).Msg("tag reference detected, pulling to check for updates")
2320 return false, nil
2321 }
2322 }
2323
2324 localImages, err := s.runtime.ListImages(ctx)
2325 if err != nil {
2326 log.WrapErr(err, "failed to list local images, will attempt pull")
2327 return false, nil
2328 }
2329
2330 normalizedRef := normalizeImageRef(imageRef)
2331 normalizedPullRef := normalizeImageRef(pullRef)
2332 for _, img := range localImages {
2333 normalizedImage := normalizeImageRef(img)
2334 if normalizedImage == normalizedRef {
2335 log.Info().Msg("image found locally, skipping pull")
2336 return true, nil
2337 }
2338 if normalizedImage == normalizedPullRef {
2339 if err := s.tagImageIfNeeded(ctx, pullRef, imageRef); err != nil {
2340 return false, err
2341 }
2342 // Clean up the temporary pull reference tag
2343 if pullRef != imageRef {
2344 if err := s.runtime.UntagImage(ctx, pullRef); err != nil {
2345 log.Debug().Err(err).Str("pull_ref", pullRef).Msg("failed to remove temporary pull tag")
2346 }
2347 }
2348 log.Info().Msg("image found locally, skipping pull")
2349 return true, nil
2350 }
2351 }
2352
2353 return false, nil
2354}
2355
2356func (s *Service) pullImage(ctx context.Context, pullRef string, isInternal bool) error {
2357 s.mu.RLock()
2358 cfg := s.config
2359 s.mu.RUnlock()
2360
2361 log := zerowrap.FromCtx(ctx)
2362
2363 switch {
2364 case isInternal && cfg.RegistryAuthEnabled:
2365 if cfg.InternalRegistryUsername == "" || cfg.InternalRegistryPassword == "" {
2366 return log.WrapErr(fmt.Errorf("internal registry auth not configured"), "failed to pull image for internal deploy")
2367 }
2368 if err := s.pullImageWithRetry(ctx, func(ctx context.Context) error {
2369 return s.runtime.PullImageWithAuth(ctx, pullRef, cfg.InternalRegistryUsername, cfg.InternalRegistryPassword)
2370 }, isInternal); err != nil {
2371 return log.WrapErr(fmt.Errorf("%w: %w", domain.ErrImagePullFailed, err), "failed to pull image with internal auth")
2372 }
2373 case isInternal:
2374 if err := s.pullImageWithRetry(ctx, func(ctx context.Context) error {
2375 return s.runtime.PullImage(ctx, pullRef)
2376 }, isInternal); err != nil {
2377 return log.WrapErr(fmt.Errorf("%w: %w", domain.ErrImagePullFailed, err), "failed to pull image")
2378 }
2379 case cfg.RegistryAuthEnabled:
2380 if cfg.ServiceTokenUsername == "" || cfg.ServiceToken == "" {
2381 return log.WrapErr(fmt.Errorf("registry service token not configured"), "failed to pull image for registry auth")
2382 }
2383 if err := s.runtime.PullImageWithAuth(ctx, pullRef, cfg.ServiceTokenUsername, cfg.ServiceToken); err != nil {
2384 return log.WrapErr(fmt.Errorf("%w: %w", domain.ErrImagePullFailed, err), "failed to pull image with auth")
2385 }
2386 default:
2387 if err := s.runtime.PullImage(ctx, pullRef); err != nil {
2388 return log.WrapErr(fmt.Errorf("%w: %w", domain.ErrImagePullFailed, err), "failed to pull image")
2389 }
2390 }
2391
2392 return nil
2393}
2394
2395func (s *Service) pullImageWithRetry(ctx context.Context, pullFn func(context.Context) error, isInternal bool) error {
2396 log := zerowrap.FromCtx(ctx)
2397
2398 for attempt := 1; attempt <= internalPullMaxAttempts; attempt++ {
2399 err := pullFn(ctx)
2400 if err == nil {
2401 return nil
2402 }
2403 if !isInternal || !isConnectionRefusedError(err) || attempt == internalPullMaxAttempts {
2404 return fmt.Errorf("internal image pull failed after %d attempts: %w", attempt, err)
2405 }
2406
2407 backoff := time.Duration(attempt) * time.Second
2408 log.Warn().
2409 Err(err).
2410 Int("attempt", attempt).
2411 Dur("backoff", backoff).
2412 Msg("internal image pull failed with connection refused, retrying")
2413
2414 select {
2415 case <-time.After(backoff):
2416 case <-ctx.Done():
2417 return ctx.Err()
2418 }
2419 }
2420 return nil
2421}
2422
2423func (s *Service) tagImageIfNeeded(ctx context.Context, sourceRef, targetRef string) error {
2424 if sourceRef == targetRef {
2425 return nil
2426 }
2427
2428 // Cannot create a tag with a digest reference - Docker/Podman doesn't allow it.
2429 // When using digest references (image@sha256:...), skip tagging as the image
2430 // is already available by its digest.
2431 if strings.Contains(targetRef, "@sha256:") {
2432 return nil
2433 }
2434
2435 log := zerowrap.FromCtx(ctx)
2436 if err := s.runtime.TagImage(ctx, sourceRef, targetRef); err != nil {
2437 return log.WrapErr(err, "failed to tag image from pull reference")
2438 }
2439 return nil
2440}
2441
2442func (s *Service) loadEnvironment(ctx context.Context, preResolved []string, domainName, imageRef string) ([]string, error) {
2443 log := zerowrap.FromCtx(ctx)
2444
2445 var userEnvVars []string
2446 if len(preResolved) > 0 {
2447 userEnvVars = preResolved
2448 } else {
2449 var err error
2450 userEnvVars, err = s.envLoader.LoadEnv(ctx, domainName)
2451 if err != nil {
2452 return nil, log.WrapErr(err, "failed to load environment variables")
2453 }
2454 }
2455
2456 dockerfileEnvVars, err := s.runtime.InspectImageEnv(ctx, imageRef)
2457 if err != nil {
2458 log.WrapErr(err, "failed to inspect image environment")
2459 dockerfileEnvVars = []string{}
2460 }
2461
2462 return mergeEnvironmentVariables(dockerfileEnvVars, userEnvVars), nil
2463}
2464
2465func (s *Service) setupVolumes(ctx context.Context, domainName, imageRef string) (map[string]string, error) {
2466 s.mu.RLock()
2467 cfg := s.config
2468 s.mu.RUnlock()
2469
2470 log := zerowrap.FromCtx(ctx)
2471 volumes := make(map[string]string)
2472
2473 if !cfg.VolumeAutoCreate {
2474 return volumes, nil
2475 }
2476
2477 volumePaths, err := s.runtime.InspectImageVolumes(ctx, imageRef)
2478 if err != nil {
2479 log.WrapErr(err, "failed to inspect image volumes")
2480 return volumes, nil
2481 }
2482
2483 for _, path := range volumePaths {
2484 name := generateVolumeName(cfg.VolumePrefix, domainName, path)
2485
2486 exists, err := s.runtime.VolumeExists(ctx, name)
2487 if err != nil {
2488 log.WrapErrWithFields(err, "failed to check volume", map[string]any{"volume": name})
2489 continue
2490 }
2491
2492 if !exists {
2493 if err := s.runtime.CreateVolume(ctx, name); err != nil {
2494 log.WrapErrWithFields(err, "failed to create volume", map[string]any{"volume": name})
2495 continue
2496 }
2497 log.Info().Str("volume", name).Str(zerowrap.FieldPath, path).Msg("created volume")
2498 }
2499
2500 volumes[path] = name
2501 }
2502
2503 return volumes, nil
2504}
2505
2506func (s *Service) getNetworkForApp(domainName string) string {
2507 var networkIsolation bool
2508 var networkGroups map[string][]string
2509
2510 if s.configProvider != nil {
2511 snap := s.configProvider.GetAttachmentConfig()
2512 networkGroups = snap.NetworkGroups
2513 s.mu.RLock()
2514 networkIsolation = s.config.NetworkIsolation
2515 s.mu.RUnlock()
2516 } else {
2517 s.mu.RLock()
2518 networkIsolation = s.config.NetworkIsolation
2519 networkGroups = s.config.NetworkGroups
2520 s.mu.RUnlock()
2521 }
2522
2523 if !networkIsolation {
2524 return "bridge"
2525 }
2526
2527 for groupName, domains := range networkGroups {
2528 if slices.Contains(domains, domainName) {
2529 return s.generateNetworkName(groupName)
2530 }
2531 }
2532
2533 return s.generateNetworkName(domainName)
2534}
2535
2536func (s *Service) generateNetworkName(identifier string) string {
2537 s.mu.RLock()
2538 cfg := s.config
2539 s.mu.RUnlock()
2540
2541 return fmt.Sprintf("%s-%s", cfg.NetworkPrefix, strings.ReplaceAll(identifier, ".", "-"))
2542}
2543
2544func (s *Service) createNetworkIfNeeded(ctx context.Context, networkName string) error {
2545 if networkName == "bridge" || networkName == "default" {
2546 return nil
2547 }
2548
2549 ctx = zerowrap.CtxWithField(ctx, "network", networkName)
2550 log := zerowrap.FromCtx(ctx)
2551
2552 exists, err := s.runtime.NetworkExists(ctx, networkName)
2553 if err != nil {
2554 return log.WrapErr(err, "failed to check network existence")
2555 }
2556
2557 if !exists {
2558 s.mu.RLock()
2559 internal := s.config.NetworkInternal
2560 s.mu.RUnlock()
2561 if err := s.runtime.CreateNetwork(ctx, networkName, domain.NetworkConfig{Driver: "bridge", Internal: internal}); err != nil {
2562 return log.WrapErr(err, "failed to create network")
2563 }
2564 log.Info().Msg("created network for app isolation")
2565 }
2566
2567 return nil
2568}
2569
2570func (s *Service) cleanupOrphanedContainers(ctx context.Context, domainName string, skipContainerID string) error {
2571 log := zerowrap.FromCtx(ctx)
2572 expectedName := fmt.Sprintf("gordon-%s", domainName)
2573 expectedNewName := expectedName + "-new"
2574 expectedNextName := expectedName + "-next"
2575
2576 allContainers, err := s.runtime.ListContainers(ctx, true)
2577 if err != nil {
2578 return err
2579 }
2580
2581 for _, c := range allContainers {
2582 if (c.Name == expectedName || c.Name == expectedNewName || c.Name == expectedNextName) && c.ID != skipContainerID {
2583 // Skip any running container regardless of name — a running
2584 // temp container may be actively serving traffic while the
2585 // new container stabilizes, or may be from a concurrent deploy.
2586 if c.Status == "running" {
2587 log.Debug().
2588 Str(zerowrap.FieldEntityID, c.ID).
2589 Str("container_name", c.Name).
2590 Str(zerowrap.FieldStatus, c.Status).
2591 Msg("skipping running container during orphan cleanup")
2592 continue
2593 }
2594
2595 log.Info().Str(zerowrap.FieldEntityID, c.ID).Str("container_name", c.Name).Str(zerowrap.FieldStatus, c.Status).Msg("found orphaned container, removing")
2596
2597 if err := s.runtime.StopContainer(ctx, c.ID); err != nil {
2598 log.WrapErrWithFields(err, "failed to stop orphaned container", map[string]any{zerowrap.FieldEntityID: c.ID})
2599 }
2600
2601 if err := s.runtime.RemoveContainer(ctx, c.ID, true); err != nil {
2602 log.WrapErrWithFields(err, "failed to remove orphaned container", map[string]any{zerowrap.FieldEntityID: c.ID})
2603 }
2604 }
2605 }
2606
2607 return nil
2608}
2609
2610func (s *Service) cleanupVolumesForDomain(_ context.Context, _ string) error {
2611 return nil
2612}
2613
2614func (s *Service) cleanupNetworkIfEmpty(ctx context.Context, networkName string) error {
2615 if networkName == "bridge" || networkName == "default" {
2616 return nil
2617 }
2618
2619 ctx = zerowrap.CtxWithField(ctx, "network", networkName)
2620 log := zerowrap.FromCtx(ctx)
2621
2622 networks, err := s.runtime.ListNetworks(ctx)
2623 if err != nil {
2624 return log.WrapErr(err, "failed to list networks")
2625 }
2626
2627 for _, n := range networks {
2628 if n.Name == networkName && len(n.Containers) == 0 {
2629 if err := s.runtime.RemoveNetwork(ctx, networkName); err != nil {
2630 return err
2631 }
2632 log.Info().Msg("cleaned up empty network")
2633 break
2634 }
2635 }
2636
2637 return nil
2638}
2639
2640func (s *Service) deployAttachments(ctx context.Context, domainName, networkName string) error {
2641 // Collect attachments from both domain-specific and network group configs
2642 attachments := s.resolveAttachmentsForDomain(domainName)
2643 if len(attachments) == 0 {
2644 return nil
2645 }
2646
2647 log := zerowrap.FromCtx(ctx)
2648 var deployed []string // track successfully deployed attachment IDs for rollback
2649 for _, svc := range attachments {
2650 if err := s.deployAttachedService(ctx, domainName, svc, networkName); err != nil {
2651 log.WrapErrWithFields(err, "failed to deploy attachment", map[string]any{zerowrap.FieldService: svc, "domain": domainName})
2652
2653 // Rollback: clean up already-deployed attachments
2654 for _, id := range deployed {
2655 if stopErr := s.runtime.StopContainer(ctx, id); stopErr != nil {
2656 log.Warn().Err(stopErr).Str("attachment_id", id).Msg("rollback: failed to stop attachment")
2657 }
2658 if rmErr := s.runtime.RemoveContainer(ctx, id, true); rmErr != nil {
2659 log.Warn().Err(rmErr).Str("attachment_id", id).Msg("rollback: failed to remove attachment")
2660 }
2661 }
2662 // Deregister rolled-back attachments
2663 if len(deployed) > 0 {
2664 s.mu.Lock()
2665 remaining := s.attachments[domainName]
2666 filtered := remaining[:0]
2667 rollbackSet := make(map[string]bool, len(deployed))
2668 for _, id := range deployed {
2669 rollbackSet[id] = true
2670 }
2671 for _, id := range remaining {
2672 if !rollbackSet[id] {
2673 filtered = append(filtered, id)
2674 }
2675 }
2676 s.attachments[domainName] = filtered
2677 s.mu.Unlock()
2678 }
2679
2680 return fmt.Errorf("failed to deploy attachment %q (rolled back %d already-deployed)", svc, len(deployed))
2681 }
2682 // Collect IDs of attachments tracked under this domain after successful deploy
2683 s.mu.RLock()
2684 ids := s.attachments[domainName]
2685 if len(ids) > 0 {
2686 latest := ids[len(ids)-1]
2687 deployed = append(deployed, latest)
2688 }
2689 s.mu.RUnlock()
2690 }
2691 return nil
2692}
2693
2694// resolveAttachmentsForDomain returns attachments for a domain by checking:
2695// 1. Direct domain attachments (attachments[domain])
2696// 2. Network group attachments (attachments[group] where domain is in network_groups[group])
2697func (s *Service) resolveAttachmentsForDomain(domainName string) []string {
2698 var attachments map[string][]string
2699 var networkGroups map[string][]string
2700
2701 if s.configProvider != nil {
2702 snap := s.configProvider.GetAttachmentConfig()
2703 attachments = snap.Attachments
2704 networkGroups = snap.NetworkGroups
2705 } else {
2706 s.mu.RLock()
2707 attachments = s.config.Attachments
2708 networkGroups = s.config.NetworkGroups
2709 s.mu.RUnlock()
2710 }
2711
2712 seen := make(map[string]bool)
2713 var result []string
2714
2715 // First, add domain-specific attachments
2716 if domainAttachments, ok := attachments[domainName]; ok {
2717 for _, img := range domainAttachments {
2718 if !seen[img] {
2719 seen[img] = true
2720 result = append(result, img)
2721 }
2722 }
2723 }
2724
2725 // Then, find which network group this domain belongs to and add group attachments
2726 for groupName, domains := range networkGroups {
2727 if slices.Contains(domains, domainName) {
2728 if groupAttachments, ok := attachments[groupName]; ok {
2729 for _, img := range groupAttachments {
2730 if !seen[img] {
2731 seen[img] = true
2732 result = append(result, img)
2733 }
2734 }
2735 }
2736 break // Domain can only be in one network group
2737 }
2738 }
2739
2740 return result
2741}
2742
2743func (s *Service) getAttachmentsForDomain(ctx context.Context, domainName string) []domain.Attachment {
2744 containers, err := s.runtime.ListContainers(ctx, true)
2745 if err != nil {
2746 return nil
2747 }
2748
2749 return s.filterAttachments(ctx, containers, domainName)
2750}
2751
2752// getAllAttachments fetches all containers once and returns attachments grouped by domain.
2753// This avoids N+1 queries when listing multiple routes.
2754func (s *Service) getAllAttachments(ctx context.Context) map[string][]domain.Attachment {
2755 containers, err := s.runtime.ListContainers(ctx, true)
2756 if err != nil {
2757 return nil
2758 }
2759
2760 result := make(map[string][]domain.Attachment)
2761 for _, container := range containers {
2762 if container.Labels == nil {
2763 continue
2764 }
2765 if container.Labels[domain.LabelAttachment] != "true" {
2766 continue
2767 }
2768 ownerDomain := container.Labels[domain.LabelAttachedTo]
2769 if ownerDomain == "" {
2770 continue
2771 }
2772 image := container.Image
2773 serviceName := container.Name
2774 if labelImage, ok := container.Labels[domain.LabelImage]; ok && labelImage != "" {
2775 image = labelImage
2776 serviceName = extractServiceName(labelImage)
2777 }
2778 // Get the container's network for display in the routes table
2779 network := ""
2780 if networkName, err := s.runtime.GetContainerNetwork(ctx, container.ID); err == nil {
2781 network = networkName
2782 }
2783 attachment := domain.Attachment{
2784 Name: serviceName,
2785 Image: s.stripRegistryPrefix(image),
2786 ContainerID: container.ID,
2787 Status: container.Status,
2788 Network: network,
2789 Ports: append([]int(nil), container.Ports...),
2790 }
2791 result[ownerDomain] = append(result[ownerDomain], attachment)
2792 }
2793
2794 return result
2795}
2796
2797// filterAttachments extracts attachments for a specific domain from a container list.
2798func (s *Service) filterAttachments(ctx context.Context, containers []*domain.Container, domainName string) []domain.Attachment {
2799 attachments := make([]domain.Attachment, 0)
2800 for _, container := range containers {
2801 if container.Labels == nil {
2802 continue
2803 }
2804 if container.Labels[domain.LabelAttachment] != "true" {
2805 continue
2806 }
2807 if container.Labels[domain.LabelAttachedTo] != domainName {
2808 continue
2809 }
2810 image := container.Image
2811 serviceName := container.Name
2812 if labelImage, ok := container.Labels[domain.LabelImage]; ok && labelImage != "" {
2813 image = labelImage
2814 serviceName = extractServiceName(labelImage)
2815 }
2816 // Get the container's network for display in the routes table
2817 network := ""
2818 if networkName, err := s.runtime.GetContainerNetwork(ctx, container.ID); err == nil {
2819 network = networkName
2820 }
2821 attachment := domain.Attachment{
2822 Name: serviceName,
2823 Image: s.stripRegistryPrefix(image),
2824 ContainerID: container.ID,
2825 Status: container.Status,
2826 Network: network,
2827 Ports: append([]int(nil), container.Ports...),
2828 }
2829 attachments = append(attachments, attachment)
2830 }
2831
2832 return attachments
2833}
2834
2835func (s *Service) deployAttachedService(ctx context.Context, ownerDomain, serviceImage, networkName string) error {
2836 log := zerowrap.FromCtx(ctx)
2837
2838 // Parse service name from image (e.g., "my-postgres:latest" → "postgres")
2839 serviceName := extractServiceName(serviceImage)
2840 containerName := fmt.Sprintf("gordon-%s-%s", sanitizeName(ownerDomain), serviceName)
2841
2842 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
2843 "attachment": serviceName,
2844 "container_name": containerName,
2845 "owner_domain": ownerDomain,
2846 })
2847
2848 // Check if already running (idempotent)
2849 // Try new name first, then fall back to legacy name for backwards compatibility
2850 existingContainer, err := s.resolveExistingAttachment(ctx, ownerDomain, serviceName)
2851 if err != nil {
2852 return err
2853 }
2854
2855 if existingContainer != nil {
2856 shouldSkip, err := s.handleRunningAttachment(ctx, existingContainer, containerName, serviceImage)
2857 if err != nil {
2858 return err
2859 }
2860 if shouldSkip {
2861 return nil
2862 }
2863 if existingContainer.Status == string(domain.ContainerStatusRunning) {
2864 existingContainer = nil
2865 }
2866 }
2867
2868 if err := s.removeStoppedAttachment(ctx, existingContainer, containerName); err != nil {
2869 return err
2870 }
2871
2872 log.Info().Str(zerowrap.FieldService, serviceImage).Msg("deploying attached service")
2873
2874 config, err := s.buildAttachmentContainerConfig(ctx, ownerDomain, serviceName, serviceImage, containerName, networkName)
2875 if err != nil {
2876 return err
2877 }
2878
2879 container, err := s.runtime.CreateContainer(ctx, config)
2880 if err != nil {
2881 return log.WrapErr(err, "failed to create attachment container")
2882 }
2883
2884 // Start container
2885 if err := s.runtime.StartContainer(ctx, container.ID); err != nil {
2886 s.runtime.RemoveContainer(ctx, container.ID, true)
2887 return log.WrapErr(err, "failed to start attachment container")
2888 }
2889
2890 // Wait for attachment to be ready before proceeding
2891 if err := s.waitForAttachmentReady(ctx, container.ID, config); err != nil {
2892 log.WrapErr(err, "attachment readiness check failed, cleaning up")
2893 if stopErr := s.runtime.StopContainer(ctx, container.ID); stopErr != nil {
2894 log.Warn().Err(stopErr).Str(zerowrap.FieldEntityID, container.ID).Msg("failed to stop unready attachment")
2895 }
2896 s.runtime.RemoveContainer(ctx, container.ID, true)
2897 return fmt.Errorf("attachment %q not ready: %w", serviceName, err)
2898 }
2899 // Track attachment
2900 s.mu.Lock()
2901 s.attachments[ownerDomain] = append(s.attachments[ownerDomain], container.ID)
2902 s.mu.Unlock()
2903
2904 // Start log collection for attachment
2905 s.startLogCollection(ctx, container.ID, containerName)
2906
2907 log.Info().Str(zerowrap.FieldEntityID, container.ID).Msg("attachment deployed successfully")
2908 return nil
2909}
2910
2911func (s *Service) buildAttachmentContainerConfig(ctx context.Context, ownerDomain, serviceName, serviceImage, containerName, networkName string) (*domain.ContainerConfig, error) {
2912 imageRef, err := s.buildValidatedImageRef(ctx, serviceImage)
2913 if err != nil {
2914 return nil, err
2915 }
2916 actualImageRef, err := s.ensureImage(ctx, imageRef)
2917 if err != nil {
2918 return nil, err
2919 }
2920
2921 exposedPorts := s.attachmentExposedPorts(ctx, actualImageRef)
2922 volumes, err := s.attachmentVolumes(ctx, containerName, actualImageRef)
2923 if err != nil {
2924 return nil, err
2925 }
2926 envVars, err := s.attachmentEnv(ctx, containerName, actualImageRef)
2927 if err != nil {
2928 return nil, err
2929 }
2930
2931 s.mu.RLock()
2932 cfg := s.config
2933 s.mu.RUnlock()
2934
2935 config := &domain.ContainerConfig{
2936 Image: actualImageRef,
2937 Name: containerName,
2938 Hostname: serviceName,
2939 Aliases: []string{serviceName},
2940 Ports: exposedPorts,
2941 Env: envVars,
2942 Volumes: volumes,
2943 NetworkMode: networkName,
2944 Labels: map[string]string{
2945 domain.LabelManaged: "true",
2946 domain.LabelAttachment: "true",
2947 domain.LabelAttachedTo: ownerDomain,
2948 domain.LabelEnvHash: hashEnvironment(envVars),
2949 domain.LabelImage: serviceImage,
2950 },
2951 MemoryLimit: cfg.DefaultMemoryLimit,
2952 NanoCPUs: cfg.DefaultNanoCPUs,
2953 PidsLimit: cfg.DefaultPidsLimit,
2954 RestartPolicy: domain.RestartPolicyAlways,
2955 }
2956 applySecurityProfile(config, cfg)
2957 return config, nil
2958}
2959
2960func (s *Service) attachmentExposedPorts(ctx context.Context, imageRef string) []int {
2961 log := zerowrap.FromCtx(ctx)
2962 exposedPorts, err := s.runtime.GetImageExposedPorts(ctx, imageRef)
2963 if err != nil {
2964 log.WrapErr(err, "failed to get exposed ports for attachment, using defaults")
2965 return []int{}
2966 }
2967 return exposedPorts
2968}
2969
2970func (s *Service) attachmentVolumes(ctx context.Context, containerName, imageRef string) (map[string]string, error) {
2971 volumes, err := s.setupVolumes(ctx, containerName, imageRef)
2972 if err != nil {
2973 return nil, fmt.Errorf("failed to setup volumes for attachment %s with image %s: %w", containerName, imageRef, err)
2974 }
2975 return volumes, nil
2976}
2977
2978func (s *Service) attachmentEnv(ctx context.Context, containerName, imageRef string) ([]string, error) {
2979 envVars, err := s.loadEnvironment(ctx, nil, containerName, imageRef)
2980 if err != nil {
2981 return nil, fmt.Errorf("failed to load environment for attachment %s with image %s: %w", containerName, imageRef, err)
2982 }
2983 return envVars, nil
2984}
2985
2986func (s *Service) attachmentEnvDrifted(ctx context.Context, existing *domain.Container, containerName, serviceImage string) (bool, error) {
2987 imageRef, err := s.buildValidatedImageRef(ctx, serviceImage)
2988 if err != nil {
2989 return false, err
2990 }
2991 currentEnv, err := s.loadEnvironment(ctx, nil, containerName, imageRef)
2992 if err != nil {
2993 return false, err
2994 }
2995
2996 currentHash := hashEnvironment(currentEnv)
2997 existingHash, ok := existing.Labels[domain.LabelEnvHash]
2998 if ok && existingHash == currentHash {
2999 return false, nil
3000 }
3001
3002 return true, nil
3003}
3004
3005func (s *Service) handleRunningAttachment(ctx context.Context, existing *domain.Container, containerName, serviceImage string) (bool, error) {
3006 if existing.Status != string(domain.ContainerStatusRunning) {
3007 return false, nil
3008 }
3009
3010 log := zerowrap.FromCtx(ctx)
3011
3012 // Check image drift
3013 existingImage := existing.Labels[domain.LabelImage]
3014 imageDrifted := existingImage != "" && existingImage != serviceImage
3015
3016 // Check env drift
3017 envDrifted, err := s.attachmentEnvDrifted(ctx, existing, containerName, serviceImage)
3018 if err != nil {
3019 return false, log.WrapErr(err, "failed to check attachment env drift")
3020 }
3021
3022 if !envDrifted && !imageDrifted {
3023 log.Debug().Str("container_name", containerName).Msg("attachment already running with current env and image, skipping")
3024 return true, nil
3025 }
3026
3027 if imageDrifted {
3028 log.Info().Str("container_name", containerName).Str("existing_image", existingImage).Str("desired_image", serviceImage).Msg("attachment image changed, recreating")
3029 }
3030 if envDrifted {
3031 log.Info().Str("container_name", containerName).Msg("attachment env changed, recreating")
3032 }
3033
3034 if err := s.runtime.StopContainer(ctx, existing.ID); err != nil {
3035 return false, log.WrapErr(err, "failed to stop attachment for drift update")
3036 }
3037 if err := s.runtime.RemoveContainer(ctx, existing.ID, true); err != nil {
3038 return false, log.WrapErr(err, "failed to remove attachment for drift update")
3039 }
3040
3041 return false, nil
3042}
3043
3044func (s *Service) removeStoppedAttachment(ctx context.Context, existing *domain.Container, containerName string) error {
3045 if existing == nil {
3046 return nil
3047 }
3048
3049 log := zerowrap.FromCtx(ctx)
3050 log.Info().Str("container_name", containerName).Msg("removing stopped attachment container")
3051 if err := s.runtime.RemoveContainer(ctx, existing.ID, true); err != nil {
3052 return log.WrapErr(err, "failed to remove existing attachment container")
3053 }
3054
3055 return nil
3056}
3057
3058func (s *Service) resolveExistingAttachment(ctx context.Context, ownerDomain, serviceName string) (*domain.Container, error) {
3059 log := zerowrap.FromCtx(ctx)
3060 containerName := fmt.Sprintf("gordon-%s-%s", sanitizeName(ownerDomain), serviceName)
3061 existingContainer := s.findContainerByName(ctx, containerName)
3062 if existingContainer != nil {
3063 return existingContainer, nil
3064 }
3065
3066 containerNameLegacy := fmt.Sprintf("gordon-%s-%s", sanitizeNameLegacy(ownerDomain), serviceName)
3067 existingContainer = s.findContainerByName(ctx, containerNameLegacy)
3068 if existingContainer == nil {
3069 return nil, nil
3070 }
3071
3072 log.Info().Str("container_name", containerNameLegacy).Msg("found attachment with legacy naming, will be replaced")
3073 if err := s.runtime.StopContainer(ctx, existingContainer.ID); err != nil {
3074 return nil, log.WrapErr(err, "failed to stop legacy attachment container")
3075 }
3076 if err := s.runtime.RemoveContainer(ctx, existingContainer.ID, true); err != nil {
3077 return nil, log.WrapErr(err, "failed to remove legacy attachment container")
3078 }
3079
3080 return nil, nil
3081}
3082
3083// Utility functions
3084
3085func normalizeImageRef(image string) string {
3086 if isDigestRef(image) {
3087 return image
3088 }
3089
3090 // Extract repository from image reference, handling port numbers correctly.
3091 // Examples:
3092 // nginx:latest -> docker.io/library/nginx
3093 // user/repo:tag -> docker.io/user/repo
3094 // localhost:5000/image:tag -> localhost:5000/image
3095 // registry.example.com/image:tag -> registry.example.com/image
3096 //
3097 // Note: This function assumes valid Docker image references. Edge cases like
3098 // "registry.com:5000" (registry with port but no image name) are not valid
3099 // image references per Docker's naming conventions, which require at least
3100 // one path component after the registry (e.g., "registry.com:5000/image").
3101
3102 // Find the tag separator: the last colon that isn't part of a port number.
3103 // A colon is part of a port if there's no slash after it until the next colon.
3104 repo := image
3105 lastColon := strings.LastIndex(image, ":")
3106 if lastColon != -1 {
3107 afterColon := image[lastColon+1:]
3108 // If there's no slash after the colon, it's the tag separator
3109 if !strings.Contains(afterColon, "/") {
3110 repo = image[:lastColon]
3111 }
3112 }
3113
3114 if !strings.Contains(repo, "/") {
3115 return "docker.io/library/" + repo
3116 }
3117
3118 if strings.Count(repo, "/") == 1 && !strings.Contains(strings.Split(repo, "/")[0], ".") && !strings.Contains(strings.Split(repo, "/")[0], ":") {
3119 return "docker.io/" + repo
3120 }
3121
3122 return repo
3123}
3124
3125func generateVolumeName(prefix, domainName, volumePath string) string {
3126 return fmt.Sprintf("%s-%s-%s",
3127 prefix,
3128 strings.ReplaceAll(domainName, ".", "-"),
3129 strings.ReplaceAll(strings.Trim(volumePath, "/"), "/", "-"))
3130}
3131
3132func mergeEnvironmentVariables(dockerfileEnv, userEnv []string) []string {
3133 envMap := make(map[string]string)
3134
3135 for _, env := range dockerfileEnv {
3136 if k, v, ok := strings.Cut(env, "="); ok {
3137 envMap[k] = v
3138 }
3139 }
3140
3141 for _, env := range userEnv {
3142 if k, v, ok := strings.Cut(env, "="); ok {
3143 envMap[k] = v
3144 }
3145 }
3146
3147 keys := make([]string, 0, len(envMap))
3148 for k := range envMap {
3149 keys = append(keys, k)
3150 }
3151 slices.Sort(keys)
3152
3153 result := make([]string, 0, len(envMap))
3154 for _, k := range keys {
3155 result = append(result, k+"="+envMap[k])
3156 }
3157
3158 return result
3159}
3160
3161// extractServiceName gets service name from image reference.
3162// "my-postgres:latest" → "postgres", "redis:7" → "redis"
3163func extractServiceName(image string) string {
3164 // Remove tag
3165 parts := strings.Split(image, ":")
3166 name := parts[0]
3167
3168 // Remove registry prefix if present
3169 if strings.Contains(name, "/") {
3170 nameParts := strings.Split(name, "/")
3171 name = nameParts[len(nameParts)-1]
3172 }
3173
3174 // Remove common prefixes like "my-"
3175 name = strings.TrimPrefix(name, "my-")
3176
3177 return name
3178}
3179
3180// sanitizeName is a convenience wrapper around domain.SanitizeDomainForContainer.
3181func sanitizeName(d string) string {
3182 return domain.SanitizeDomainForContainer(d)
3183}
3184
3185// sanitizeNameLegacy is a convenience wrapper around domain.SanitizeDomainForContainerLegacy.
3186func sanitizeNameLegacy(d string) string {
3187 return domain.SanitizeDomainForContainerLegacy(d)
3188}
3189
3190func isContainerNotFoundError(err error) bool {
3191 if err == nil {
3192 return false
3193 }
3194 if errors.Is(err, domain.ErrContainerNotFound) {
3195 return true
3196 }
3197 msg := strings.ToLower(err.Error())
3198 return strings.Contains(msg, "no such container") ||
3199 strings.Contains(msg, "no container with name or id") ||
3200 strings.Contains(msg, "container not found")
3201}
3202
3203func isConnectionRefusedError(err error) bool {
3204 if err == nil {
3205 return false
3206 }
3207 return strings.Contains(strings.ToLower(err.Error()), "connection refused")
3208}
3209
3210// findContainerByName finds a container by its name.
3211func (s *Service) findContainerByName(ctx context.Context, name string) *domain.Container {
3212 containers, err := s.runtime.ListContainers(ctx, true)
3213 if err != nil {
3214 return nil
3215 }
3216
3217 for _, c := range containers {
3218 if c.Name == name {
3219 return c
3220 }
3221 }
3222 return nil
3223}
3224
3225func (s *Service) waitForReady(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig) error {
3226 if err := s.pollContainerRunning(ctx, containerID); err != nil {
3227 return err
3228 }
3229
3230 s.mu.RLock()
3231 cfg := s.config
3232 s.mu.RUnlock()
3233
3234 readinessMode := cfg.ReadinessMode
3235 if readinessMode == "" {
3236 readinessMode = "delay"
3237 }
3238
3239 switch readinessMode {
3240 case "delay":
3241 return s.waitForReadyByDelay(ctx, containerID)
3242 case "docker-health":
3243 _, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(ctx, containerID)
3244 if err != nil {
3245 return err
3246 }
3247 if !hasHealthcheck {
3248 return errors.New("no healthcheck detected")
3249 }
3250 return s.waitForHealthy(ctx, containerID, cfg.HealthTimeout)
3251 default: // auto
3252 return s.readinessCascade(ctx, containerID, containerConfig, cfg)
3253 }
3254}
3255
3256// readinessCascade auto-detects the strongest available readiness signal:
3257// 1. Docker healthcheck (if present) → wait for healthy status
3258// 2. HTTP probe (if gordon.health label set) → GET until 2xx/3xx
3259// 3. Default HTTP probe (GET / on exposed port) → wait for 2xx/3xx
3260// 4. TCP probe (if port info available) → connect until accepted
3261// 5. Delay fallback (last resort) → waitForReadyByDelay
3262func (s *Service) readinessCascade(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) error {
3263 log := zerowrap.FromCtx(ctx)
3264
3265 // 1. Docker healthcheck
3266 _, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(ctx, containerID)
3267 if err != nil {
3268 return err
3269 }
3270 if hasHealthcheck {
3271 log.Info().Msg("readiness cascade: using Docker healthcheck")
3272 return s.waitForHealthy(ctx, containerID, cfg.HealthTimeout)
3273 }
3274
3275 // 2. HTTP probe via gordon.health label
3276 if probed, probeErr := s.tryHTTPProbe(ctx, containerID, containerConfig, cfg); probed {
3277 return probeErr
3278 }
3279
3280 // 3. Default HTTP probe on root path (if port info available)
3281 if probed, probeErr := s.tryDefaultHTTPProbe(ctx, containerID, containerConfig, cfg); probed {
3282 return probeErr
3283 }
3284
3285 // 4. TCP probe (if port info available)
3286 if probed, probeErr := s.tryTCPProbe(ctx, containerID, containerConfig, cfg); probed {
3287 return probeErr
3288 }
3289
3290 // 5. Delay fallback
3291 log.Info().Msg("readiness cascade: using delay fallback")
3292 return s.waitForReadyByDelay(ctx, containerID)
3293}
3294
3295// waitForAttachmentReady runs a reduced readiness cascade for attachments:
3296// Docker healthcheck → TCP probe → delay fallback.
3297// HTTP probes are skipped — attachments are typically databases/caches.
3298func (s *Service) waitForAttachmentReady(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig) error {
3299 if err := s.pollContainerRunning(ctx, containerID); err != nil {
3300 return err
3301 }
3302
3303 log := zerowrap.FromCtx(ctx)
3304
3305 s.mu.RLock()
3306 cfg := s.config
3307 s.mu.RUnlock()
3308
3309 // 1. Docker healthcheck (if present)
3310 _, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(ctx, containerID)
3311 if err != nil {
3312 return err
3313 }
3314 if hasHealthcheck {
3315 log.Info().Msg("attachment readiness: using Docker healthcheck")
3316 timeout := cfg.HealthTimeout
3317 if timeout == 0 {
3318 timeout = 90 * time.Second
3319 }
3320 return s.waitForHealthy(ctx, containerID, timeout)
3321 }
3322
3323 // 2. TCP probe (if port info available)
3324 if containerConfig != nil && len(containerConfig.Ports) > 0 {
3325 ip, port, probeErr := s.resolveProbeEndpoint(ctx, containerID, containerConfig)
3326 if probeErr == nil && ip != "" && port > 0 {
3327 addr := fmt.Sprintf("%s:%d", ip, port)
3328 timeout := cfg.AttachmentReadinessTimeout
3329 if timeout == 0 {
3330 timeout = 30 * time.Second
3331 }
3332 log.Info().Str("addr", addr).Dur("timeout", timeout).Msg("attachment readiness: using TCP probe")
3333 return tcpProbe(ctx, addr, timeout)
3334 }
3335 log.Debug().Err(probeErr).Msg("attachment readiness: TCP probe skipped, could not resolve endpoint")
3336 }
3337
3338 // 3. Delay fallback
3339 log.Info().Msg("attachment readiness: using delay fallback")
3340 return s.waitForReadyByDelay(ctx, containerID)
3341}
3342
3343// tryHTTPProbe attempts an HTTP probe if the gordon.health label is set.
3344// Returns (true, err) if the probe was attempted, (false, nil) if skipped.
3345func (s *Service) tryHTTPProbe(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) (bool, error) {
3346 log := zerowrap.FromCtx(ctx)
3347 if containerConfig == nil {
3348 return false, nil
3349 }
3350 healthPath, ok := containerConfig.Labels[domain.LabelHealth]
3351 if !ok || healthPath == "" {
3352 return false, nil
3353 }
3354 ip, port, probeErr := s.resolveProbeEndpoint(ctx, containerID, containerConfig)
3355 if probeErr != nil || ip == "" || port <= 0 {
3356 log.Debug().Err(probeErr).Msg("readiness cascade: HTTP probe skipped, could not resolve container endpoint")
3357 return false, nil
3358 }
3359 url := fmt.Sprintf("http://%s:%d%s", ip, port, healthPath)
3360 timeout := cfg.HTTPProbeTimeout
3361 if timeout == 0 {
3362 timeout = 60 * time.Second
3363 }
3364 log.Info().Str("url", url).Dur("timeout", timeout).Msg("readiness cascade: using HTTP probe")
3365 return true, httpProbe(ctx, url, timeout)
3366}
3367
3368// tryDefaultHTTPProbe attempts an HTTP GET on "/" using the container's
3369// exposed port. This is the default readiness check for web containers —
3370// it verifies the app is actually serving HTTP, not just accepting TCP.
3371// Returns (true, err) if the probe was attempted, (false, nil) if skipped.
3372func (s *Service) tryDefaultHTTPProbe(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) (bool, error) {
3373 log := zerowrap.FromCtx(ctx)
3374 if containerConfig == nil || len(containerConfig.Ports) == 0 {
3375 return false, nil
3376 }
3377 ip, port, probeErr := s.resolveProbeEndpoint(ctx, containerID, containerConfig)
3378 if probeErr != nil || ip == "" || port <= 0 {
3379 log.Debug().Err(probeErr).Msg("readiness cascade: default HTTP probe skipped, could not resolve container endpoint")
3380 return false, nil
3381 }
3382 url := fmt.Sprintf("http://%s:%d/", ip, port)
3383 timeout := cfg.HTTPProbeTimeout
3384 if timeout == 0 {
3385 timeout = 60 * time.Second
3386 }
3387 log.Info().Str("url", url).Dur("timeout", timeout).Msg("readiness cascade: using default HTTP alive probe")
3388 return true, httpAliveProbe(ctx, url, timeout)
3389}
3390
3391// tryTCPProbe attempts a TCP probe if port info is available.
3392// Returns (true, err) if the probe was attempted, (false, nil) if skipped.
3393func (s *Service) tryTCPProbe(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) (bool, error) {
3394 log := zerowrap.FromCtx(ctx)
3395 if containerConfig == nil || len(containerConfig.Ports) == 0 {
3396 return false, nil
3397 }
3398 ip, port, probeErr := s.resolveProbeEndpoint(ctx, containerID, containerConfig)
3399 if probeErr != nil || ip == "" || port <= 0 {
3400 log.Debug().Err(probeErr).Msg("readiness cascade: TCP probe skipped, could not resolve container endpoint")
3401 return false, nil
3402 }
3403 addr := fmt.Sprintf("%s:%d", ip, port)
3404 timeout := cfg.TCPProbeTimeout
3405 if timeout == 0 {
3406 timeout = 30 * time.Second
3407 }
3408 log.Info().Str("addr", addr).Dur("timeout", timeout).Msg("readiness cascade: using TCP probe")
3409 return true, tcpProbe(ctx, addr, timeout)
3410}
3411
3412// resolveContainerEndpoint returns a host-reachable address for probing.
3413// In rootless podman/Docker setups, container internal IPs are not routable
3414// from the host. We use the host port binding (127.0.0.1:<mapped_port>)
3415// which is always reachable. Falls back to the container's internal IP
3416// only if no host port mapping exists (e.g. host-network mode).
3417func (s *Service) resolveProbeEndpoint(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig) (string, int, error) {
3418 log := zerowrap.FromCtx(ctx)
3419
3420 internalPort := 0
3421 if containerConfig != nil {
3422 if labels := containerConfig.Labels; labels != nil {
3423 for _, key := range []string{domain.LabelProxyPort, domain.LabelPort} {
3424 if portStr, ok := labels[key]; ok && portStr != "" {
3425 port, err := strconv.Atoi(portStr)
3426 if err == nil && port > 0 && port <= 65535 {
3427 internalPort = port
3428 break
3429 }
3430 log.Warn().Str("label", key).Str("port_value", portStr).Msg("invalid probe port label value")
3431 }
3432 }
3433 }
3434 }
3435
3436 if internalPort == 0 {
3437 _, resolvedPort, err := s.runtime.GetContainerNetworkInfo(ctx, containerID)
3438 if err != nil {
3439 return "", 0, err
3440 }
3441 internalPort = resolvedPort
3442 }
3443
3444 hostPort, hostErr := s.runtime.GetContainerPort(ctx, containerID, internalPort)
3445 if hostErr == nil && hostPort > 0 {
3446 log.Debug().
3447 Int("internal_port", internalPort).
3448 Int("host_port", hostPort).
3449 Msg("resolved container endpoint via host port binding")
3450 return "127.0.0.1", hostPort, nil
3451 }
3452
3453 ip, _, fallbackErr := s.runtime.GetContainerNetworkInfo(ctx, containerID)
3454 if fallbackErr != nil {
3455 return "", 0, fallbackErr
3456 }
3457 log.Debug().
3458 Str("ip", ip).
3459 Int("port", internalPort).
3460 Msg("resolved container endpoint via internal IP (no host port binding)")
3461 return ip, internalPort, nil
3462}
3463
3464// waitForReadyByDelay waits using the legacy running+delay strategy.
3465func (s *Service) waitForReadyByDelay(ctx context.Context, containerID string) error {
3466 log := zerowrap.FromCtx(ctx)
3467
3468 s.mu.RLock()
3469 cfg := s.config
3470 s.mu.RUnlock()
3471
3472 delay := cfg.ReadinessDelay
3473 if delay == 0 {
3474 delay = 5 * time.Second
3475 }
3476
3477 log.Debug().Dur("delay", delay).Msg("waiting for container readiness")
3478 select {
3479 case <-time.After(delay):
3480 case <-ctx.Done():
3481 return ctx.Err()
3482 }
3483
3484 // Verify still running after delay
3485 running, err := s.runtime.IsContainerRunning(ctx, containerID)
3486 if err != nil {
3487 return err
3488 }
3489 if running {
3490 return nil
3491 }
3492
3493 return s.waitForRecovery(ctx, containerID)
3494}
3495
3496func (s *Service) waitForHealthy(ctx context.Context, containerID string, timeout time.Duration) error {
3497 if timeout == 0 {
3498 timeout = 90 * time.Second
3499 }
3500 waitCtx, cancel := context.WithTimeout(ctx, timeout)
3501 defer cancel()
3502
3503 var lastStatus string
3504 for {
3505 status, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(waitCtx, containerID)
3506 if err != nil {
3507 return err
3508 }
3509 if !hasHealthcheck {
3510 return errors.New("no healthcheck detected")
3511 }
3512 // Treat empty health status as transitional startup. Some runtimes can
3513 // temporarily report an empty status before first probe results.
3514 if status == "" {
3515 status = "starting"
3516 }
3517 if status == "healthy" {
3518 return nil
3519 }
3520 if status == "unhealthy" {
3521 return errors.New("container reported unhealthy")
3522 }
3523 lastStatus = status
3524
3525 select {
3526 case <-time.After(time.Second):
3527 case <-waitCtx.Done():
3528 return fmt.Errorf("container healthcheck timeout after %s (last status: %s)", timeout, lastStatus)
3529 }
3530 }
3531}
3532
3533// pollContainerRunning polls until the container is running (max 30 seconds).
3534func (s *Service) pollContainerRunning(ctx context.Context, containerID string) error {
3535 for i := 0; i < 30; i++ {
3536 running, err := s.runtime.IsContainerRunning(ctx, containerID)
3537 if err != nil {
3538 return err
3539 }
3540 if running {
3541 return nil
3542 }
3543 // Check if the container has already exited (crash) rather than still starting up.
3544 if info, inspectErr := s.runtime.InspectContainer(ctx, containerID); inspectErr == nil && info != nil && info.Status == string(domain.ContainerStatusExited) {
3545 return fmt.Errorf("container exited immediately with code %d: %w", info.ExitCode, domain.ErrContainerExited)
3546 }
3547 if i == 29 {
3548 return fmt.Errorf("container did not start within 30 seconds")
3549 }
3550 select {
3551 case <-time.After(time.Second):
3552 case <-ctx.Done():
3553 return ctx.Err()
3554 }
3555 }
3556 return fmt.Errorf("container did not start within 30 seconds")
3557}
3558
3559// waitForRecovery waits for a container to recover within the recovery window.
3560func (s *Service) waitForRecovery(ctx context.Context, containerID string) error {
3561 log := zerowrap.FromCtx(ctx)
3562 log.Warn().
3563 Dur("recovery_window", readinessRecoveryWindow).
3564 Msg("container not running after readiness delay, waiting for recovery")
3565
3566 deadline := time.Now().Add(readinessRecoveryWindow)
3567 for time.Now().Before(deadline) {
3568 select {
3569 case <-time.After(time.Second):
3570 case <-ctx.Done():
3571 return ctx.Err()
3572 }
3573
3574 running, err := s.runtime.IsContainerRunning(ctx, containerID)
3575 if err != nil {
3576 return err
3577 }
3578 if running {
3579 log.Info().Msg("container recovered during readiness recovery window")
3580 return nil
3581 }
3582 }
3583
3584 return fmt.Errorf("container not running after readiness delay and recovery window")
3585}
3586
3587// publishContainerDeployed publishes a container.deployed event.
3588func (s *Service) publishContainerDeployed(ctx context.Context, domainName, containerID string) {
3589 payload := &domain.ContainerEventPayload{
3590 ContainerID: containerID,
3591 Domain: domainName,
3592 Action: "deployed",
3593 }
3594
3595 if err := s.eventBus.Publish(domain.EventContainerDeployed, payload); err != nil {
3596 log := zerowrap.FromCtx(ctx)
3597 log.Warn().Err(err).Msg("failed to publish container deployed event")
3598 }
3599}
3600
3601// rewriteToRegistryDomain rewrites an image reference to use the configured registry domain.
3602// e.g., "myapp:latest" -> "registry.example.com/myapp:latest"
3603func rewriteToRegistryDomain(imageRef, registryDomain string) string {
3604 if registryDomain == "" {
3605 return imageRef
3606 }
3607
3608 prefix := registryDomain + "/"
3609 if strings.HasPrefix(imageRef, prefix) {
3610 return imageRef
3611 }
3612
3613 return prefix + imageRef
3614}
3615
3616// rewriteToLocalRegistry rewrites Gordon-managed image references to use the
3617// local registry address. Current and legacy Gordon registry domains are
3618// stripped before prefixing localhost:<registryPort>/. Bare refs are prefixed.
3619// External registries are preserved unchanged.
3620func rewriteToLocalRegistry(imageRef, registryDomain string, legacyRegistryDomains []string, registryPort int) string {
3621 if imageRef == "" {
3622 return imageRef
3623 }
3624
3625 localRegistry := fmt.Sprintf("localhost:%d", registryPort)
3626 localPrefix := localRegistry + "/"
3627 if strings.HasPrefix(imageRef, localPrefix) {
3628 return imageRef
3629 }
3630
3631 if domain.IsGordonRegistryImageRef(imageRef, registryDomain, legacyRegistryDomains) {
3632 return localPrefix + domain.StripKnownGordonRegistry(imageRef, registryDomain, legacyRegistryDomains)
3633 }
3634 if hasExplicitRegistry(imageRef) {
3635 return imageRef
3636 }
3637
3638 return localPrefix + imageRef
3639}