1// Package container implements the container management use case.
2package container
3
4import (
5 "bufio"
6 "context"
7 "encoding/binary"
8 "encoding/hex"
9 "errors"
10 "fmt"
11 "io"
12 "maps"
13 "net"
14 "slices"
15 "strconv"
16 "strings"
17 "sync"
18 "time"
19
20 "github.com/bnema/zerowrap"
21 "go.opentelemetry.io/otel"
22 "go.opentelemetry.io/otel/attribute"
23 "go.opentelemetry.io/otel/codes"
24 "go.opentelemetry.io/otel/metric"
25 "go.opentelemetry.io/otel/trace"
26
27 "github.com/bnema/gordon/internal/adapters/out/telemetry"
28 "github.com/bnema/gordon/internal/boundaries/out"
29 "github.com/bnema/gordon/internal/domain"
30)
31
32// Config holds configuration needed by the container service.
33type Config struct {
34 RegistryAuthEnabled bool
35 RegistryDomain string
36 LegacyRegistryDomains []string
37 RegistryPort int
38 ServiceTokenUsername string
39 ServiceToken string
40 InternalRegistryUsername string
41 InternalRegistryPassword string
42 PullPolicy string
43 VolumeAutoCreate bool
44 VolumePrefix string
45 VolumePreserve bool
46 NetworkIsolation bool
47 NetworkPrefix string
48 NetworkGroups map[string][]string
49 NetworkInternal bool
50 Attachments map[string][]string
51 AllowedRegistries []string
52 RequireImageDigest bool
53 SecurityProfile string
54 ReadinessDelay time.Duration // Delay after container starts before considering it ready
55 ReadinessMode string // Readiness strategy: auto, docker-health, delay
56 HealthTimeout time.Duration // Max wait for health-based readiness
57 DrainDelay time.Duration // Grace period after cache invalidation before stopping old container
58 DrainDelayConfigured bool // True when deploy.drain_delay was explicitly configured
59 DrainMode string // Drain strategy: auto, inflight, delay
60 DrainTimeout time.Duration // Max wait for in-flight requests to drain
61 StabilizationDelay time.Duration // Post-switch monitoring window (default 2s)
62 TCPProbeTimeout time.Duration // TCP probe timeout (default 30s)
63 HTTPProbeTimeout time.Duration // HTTP probe timeout (default 60s)
64 AttachmentReadinessTimeout time.Duration // Max wait for attachment readiness (default 30s)
65 DefaultMemoryLimit int64 // Default memory limit in bytes for containers (0 = no limit)
66 DefaultNanoCPUs int64 // Default CPU quota in nanoseconds for containers (0 = no limit)
67 DefaultPidsLimit int64 // Default max PIDs for containers (0 = no limit)
68}
69
70var tracer = otel.Tracer("gordon.container")
71
72const (
73 PullPolicyAlways = "always"
74 PullPolicyIfNotPresent = "if-not-present"
75 PullPolicyIfTagChanged = "if-tag-changed"
76
77 // readinessRecoveryWindow is an additional grace window used when a container
78 // is briefly not running at the end of readiness delay. This avoids false
79 // negatives during short startup flaps.
80 readinessRecoveryWindow = 30 * time.Second
81 failedContainerCleanupTimeout = 30 * time.Second
82 internalPullMaxAttempts = 3
83 deployFailureLogLimit = 20
84)
85
86// Service implements the ContainerService interface.
87type Service struct {
88 runtime out.ContainerRuntime
89 envLoader out.EnvLoader
90 eventBus out.EventPublisher
91 logWriter out.ContainerLogWriter
92 cacheInvalidator out.ProxyCacheInvalidator
93 drainWaiter out.ProxyDrainWaiter
94 config Config
95 configProvider AttachmentConfigProvider // live config reads for attachments/networks (may be nil)
96 metrics *telemetry.Metrics
97 containers map[string]*domain.Container
98 attachments map[string][]string // ownerDomain → []containerIDs
99 managedCount int64 // tracks UpDownCounter value for delta computation
100 mu sync.RWMutex
101 deployMu sync.Map // per-domain deploy locks (domain → *domainDeployLock)
102 cleanupWg sync.WaitGroup // tracks background old-container cleanup goroutines
103 monitor *Monitor
104}
105
106// domainDeployLock is a context-aware mutex using a buffered channel.
107// It allows Lock() to be interrupted by context cancellation.
108type domainDeployLock struct {
109 ch chan struct{}
110}
111
112// newDomainDeployLock creates a new lock that is initially unlocked.
113func newDomainDeployLock() *domainDeployLock {
114 l := &domainDeployLock{ch: make(chan struct{}, 1)}
115 l.ch <- struct{}{}
116 return l
117}
118
119// Lock acquires the lock or returns an error if the context is cancelled.
120func (l *domainDeployLock) Lock(ctx context.Context) error {
121 // Fast path: honor already-canceled contexts before blocking.
122 if err := ctx.Err(); err != nil {
123 return err
124 }
125 select {
126 case <-ctx.Done():
127 return ctx.Err()
128 case <-l.ch:
129 // Re-check context after acquisition: if canceled while we were
130 // in the select, release the token so the lock isn't consumed.
131 if err := ctx.Err(); err != nil {
132 l.ch <- struct{}{}
133 return err
134 }
135 return nil
136 }
137}
138
139// Unlock releases the lock.
140func (l *domainDeployLock) Unlock() {
141 select {
142 case l.ch <- struct{}{}:
143 default:
144 // This should never happen if Lock/Unlock are used correctly.
145 // Panic to detect double-unlocks or other misuse early.
146 panic("domainDeployLock: double unlock detected")
147 }
148}
149
150// NewService creates a new container service.
151func NewService(
152 runtime out.ContainerRuntime,
153 envLoader out.EnvLoader,
154 eventBus out.EventPublisher,
155 logWriter out.ContainerLogWriter,
156 config Config,
157 configProvider AttachmentConfigProvider,
158) *Service {
159 return &Service{
160 runtime: runtime,
161 envLoader: envLoader,
162 eventBus: eventBus,
163 logWriter: logWriter,
164 config: config,
165 configProvider: configProvider,
166 containers: make(map[string]*domain.Container),
167 attachments: make(map[string][]string),
168 }
169}
170
171// SetMetrics sets the telemetry metrics for the container service.
172func (s *Service) SetMetrics(m *telemetry.Metrics) {
173 s.metrics = m
174}
175
176// SetProxyCacheInvalidator sets the proxy cache invalidator for synchronous
177// cache invalidation during zero-downtime deployments. This must be called
178// after construction because the proxy service is created after the container
179// service during application initialization.
180func (s *Service) SetProxyCacheInvalidator(inv out.ProxyCacheInvalidator) {
181 s.mu.Lock()
182 s.cacheInvalidator = inv
183 s.mu.Unlock()
184}
185
186// SetProxyDrainWaiter sets the proxy in-flight drain waiter used during
187// zero-downtime replacement before stopping the old container.
188func (s *Service) SetProxyDrainWaiter(waiter out.ProxyDrainWaiter) {
189 s.mu.Lock()
190 s.drainWaiter = waiter
191 s.mu.Unlock()
192}
193
194// domainDeployMu returns a per-domain lock for serializing deploy operations.
195func (s *Service) domainDeployMu(domain string) *domainDeployLock {
196 v, _ := s.deployMu.LoadOrStore(domain, newDomainDeployLock())
197 // Type assertion is safe: LoadOrStore always stores *domainDeployLock values,
198 // and any concurrent LoadOrStore calls will also store *domainDeployLock.
199 return v.(*domainDeployLock)
200}
201
202// acquireDomainDeployLock acquires the per-domain deploy lock with context cancellation support.
203// Returns an unlock function that must be called to release the lock.
204func (s *Service) acquireDomainDeployLock(ctx context.Context, domain string) (func(), error) {
205 lock := s.domainDeployMu(domain)
206 if err := lock.Lock(ctx); err != nil {
207 return nil, err
208 }
209 return func() { lock.Unlock() }, nil
210}
211
212// buildContainerConfig constructs the container configuration for deployment.
213func (s *Service) deploymentContainerName(containerDomain string, existing *domain.Container) string {
214 canonicalName := fmt.Sprintf("gordon-%s", containerDomain)
215 if existing == nil {
216 return canonicalName
217 }
218
219 // Alternate temporary names if the tracked container was left with a temp suffix.
220 // This prevents name collisions after interrupted zero-downtime deploys.
221 newName := canonicalName + "-new"
222 nextName := canonicalName + "-next"
223 switch existing.Name {
224 case newName:
225 return nextName
226 case nextName:
227 return newName
228 default:
229 return newName
230 }
231}
232
233// containerConfigInput groups the parameters for building a container configuration.
234type containerConfigInput struct {
235 Domain string
236 Image string
237 ImageRef string
238 ExposedPorts []int
239 EnvVars []string
240 EnvHash string
241 Volumes map[string]string
242 NetworkName string
243 ImageLabels map[string]string
244 Existing *domain.Container
245}
246
247func (s *Service) buildContainerConfig(in containerConfigInput) *domain.ContainerConfig {
248 s.mu.RLock()
249 cfg := s.config
250 s.mu.RUnlock()
251
252 containerName := s.deploymentContainerName(in.Domain, in.Existing)
253 labels := map[string]string{
254 domain.LabelDomain: in.Domain,
255 domain.LabelEnvHash: in.EnvHash,
256 domain.LabelImage: in.Image,
257 domain.LabelManaged: "true",
258 domain.LabelRoute: in.Domain,
259 }
260
261 // Propagate proxy/port labels from image so readiness probes can find them
262 for _, key := range []string{domain.LabelProxyPort, domain.LabelPort, domain.LabelHealth} {
263 if v, ok := in.ImageLabels[key]; ok && v != "" {
264 labels[key] = v
265 }
266 }
267
268 // Ensure the label-specified proxy port is included in exposed ports
269 // so Docker creates a host port binding for it.
270 ports := in.ExposedPorts
271 if portStr, ok := labels[domain.LabelProxyPort]; ok {
272 if port, err := strconv.Atoi(portStr); err == nil && port > 0 && port <= 65535 {
273 if !slices.Contains(ports, port) {
274 ports = append(ports, port)
275 }
276 }
277 }
278
279 containerConfig := &domain.ContainerConfig{
280 Image: in.ImageRef,
281 Name: containerName,
282 Ports: ports,
283 Env: in.EnvVars,
284 Volumes: in.Volumes,
285 NetworkMode: in.NetworkName,
286 Hostname: in.Domain,
287 Labels: labels,
288 AutoRemove: false,
289 RestartPolicy: domain.RestartPolicyAlways,
290 MemoryLimit: cfg.DefaultMemoryLimit,
291 NanoCPUs: cfg.DefaultNanoCPUs,
292 PidsLimit: cfg.DefaultPidsLimit,
293 }
294 applySecurityProfile(containerConfig, cfg)
295 return containerConfig
296}
297
298// Deploy creates and starts a container for the given route.
299// Implements zero-downtime deployment: new container starts before old one stops.
300func (s *Service) Deploy(ctx context.Context, route domain.Route) (*domain.Container, error) {
301 ctx, span := tracer.Start(ctx, "container.deploy",
302 trace.WithAttributes(
303 attribute.String("domain", route.Domain),
304 attribute.String("image", route.Image),
305 ))
306 defer span.End()
307
308 deployStart := time.Now()
309
310 // Serialize deploys for the same domain to prevent race conditions
311 // (e.g. multiple image.pushed events + explicit deploy call from CLI).
312 unlock, err := s.acquireDomainDeployLock(ctx, route.Domain)
313 if err != nil {
314 return nil, err
315 }
316 defer unlock()
317
318 // Enrich context with use case fields for all downstream logs
319 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
320 zerowrap.FieldLayer: "usecase",
321 zerowrap.FieldUseCase: "Deploy",
322 "domain": route.Domain,
323 })
324 log := zerowrap.FromCtx(ctx)
325
326 // Record deploy metrics and trace status
327 defer s.recordDeployMetrics(ctx, span, route, deployStart, &err)
328
329 existing, hasExisting := s.resolveExistingContainer(ctx, route.Domain)
330
331 resources, err := s.prepareDeployResources(ctx, route, existing)
332 if err != nil {
333 if deployErr := s.wrapPullDeployFailure(route, existing, err); deployErr != nil {
334 err = deployErr
335 return nil, err
336 }
337 return nil, err
338 }
339
340 // Skip redundant deploy: if the existing container is already running
341 // the exact same image (by Docker image ID), return it immediately.
342 // This prevents the double-deploy caused by the event-based deploy
343 // (triggered by image.pushed) racing with the explicit CLI deploy call.
344 if hasExisting {
345 existingForSkip := existing
346 if existingForSkip.ImageID == "" && existingForSkip.Image != "" && normalizeImageRef(existingForSkip.Image) == normalizeImageRef(route.Image) {
347 existingForSkip = s.containerForRedundantCheck(ctx, existingForSkip)
348 }
349 if existingForSkip.ImageID != "" {
350 if skip, container := s.skipRedundantDeploy(ctx, existingForSkip, resources.actualImageRef, resources.envHash); skip {
351 return container, nil
352 }
353 }
354 }
355
356 newContainer, err := s.createStartedContainer(ctx, route, existing, resources)
357 if err != nil {
358 return nil, err
359 }
360
361 invalidated := s.activateDeployedContainer(ctx, route.Domain, newContainer)
362
363 // Post-switch stabilization: verify new container stays running
364 if hasExisting {
365 stable, stabilizeErr := s.stabilizeNewContainer(ctx, route.Domain, newContainer, existing)
366 if stabilizeErr != nil {
367 // Both old and new containers are dead; assign to named return so
368 // the deferred recordDeployMetrics and span see the failure.
369 err = stabilizeErr
370 return nil, err
371 }
372 if !stable {
373 // Rollback performed — old container is restored
374 return existing, nil
375 }
376 }
377
378 // Finalize old container in the background — the new container is already
379 // serving traffic, so there's no reason to block the deploy response while
380 // waiting for the old container to stop (which can take 20s if the app
381 // doesn't handle SIGTERM).
382 s.cleanupWg.Add(1)
383 go func() {
384 defer s.cleanupWg.Done()
385 s.finalizePreviousContainer(context.WithoutCancel(ctx), route.Domain, existing, hasExisting, invalidated, newContainer.ID)
386 }()
387
388 // Start container log collection (non-blocking, errors don't fail deployment)
389 s.startLogCollection(ctx, newContainer.ID, route.Domain)
390
391 log.Info().
392 Str("image", route.Image).
393 Str(zerowrap.FieldEntityID, newContainer.ID).
394 Ints("ports", newContainer.Ports).
395 Str("network", resources.networkName).
396 Bool("zero_downtime", hasExisting).
397 Msg("container deployed successfully")
398
399 return newContainer, nil
400}
401
402// recordDeployMetrics records span error status and deploy metrics at the end of a Deploy call.
403// It is called via defer and receives a pointer to the named return error so it can observe
404// the final error value after all deferred functions have run.
405func (s *Service) recordDeployMetrics(ctx context.Context, span trace.Span, route domain.Route, start time.Time, errPtr *error) {
406 err := *errPtr
407 if err != nil {
408 span.RecordError(err)
409 span.SetStatus(codes.Error, err.Error())
410 }
411 if s.metrics == nil {
412 return
413 }
414 attrs := metric.WithAttributes(
415 attribute.String("domain", route.Domain),
416 attribute.String("image", route.Image),
417 )
418 s.metrics.DeployTotal.Add(ctx, 1, attrs)
419 s.metrics.DeployDuration.Record(ctx, time.Since(start).Seconds(), attrs)
420 if err != nil {
421 s.metrics.DeployErrors.Add(ctx, 1, attrs)
422 }
423}
424
425// skipRedundantDeploy checks whether the existing container is already running
426// the same image (by Docker image ID) as the one we are about to deploy.
427// When a push triggers both an event-based deploy and an explicit CLI deploy,
428// the second one arrives after the first has already completed; this avoids
429// a full redundant create-start-readiness cycle.
430func (s *Service) skipRedundantDeploy(ctx context.Context, existing *domain.Container, actualImageRef, envHash string) (bool, *domain.Container) {
431 log := zerowrap.FromCtx(ctx)
432
433 newImageID, err := s.runtime.GetImageID(ctx, actualImageRef)
434 if err != nil {
435 log.Debug().Err(err).Msg("cannot resolve image ID for redundancy check, proceeding with deploy")
436 return false, nil
437 }
438
439 if existing.ImageID != newImageID {
440 return false, nil
441 }
442
443 // Verify the container is actually still running before skipping.
444 running, err := s.runtime.IsContainerRunning(ctx, existing.ID)
445 if err != nil || !running {
446 return false, nil
447 }
448
449 if existing.Labels != nil {
450 existingEnvHash, hasEnvHash := existing.Labels[domain.LabelEnvHash]
451 if !hasEnvHash || existingEnvHash != envHash {
452 log.Info().
453 Str("container_id", existing.ID).
454 Bool("has_env_hash", hasEnvHash).
455 Msg("env changed, proceeding with deploy despite same image")
456 return false, nil
457 }
458 } else {
459 return false, nil
460 }
461
462 log.Info().
463 Str("container_id", existing.ID).
464 Str("image_id", newImageID).
465 Msg("skipping redundant deploy: container already running this image")
466
467 return true, existing
468}
469
470func (s *Service) containerForRedundantCheck(ctx context.Context, existing *domain.Container) *domain.Container {
471 if existing == nil || existing.ImageID != "" || existing.ID == "" {
472 return existing
473 }
474
475 log := zerowrap.FromCtx(ctx)
476 inspected, err := s.runtime.InspectContainer(ctx, existing.ID)
477 if err != nil {
478 log.Debug().Err(err).Str("container_id", existing.ID).Msg("cannot inspect existing container for redundancy check")
479 return existing
480 }
481 if inspected == nil || inspected.ImageID == "" {
482 return existing
483 }
484
485 existingCopy := *existing
486 existingCopy.ImageID = inspected.ImageID
487 return &existingCopy
488}
489
490type deployResources struct {
491 networkName string
492 actualImageRef string
493 exposedPorts []int
494 imageLabels map[string]string
495 envVars []string
496 envHash string
497 volumes map[string]string
498}
499
500// resolveExistingContainer returns the currently running container for a domain.
501// It first checks the in-memory map (fast path). If not found, it queries the
502// runtime for a running container with matching name and managed label. This
503// handles cases where Gordon restarted and in-memory state is stale.
504//
505// The slow path checks canonical, -new, and -next names because a previous
506// deploy may have been interrupted (e.g. eventbus timeout, Gordon restart)
507// leaving the active container under a temp name. Canonical is preferred when
508// multiple matches exist.
509func (s *Service) resolveExistingContainer(ctx context.Context, domainName string) (*domain.Container, bool) {
510 // Fast path: check in-memory state
511 s.mu.RLock()
512 container, ok := s.containers[domainName]
513 s.mu.RUnlock()
514 if ok {
515 return container, true
516 }
517
518 // Slow path: query runtime for running containers
519 log := zerowrap.FromCtx(ctx)
520 canonicalName := fmt.Sprintf("gordon-%s", domainName)
521 candidateNames := map[string]bool{
522 canonicalName: true,
523 canonicalName + "-new": true,
524 canonicalName + "-next": true,
525 }
526
527 running, err := s.runtime.ListContainers(ctx, false)
528 if err != nil {
529 log.Warn().Err(err).Msg("failed to list running containers for existing container resolution")
530 return nil, false
531 }
532
533 // Prefer canonical name; fall back to any running managed temp container.
534 var best *domain.Container
535 for _, c := range running {
536 if !candidateNames[c.Name] || c.Labels[domain.LabelManaged] != "true" {
537 continue
538 }
539 if best == nil || c.Name == canonicalName {
540 best = c
541 }
542 }
543
544 if best != nil {
545 log.Info().
546 Str("container_id", best.ID).
547 Str("container_name", best.Name).
548 Msg("resolved existing container from runtime (in-memory state was stale)")
549
550 // Update in-memory state so subsequent lookups are fast
551 s.mu.Lock()
552 if _, alreadyTracked := s.containers[domainName]; !alreadyTracked {
553 s.managedCount++
554 }
555 s.containers[domainName] = best
556 s.mu.Unlock()
557
558 return best, true
559 }
560
561 return nil, false
562}
563
564func (s *Service) prepareDeployResources(ctx context.Context, route domain.Route, existing *domain.Container) (*deployResources, error) {
565 log := zerowrap.FromCtx(ctx)
566
567 existingID := ""
568 if existing != nil {
569 existingID = existing.ID
570 }
571 if err := s.cleanupOrphanedContainers(ctx, route.Domain, existingID); err != nil {
572 log.WrapErr(err, "failed to cleanup orphaned containers")
573 }
574
575 imageRef, err := s.buildValidatedImageRef(ctx, route.Image)
576 if err != nil {
577 return nil, err
578 }
579 actualImageRef, err := s.ensureImage(ctx, imageRef)
580 if err != nil {
581 return nil, err
582 }
583
584 networkName := s.getNetworkForApp(route.Domain)
585 if err := s.createNetworkIfNeeded(ctx, networkName); err != nil {
586 return nil, log.WrapErr(err, "failed to create network")
587 }
588 if err := s.deployAttachments(ctx, route.Domain, networkName); err != nil {
589 return nil, log.WrapErr(err, "failed to deploy attachments")
590 }
591
592 exposedPorts, err := s.runtime.GetImageExposedPorts(ctx, actualImageRef)
593 if err != nil {
594 log.WrapErr(err, "failed to get exposed ports, using defaults")
595 exposedPorts = []int{80, 8080, 3000}
596 }
597
598 imageLabels, err := s.runtime.GetImageLabels(ctx, actualImageRef)
599 if err != nil {
600 log.Warn().Err(err).Msg("failed to get image labels, skipping label propagation")
601 imageLabels = nil
602 }
603
604 envVars, err := s.loadEnvironment(ctx, route.Env, route.Domain, actualImageRef)
605 if err != nil {
606 return nil, err
607 }
608 envHash := hashEnvironment(envVars)
609
610 volumes, err := s.setupVolumes(ctx, route.Domain, actualImageRef)
611 if err != nil {
612 return nil, err
613 }
614
615 return &deployResources{
616 networkName: networkName,
617 actualImageRef: actualImageRef,
618 exposedPorts: exposedPorts,
619 imageLabels: imageLabels,
620 envVars: envVars,
621 envHash: envHash,
622 volumes: volumes,
623 }, nil
624}
625
626func (s *Service) createStartedContainer(ctx context.Context, route domain.Route, existing *domain.Container, resources *deployResources) (*domain.Container, error) {
627 ctx, span := tracer.Start(ctx, "container.create_and_start")
628 defer span.End()
629
630 log := zerowrap.FromCtx(ctx)
631
632 containerConfig := s.buildContainerConfig(containerConfigInput{
633 Domain: route.Domain,
634 Image: route.Image,
635 ImageRef: resources.actualImageRef,
636 ExposedPorts: resources.exposedPorts,
637 EnvVars: resources.envVars,
638 EnvHash: resources.envHash,
639 Volumes: resources.volumes,
640 NetworkName: resources.networkName,
641 ImageLabels: resources.imageLabels,
642 Existing: existing,
643 })
644
645 newContainer, err := s.runtime.CreateContainer(ctx, containerConfig)
646 if err != nil {
647 return nil, log.WrapErr(err, "failed to create container")
648 }
649 if err := s.runtime.StartContainer(ctx, newContainer.ID); err != nil {
650 s.runtime.RemoveContainer(ctx, newContainer.ID, true)
651 return nil, log.WrapErr(err, "failed to start container")
652 }
653 if !domain.IsSkipReadiness(ctx) {
654 if err := s.waitForReady(ctx, newContainer.ID, containerConfig); err != nil {
655 deployErr := s.newDeployFailure(newContainer, "container failed readiness check", err, s.captureRecentContainerLogs(ctx, newContainer.ID))
656 s.cleanupFailedContainer(ctx, newContainer.ID)
657 return nil, deployErr
658 }
659 }
660
661 inspected, err := s.runtime.InspectContainer(ctx, newContainer.ID)
662 if err != nil {
663 s.cleanupFailedContainer(ctx, newContainer.ID)
664 return nil, log.WrapErr(err, "failed to inspect started container")
665 }
666
667 return inspected, nil
668}
669
670func (s *Service) newDeployFailure(container *domain.Container, cause string, err error, logs []string) error {
671 deployErr := &domain.DeployFailureError{
672 Summary: "failed to deploy",
673 Cause: cause,
674 Hint: inferDeployFailureHint(err),
675 Logs: logs,
676 Err: err,
677 }
678 if container != nil {
679 deployErr.ContainerName = container.Name
680 deployErr.ContainerID = container.ID
681 }
682
683 return deployErr
684}
685
686func (s *Service) wrapPullDeployFailure(route domain.Route, existing *domain.Container, err error) error {
687 if !isPullFailure(err) {
688 return nil
689 }
690
691 return s.newDeployFailure(&domain.Container{Name: s.deploymentContainerName(route.Domain, existing)}, "failed to pull image", err, nil)
692}
693
694func (s *Service) captureRecentContainerLogs(ctx context.Context, containerID string) []string {
695 if ctx.Err() != nil {
696 return nil
697 }
698
699 logStream, err := s.runtime.GetContainerLogs(ctx, containerID, false)
700 if err != nil {
701 return nil
702 }
703 defer func() {
704 _ = logStream.Close()
705 }()
706
707 logs, err := parseDockerLogLines(logStream, deployFailureLogLimit)
708 if err != nil {
709 return nil
710 }
711
712 return logs
713}
714
715func parseDockerLogLines(r io.Reader, maxLines int) ([]string, error) {
716 reader := bufio.NewReader(r)
717 collector := newLogLineCollector(maxLines)
718
719 header, err := reader.Peek(8)
720 if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, bufio.ErrBufferFull) {
721 return nil, err
722 }
723
724 if len(header) >= 8 && looksLikeDockerLogStream(header) {
725 return parseDockerFramedLogLines(reader, collector)
726 }
727
728 return parsePlainLogLines(reader, collector)
729}
730
731func parseDockerFramedLogLines(reader *bufio.Reader, collector *logLineCollector) ([]string, error) {
732 for {
733 frameHeader := make([]byte, 8)
734 if _, err := io.ReadFull(reader, frameHeader); err != nil {
735 if isLogStreamEOF(err) {
736 collector.flush()
737 return collector.lines, nil
738 }
739 return nil, err
740 }
741
742 size := int(binary.BigEndian.Uint32(frameHeader[4:8]))
743 if size <= 0 {
744 continue
745 }
746
747 payload, err := readLogPayload(reader, size)
748 if err != nil {
749 collector.addChunk(string(payload))
750 collector.flush()
751 if isLogStreamEOF(err) {
752 return collector.lines, nil
753 }
754 return nil, err
755 }
756
757 collector.addChunk(string(payload))
758 }
759}
760
761func parsePlainLogLines(reader *bufio.Reader, collector *logLineCollector) ([]string, error) {
762
763 scanner := bufio.NewScanner(reader)
764 for scanner.Scan() {
765 collector.addLine(scanner.Text())
766 }
767 if err := scanner.Err(); err != nil {
768 return nil, err
769 }
770
771 collector.flush()
772 return collector.lines, nil
773}
774
775func readLogPayload(reader *bufio.Reader, size int) ([]byte, error) {
776 payload := make([]byte, size)
777 _, err := io.ReadFull(reader, payload)
778 return payload, err
779}
780
781func isLogStreamEOF(err error) bool {
782 return errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF)
783}
784
785func looksLikeDockerLogStream(data []byte) bool {
786 return len(data) >= 8 && data[1] == 0 && data[2] == 0 && data[3] == 0
787}
788
789type logLineCollector struct {
790 lines []string
791 pending string
792 maxLines int
793}
794
795func newLogLineCollector(maxLines int) *logLineCollector {
796 return &logLineCollector{maxLines: maxLines}
797}
798
799func (c *logLineCollector) addChunk(chunk string) {
800 for _, part := range strings.SplitAfter(chunk, "\n") {
801 c.pending += part
802 if strings.HasSuffix(part, "\n") {
803 c.flush()
804 }
805 }
806}
807
808func (c *logLineCollector) addLine(line string) {
809 line = strings.TrimRight(line, "\r")
810 if line == "" {
811 return
812 }
813 c.lines = append(c.lines, line)
814 if c.maxLines > 0 && len(c.lines) > c.maxLines {
815 c.lines = c.lines[len(c.lines)-c.maxLines:]
816 }
817}
818
819func (c *logLineCollector) flush() {
820 if c.pending == "" {
821 return
822 }
823 c.addLine(strings.TrimRight(c.pending, "\r\n"))
824 c.pending = ""
825}
826
827func inferDeployFailureHint(err error) string {
828 if err == nil {
829 return ""
830 }
831
832 msg := strings.ToLower(err.Error())
833 switch {
834 case strings.Contains(msg, "healthcheck timeout"):
835 return "check startup logs and confirm the health endpoint becomes ready within the configured timeout"
836 case strings.Contains(msg, "unhealthy"):
837 return "inspect recent app logs and verify the healthcheck command reflects actual readiness"
838 case strings.Contains(msg, "no healthcheck detected"):
839 return "add a Docker healthcheck or switch readiness mode"
840 case isPullFailure(err), strings.Contains(msg, "access denied"), strings.Contains(msg, "unauthorized"):
841 return "verify registry auth and confirm the image exists at the requested tag"
842 default:
843 return ""
844 }
845}
846
847func isPullFailure(err error) bool {
848 if err == nil {
849 return false
850 }
851
852 if errors.Is(err, domain.ErrImagePullFailed) {
853 return true
854 }
855
856 msg := strings.ToLower(err.Error())
857 return strings.Contains(msg, "failed to pull image") ||
858 strings.Contains(msg, "pull access denied")
859}
860
861func (s *Service) activateDeployedContainer(ctx context.Context, domainName string, container *domain.Container) bool {
862 s.mu.Lock()
863 _, wasTracked := s.containers[domainName]
864 s.containers[domainName] = container
865 if !wasTracked {
866 s.managedCount++
867 }
868 s.mu.Unlock()
869
870 // Track managed container count (only increment for new domains, not replacements)
871 if !wasTracked && s.metrics != nil {
872 s.metrics.ManagedContainers.Add(ctx, 1)
873 }
874
875 s.publishContainerDeployed(ctx, domainName, container.ID)
876
877 s.mu.RLock()
878 inv := s.cacheInvalidator
879 s.mu.RUnlock()
880 if inv != nil {
881 inv.InvalidateTarget(ctx, domainName)
882 return true
883 }
884
885 return false
886}
887
888// stabilizeNewContainer monitors the new container briefly after traffic switch.
889// If it crashes during this window, rolls back to old container.
890// Returns true if stabilization succeeded, false if rollback was performed.
891// Returns an error only when both new and old containers are dead.
892func (s *Service) stabilizeNewContainer(ctx context.Context, domainName string, newContainer, oldContainer *domain.Container) (bool, error) {
893 log := zerowrap.FromCtx(ctx)
894
895 if oldContainer == nil {
896 return true, nil
897 }
898
899 s.mu.RLock()
900 delay := s.config.StabilizationDelay
901 s.mu.RUnlock()
902 if delay == 0 {
903 delay = 2 * time.Second
904 }
905
906 // Brief stabilization: verify new container is still running after delay
907 select {
908 case <-time.After(delay):
909 case <-ctx.Done():
910 return true, nil
911 }
912
913 running, err := s.runtime.IsContainerRunning(ctx, newContainer.ID)
914 if err != nil || !running {
915 log.Error().
916 Str("new_container", newContainer.ID).
917 Str("old_container", oldContainer.ID).
918 Msg("new container crashed during stabilization, rolling back to old")
919
920 // Verify old container is still running before restoring it.
921 // If both old and new are dead (e.g., OOM), returning a dead
922 // container as "existing" would leave the domain in a broken state.
923 oldRunning, oldErr := s.runtime.IsContainerRunning(ctx, oldContainer.ID)
924 if oldErr != nil || !oldRunning {
925 log.Error().
926 Str("old_container", oldContainer.ID).
927 Msg("old container is also not running, cannot rollback")
928
929 // Cleanup failed new container
930 if stopErr := s.runtime.StopContainer(ctx, newContainer.ID); stopErr != nil {
931 log.WrapErrWithFields(stopErr, "failed to stop failed new container during rollback", map[string]any{zerowrap.FieldEntityID: newContainer.ID})
932 }
933 if removeErr := s.runtime.RemoveContainer(ctx, newContainer.ID, true); removeErr != nil {
934 log.WrapErrWithFields(removeErr, "failed to remove failed new container during rollback", map[string]any{zerowrap.FieldEntityID: newContainer.ID})
935 }
936
937 return false, fmt.Errorf("stabilization failed: new container crashed and old container %s is also not running", oldContainer.ID)
938 }
939
940 // Rollback: restore old container as tracked
941 s.mu.Lock()
942 s.containers[domainName] = oldContainer
943 s.mu.Unlock()
944
945 // Re-invalidate proxy cache to point back to old
946 s.mu.RLock()
947 inv := s.cacheInvalidator
948 s.mu.RUnlock()
949 if inv != nil {
950 inv.InvalidateTarget(ctx, domainName)
951 }
952
953 // Cleanup failed new container
954 if stopErr := s.runtime.StopContainer(ctx, newContainer.ID); stopErr != nil {
955 log.WrapErrWithFields(stopErr, "failed to stop failed new container during rollback", map[string]any{zerowrap.FieldEntityID: newContainer.ID})
956 }
957 if removeErr := s.runtime.RemoveContainer(ctx, newContainer.ID, true); removeErr != nil {
958 log.WrapErrWithFields(removeErr, "failed to remove failed new container during rollback", map[string]any{zerowrap.FieldEntityID: newContainer.ID})
959 }
960
961 return false, nil
962 }
963
964 return true, nil
965}
966
967func (s *Service) finalizePreviousContainer(ctx context.Context, domainName string, existing *domain.Container, hasExisting, invalidated bool, newContainerID string) {
968 if !hasExisting {
969 return
970 }
971
972 if invalidated {
973 s.waitForDrain(ctx, existing.ID)
974 }
975
976 s.cleanupOldContainer(ctx, existing, newContainerID, domainName)
977}
978
979func (s *Service) waitForDrain(ctx context.Context, oldContainerID string) {
980 log := zerowrap.FromCtx(ctx)
981
982 s.mu.RLock()
983 cfg := s.config
984 waiter := s.drainWaiter
985 s.mu.RUnlock()
986
987 drainMode := cfg.DrainMode
988 if drainMode == "" {
989 drainMode = "delay"
990 }
991
992 shouldUseInFlight := (drainMode == "inflight" || drainMode == "auto") && waiter != nil
993 if shouldUseInFlight {
994 timeout := cfg.DrainTimeout
995 if timeout == 0 {
996 timeout = 30 * time.Second
997 }
998 drained := waiter.WaitForNoInFlight(ctx, oldContainerID, timeout)
999 if !drained {
1000 log.Warn().
1001 Str("old_container_id", oldContainerID).
1002 Dur("drain_timeout", timeout).
1003 Msg("drain wait timed out; old container may still have in-flight traffic")
1004 }
1005 return
1006 }
1007
1008 drainDelay := 2 * time.Second
1009 if cfg.DrainDelayConfigured {
1010 drainDelay = cfg.DrainDelay
1011 }
1012 if drainDelay <= 0 {
1013 return
1014 }
1015 select {
1016 case <-time.After(drainDelay):
1017 case <-ctx.Done():
1018 }
1019}
1020
1021// restartContainerWithRecovery restarts a container, reconciling stale state if needed.
1022// It returns the (possibly refreshed) container and updated attachment IDs.
1023func (s *Service) restartContainerWithRecovery(ctx context.Context, domainName string, container *domain.Container, attachmentIDs []string) (*domain.Container, []string, error) {
1024 log := zerowrap.FromCtx(ctx)
1025
1026 if err := s.runtime.RestartContainer(ctx, container.ID); err != nil {
1027 if !isContainerNotFoundError(err) {
1028 return nil, nil, log.WrapErr(err, "failed to restart container")
1029 }
1030
1031 // In-memory container ID can become stale after external runtime changes.
1032 // Re-sync and retry once with the latest tracked container.
1033 log.Warn().
1034 Err(err).
1035 Str(zerowrap.FieldEntityID, container.ID).
1036 Msg("tracked container missing during restart, attempting state reconciliation")
1037
1038 if syncErr := s.SyncContainers(ctx); syncErr != nil {
1039 log.Warn().Err(syncErr).Msg("failed to sync container state during restart recovery")
1040 }
1041
1042 s.mu.RLock()
1043 refreshed, refreshedExists := s.containers[domainName]
1044 attachmentIDs = append([]string{}, s.attachments[domainName]...)
1045 s.mu.RUnlock()
1046
1047 if !refreshedExists || refreshed == nil {
1048 return nil, nil, domain.ErrContainerNotFound
1049 }
1050 if err := s.runtime.RestartContainer(ctx, refreshed.ID); err != nil {
1051 return nil, nil, log.WrapErr(err, "failed to restart container after state reconciliation")
1052 }
1053 return refreshed, attachmentIDs, nil
1054 }
1055
1056 return container, attachmentIDs, nil
1057}
1058
1059// Restart restarts a running container for the given domain.
1060func (s *Service) Restart(ctx context.Context, domainName string, withAttachments bool) error {
1061 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1062 zerowrap.FieldLayer: "usecase",
1063 zerowrap.FieldUseCase: "Restart",
1064 "domain": domainName,
1065 })
1066 log := zerowrap.FromCtx(ctx)
1067
1068 s.mu.RLock()
1069 container, exists := s.containers[domainName]
1070 attachments := s.attachments[domainName]
1071 var attachmentIDs []string
1072 if len(attachments) > 0 {
1073 attachmentIDs = make([]string, len(attachments))
1074 copy(attachmentIDs, attachments)
1075 }
1076 s.mu.RUnlock()
1077
1078 if !exists || container == nil {
1079 return domain.ErrContainerNotFound
1080 }
1081
1082 // When attachments are requested, sync runtime state first to get accurate
1083 // attachment tracking, then check if configured attachments are deployed.
1084 // If fewer attachment containers are tracked than configured, fail early with
1085 // guidance rather than restarting the main container into a broken state
1086 // (e.g., missing database hostname).
1087 if withAttachments {
1088 if syncErr := s.SyncContainers(ctx); syncErr != nil {
1089 log.Warn().Err(syncErr).Msg("failed to sync container state before attachment check")
1090 }
1091
1092 // Re-read attachment IDs after sync
1093 s.mu.RLock()
1094 if freshAttachments := s.attachments[domainName]; len(freshAttachments) > 0 {
1095 attachmentIDs = make([]string, len(freshAttachments))
1096 copy(attachmentIDs, freshAttachments)
1097 } else {
1098 attachmentIDs = nil
1099 }
1100 s.mu.RUnlock()
1101
1102 configuredAttachments := s.resolveAttachmentsForDomain(domainName)
1103 if len(configuredAttachments) > 0 && len(attachmentIDs) < len(configuredAttachments) {
1104 missing := len(configuredAttachments) - len(attachmentIDs)
1105 return fmt.Errorf("%w: domain %q has %d configured attachment(s) but only %d deployed (%d missing)",
1106 domain.ErrAttachmentNotDeployed, domainName, len(configuredAttachments), len(attachmentIDs), missing)
1107 }
1108 }
1109
1110 // Restart the main container (with stale-state recovery)
1111 container, attachmentIDs, err := s.restartContainerWithRecovery(ctx, domainName, container, attachmentIDs)
1112 if err != nil {
1113 return err
1114 }
1115 log.Info().Str(zerowrap.FieldEntityID, container.ID).Msg("container restarted")
1116
1117 // Record restart metric
1118 if s.metrics != nil {
1119 s.metrics.ContainerRestarts.Add(ctx, 1, metric.WithAttributes(
1120 attribute.String("domain", domainName),
1121 attribute.String("source", "api"),
1122 ))
1123 }
1124
1125 // Restart attachments if requested
1126 if withAttachments && len(attachmentIDs) > 0 {
1127 for _, attachID := range attachmentIDs {
1128 if err := s.runtime.RestartContainer(ctx, attachID); err != nil {
1129 log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to restart attachment")
1130 continue // Don't fail the whole operation for one attachment
1131 }
1132 log.Info().Str("attachment_id", attachID).Msg("attachment restarted")
1133 }
1134 }
1135
1136 return nil
1137}
1138
1139// Stop stops a running container.
1140func (s *Service) Stop(ctx context.Context, containerID string) error {
1141 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1142 zerowrap.FieldLayer: "usecase",
1143 zerowrap.FieldUseCase: "Stop",
1144 zerowrap.FieldEntityID: containerID,
1145 })
1146 log := zerowrap.FromCtx(ctx)
1147
1148 // Stop log collection before stopping container
1149 if s.logWriter != nil {
1150 if err := s.logWriter.StopLogging(containerID); err != nil {
1151 log.Warn().Err(err).Msg("failed to stop container log collection")
1152 }
1153 }
1154
1155 if err := s.runtime.StopContainer(ctx, containerID); err != nil {
1156 return log.WrapErr(err, "failed to stop container")
1157 }
1158
1159 log.Info().Msg("container stopped")
1160 return nil
1161}
1162
1163// PreviewRemovedRouteCleanup reports retained runtime state for a route that is
1164// no longer configured, without mutating containers or in-memory tracking.
1165func (s *Service) PreviewRemovedRouteCleanup(ctx context.Context, domainName string) (*domain.CleanupReport, error) {
1166 canonicalDomain, ok := domain.CanonicalRouteDomain(domainName)
1167 if !ok {
1168 return &domain.CleanupReport{Domain: domainName}, domain.ErrRouteDomainInvalid
1169 }
1170 domainName = canonicalDomain
1171
1172 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1173 zerowrap.FieldLayer: "usecase",
1174 zerowrap.FieldUseCase: "PreviewRemovedRouteCleanup",
1175 "domain": domainName,
1176 })
1177 log := zerowrap.FromCtx(ctx)
1178
1179 report := &domain.CleanupReport{Domain: domainName}
1180
1181 unlock, err := s.acquireDomainDeployLock(ctx, domainName)
1182 if err != nil {
1183 return report, err
1184 }
1185 defer unlock()
1186
1187 allContainers, err := s.runtime.ListContainers(ctx, true)
1188 if err != nil {
1189 return report, log.WrapErr(err, "failed to list containers for removed route cleanup preview")
1190 }
1191
1192 routeContainers := routeContainersForDomain(allContainers, domainName)
1193 report.PreservedAttachments = preservedAttachmentsForDomain(allContainers, domainName)
1194 if len(report.PreservedAttachments) > 0 {
1195 report.Hints = append(report.Hints, "attachments preserved; remove or purge attachments explicitly if they are no longer needed")
1196 }
1197 for _, c := range routeContainers {
1198 report.OrphanedEntities = append(report.OrphanedEntities, domain.CleanupOrphanedEntity{
1199 Kind: "route_container",
1200 ID: c.ID,
1201 Name: c.Name,
1202 Status: c.Status,
1203 Reason: "route runtime state still exists after route removal",
1204 })
1205 }
1206
1207 return report, nil
1208}
1209
1210// ReconcileRemovedRoute removes active runtime containers for a route that was
1211// removed from configuration while preserving stateful resources for explicit
1212// follow-up cleanup. It intentionally removes only main route containers, never
1213// attachment containers or volumes.
1214func (s *Service) ReconcileRemovedRoute(ctx context.Context, domainName string) (*domain.CleanupReport, error) {
1215 canonicalDomain, ok := domain.CanonicalRouteDomain(domainName)
1216 if !ok {
1217 return &domain.CleanupReport{Domain: domainName}, domain.ErrRouteDomainInvalid
1218 }
1219 domainName = canonicalDomain
1220
1221 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1222 zerowrap.FieldLayer: "usecase",
1223 zerowrap.FieldUseCase: "ReconcileRemovedRoute",
1224 "domain": domainName,
1225 })
1226 log := zerowrap.FromCtx(ctx)
1227
1228 report := &domain.CleanupReport{Domain: domainName}
1229
1230 unlock, err := s.acquireDomainDeployLock(ctx, domainName)
1231 if err != nil {
1232 return report, err
1233 }
1234 defer unlock()
1235
1236 allContainers, err := s.runtime.ListContainers(ctx, true)
1237 if err != nil {
1238 return report, log.WrapErr(err, "failed to list containers for removed route cleanup")
1239 }
1240
1241 routeContainers := routeContainersForDomain(allContainers, domainName)
1242
1243 // Clear in-memory state before stopping containers so the monitor cannot
1244 // interpret the deliberate removal as a crash that should be restarted.
1245 s.clearRouteContainerTracking(ctx, domainName)
1246 if inv := s.proxyCacheInvalidator(); inv != nil {
1247 inv.InvalidateTarget(ctx, domainName)
1248 }
1249
1250 report.PreservedAttachments = preservedAttachmentsForDomain(allContainers, domainName)
1251 if len(report.PreservedAttachments) > 0 {
1252 report.Hints = append(report.Hints, "attachments preserved; remove or purge attachments explicitly if they are no longer needed")
1253 }
1254
1255 if len(routeContainers) == 0 {
1256 report.Warnings = append(report.Warnings, "no active route container found for removed route")
1257 return report, nil
1258 }
1259
1260 s.removeRouteContainersForCleanup(ctx, log, routeContainers, report)
1261
1262 if len(report.PartialFailures) > 0 {
1263 report.Warnings = append(report.Warnings, "route cleanup completed with partial failures; manual follow-up may be required")
1264 }
1265
1266 return report, nil
1267}
1268
1269func routeContainersForDomain(containers []*domain.Container, domainName string) []*domain.Container {
1270 var routeContainers []*domain.Container
1271 for _, c := range containers {
1272 if isManagedRouteContainerForDomain(c, domainName) {
1273 routeContainers = append(routeContainers, c)
1274 }
1275 }
1276 return routeContainers
1277}
1278
1279func preservedAttachmentsForDomain(containers []*domain.Container, domainName string) []domain.CleanupAttachment {
1280 var attachments []domain.CleanupAttachment
1281 for _, c := range containers {
1282 if isPreservedAttachmentForDomain(c, domainName) {
1283 attachments = append(attachments, domain.CleanupAttachment{
1284 Name: c.Name,
1285 Image: c.Image,
1286 ContainerID: c.ID,
1287 Status: c.Status,
1288 Owner: c.Labels[domain.LabelAttachedTo],
1289 Reason: "attachments are preserved by default; remove or purge them explicitly",
1290 })
1291 }
1292 }
1293 return attachments
1294}
1295
1296func (s *Service) removeRouteContainersForCleanup(ctx context.Context, log zerowrap.Logger, containers []*domain.Container, report *domain.CleanupReport) {
1297 for _, c := range containers {
1298 s.stopRouteLogCollectionForCleanup(log, c, report)
1299 s.stopRouteContainerForCleanup(ctx, log, c, report)
1300 if err := s.runtime.RemoveContainer(ctx, c.ID, true); err != nil {
1301 report.PartialFailures = append(report.PartialFailures, domain.CleanupFailure{Action: "remove", Kind: "container", ID: c.ID, Name: c.Name, Error: err.Error()})
1302 log.Warn().Err(err).Str(zerowrap.FieldEntityID, c.ID).Msg("failed to remove removed route container")
1303 continue
1304 }
1305 report.RemovedContainers = append(report.RemovedContainers, domain.CleanupContainer{ID: c.ID, Name: c.Name, Image: c.Image, Status: c.Status})
1306 }
1307}
1308
1309func (s *Service) stopRouteLogCollectionForCleanup(log zerowrap.Logger, c *domain.Container, report *domain.CleanupReport) {
1310 if s.logWriter == nil {
1311 return
1312 }
1313 if err := s.logWriter.StopLogging(c.ID); err != nil {
1314 report.PartialFailures = append(report.PartialFailures, domain.CleanupFailure{Action: "stop_logging", Kind: "container", ID: c.ID, Name: c.Name, Error: err.Error()})
1315 log.Warn().Err(err).Str(zerowrap.FieldEntityID, c.ID).Msg("failed to stop removed route log collection")
1316 }
1317}
1318
1319func (s *Service) stopRouteContainerForCleanup(ctx context.Context, log zerowrap.Logger, c *domain.Container, report *domain.CleanupReport) {
1320 if c.Status != string(domain.ContainerStatusRunning) {
1321 return
1322 }
1323 if err := s.runtime.StopContainer(ctx, c.ID); err != nil {
1324 report.PartialFailures = append(report.PartialFailures, domain.CleanupFailure{Action: "stop", Kind: "container", ID: c.ID, Name: c.Name, Error: err.Error()})
1325 log.Warn().Err(err).Str(zerowrap.FieldEntityID, c.ID).Msg("failed to stop removed route container")
1326 }
1327}
1328
1329func isManagedRouteContainerForDomain(c *domain.Container, domainName string) bool {
1330 if c == nil || c.Labels == nil || c.Labels[domain.LabelManaged] != "true" {
1331 return false
1332 }
1333 if c.Labels[domain.LabelAttachment] == "true" {
1334 return false
1335 }
1336 return c.Labels[domain.LabelRoute] == domainName || c.Labels[domain.LabelDomain] == domainName || c.Name == fmt.Sprintf("gordon-%s", domainName)
1337}
1338
1339func isPreservedAttachmentForDomain(c *domain.Container, domainName string) bool {
1340 if c == nil || c.Labels == nil || c.Labels[domain.LabelManaged] != "true" {
1341 return false
1342 }
1343 return c.Labels[domain.LabelAttachment] == "true" && c.Labels[domain.LabelAttachedTo] == domainName
1344}
1345
1346func (s *Service) proxyCacheInvalidator() out.ProxyCacheInvalidator {
1347 s.mu.RLock()
1348 defer s.mu.RUnlock()
1349 return s.cacheInvalidator
1350}
1351
1352func (s *Service) clearRouteContainerTracking(ctx context.Context, domainName string) {
1353 s.mu.Lock()
1354 removed := false
1355 if _, exists := s.containers[domainName]; exists {
1356 delete(s.containers, domainName)
1357 if s.managedCount > 0 {
1358 s.managedCount--
1359 }
1360 removed = true
1361 }
1362 s.mu.Unlock()
1363
1364 if removed && s.metrics != nil {
1365 s.metrics.ManagedContainers.Add(ctx, -1)
1366 }
1367}
1368
1369// Remove removes a container.
1370func (s *Service) Remove(ctx context.Context, containerID string, force bool) error {
1371 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1372 zerowrap.FieldLayer: "usecase",
1373 zerowrap.FieldUseCase: "Remove",
1374 zerowrap.FieldEntityID: containerID,
1375 })
1376 log := zerowrap.FromCtx(ctx)
1377
1378 // Stop log collection before removing container
1379 if s.logWriter != nil {
1380 if err := s.logWriter.StopLogging(containerID); err != nil {
1381 log.Warn().Err(err).Msg("failed to stop container log collection")
1382 }
1383 }
1384
1385 // Find domain and attachment IDs for cleanup
1386 containerDomain, attachmentIDs, cfg := s.findContainerContext(containerID)
1387
1388 if err := s.runtime.RemoveContainer(ctx, containerID, force); err != nil {
1389 return log.WrapErr(err, "failed to remove container")
1390 }
1391
1392 // Cleanup volumes
1393 if containerDomain != "" && !cfg.VolumePreserve {
1394 if err := s.cleanupVolumesForDomain(ctx, containerDomain); err != nil {
1395 log.WrapErrWithFields(err, "failed to cleanup volumes", map[string]any{"domain": containerDomain})
1396 }
1397 }
1398
1399 // Clean up attachment containers
1400 s.removeAttachments(ctx, attachmentIDs)
1401
1402 // Remove from tracking
1403 s.mu.Lock()
1404 var removedDomain string
1405 for d, c := range s.containers {
1406 if c.ID == containerID {
1407 delete(s.containers, d)
1408 removedDomain = d
1409 s.managedCount--
1410 log.Info().Str("domain", d).Msg("container removed")
1411 break
1412 }
1413 }
1414 if removedDomain != "" {
1415 delete(s.attachments, removedDomain)
1416 }
1417 s.mu.Unlock()
1418
1419 // Decrement managed container count
1420 if removedDomain != "" && s.metrics != nil {
1421 s.metrics.ManagedContainers.Add(ctx, -1)
1422 }
1423
1424 // Note: We intentionally do NOT clean up deployMu entries to avoid a race condition:
1425 // If Remove deletes the mutex entry while a concurrent Deploy is acquiring the lock,
1426 // the Deploy would create a fresh mutex, breaking serialization and allowing concurrent
1427 // deploys to the same domain. The memory footprint is acceptable (one small struct per
1428 // domain ever deployed), and this is the safer choice for correctness.
1429
1430 // Cleanup network
1431 if removedDomain != "" && cfg.NetworkIsolation {
1432 networkName := s.getNetworkForApp(removedDomain)
1433 if err := s.cleanupNetworkIfEmpty(ctx, networkName); err != nil {
1434 log.WrapErrWithFields(err, "failed to cleanup network", map[string]any{"network": networkName})
1435 }
1436 }
1437
1438 return nil
1439}
1440
1441// findContainerContext looks up the domain, attachment IDs, and config snapshot for a container.
1442func (s *Service) findContainerContext(containerID string) (string, []string, Config) {
1443 s.mu.RLock()
1444 defer s.mu.RUnlock()
1445 cfg := s.config
1446 var containerDomain string
1447 for d, c := range s.containers {
1448 if c.ID == containerID {
1449 containerDomain = d
1450 break
1451 }
1452 }
1453 var attachmentIDs []string
1454 if containerDomain != "" {
1455 attachmentIDs = append(attachmentIDs, s.attachments[containerDomain]...)
1456 }
1457 return containerDomain, attachmentIDs, cfg
1458}
1459
1460// removeAttachments stops and removes attachment containers best-effort.
1461func (s *Service) removeAttachments(ctx context.Context, attachmentIDs []string) {
1462 log := zerowrap.FromCtx(ctx)
1463 for _, attachID := range attachmentIDs {
1464 if s.logWriter != nil {
1465 if err := s.logWriter.StopLogging(attachID); err != nil {
1466 log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to stop attachment log collection")
1467 }
1468 }
1469 if err := s.runtime.StopContainer(ctx, attachID); err != nil {
1470 log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to stop attachment container")
1471 }
1472 if err := s.runtime.RemoveContainer(ctx, attachID, true); err != nil {
1473 log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to remove attachment container")
1474 }
1475 }
1476}
1477
1478// Get retrieves a container by domain.
1479func (s *Service) Get(_ context.Context, domainName string) (*domain.Container, bool) {
1480 s.mu.RLock()
1481 defer s.mu.RUnlock()
1482 container, exists := s.containers[domainName]
1483 return container, exists
1484}
1485
1486// List returns all managed containers.
1487func (s *Service) List(_ context.Context) map[string]*domain.Container {
1488 s.mu.RLock()
1489 defer s.mu.RUnlock()
1490
1491 result := make(map[string]*domain.Container, len(s.containers))
1492 maps.Copy(result, s.containers)
1493 return result
1494}
1495
1496// ListRoutesWithDetails returns routes with network and attachment info.
1497// Note: Container map is copied under lock, then external calls are made without lock.
1498// If containers are removed between copy and runtime calls, errors are handled gracefully
1499// (network becomes empty string). This trade-off avoids holding locks during I/O.
1500func (s *Service) ListRoutesWithDetails(ctx context.Context) []domain.RouteInfo {
1501 s.mu.RLock()
1502 containers := make(map[string]*domain.Container, len(s.containers))
1503 maps.Copy(containers, s.containers)
1504 s.mu.RUnlock()
1505
1506 // Fetch all attachments once to avoid N+1 queries
1507 attachmentsByDomain := s.getAllAttachments(ctx)
1508
1509 results := make([]domain.RouteInfo, 0, len(containers))
1510 for domainName, container := range containers {
1511 network := ""
1512 image := ""
1513 containerID := ""
1514 status := ""
1515 if container != nil {
1516 containerID = container.ID
1517 status = container.Status
1518 image = container.Image
1519 if container.Labels != nil {
1520 if labelImage, ok := container.Labels[domain.LabelImage]; ok && labelImage != "" {
1521 image = labelImage
1522 }
1523 }
1524 // Strip registry domain prefix from image for cleaner display
1525 image = s.stripRegistryPrefix(image)
1526 if networkName, err := s.runtime.GetContainerNetwork(ctx, container.ID); err == nil {
1527 network = networkName
1528 }
1529 }
1530
1531 results = append(results, domain.RouteInfo{
1532 Domain: domainName,
1533 Image: image,
1534 ContainerID: containerID,
1535 ContainerStatus: status,
1536 Network: network,
1537 Attachments: attachmentsByDomain[domainName],
1538 })
1539 }
1540
1541 return results
1542}
1543
1544// stripRegistryPrefix removes the configured current or legacy Gordon registry
1545// domain prefix from an image reference. For example,
1546// "reg.example.com/myapp:latest" becomes "myapp:latest" when
1547// "reg.example.com" is a current or legacy Gordon registry domain.
1548func (s *Service) stripRegistryPrefix(image string) string {
1549 s.mu.RLock()
1550 cfg := s.config
1551 s.mu.RUnlock()
1552
1553 return domain.StripKnownGordonRegistry(image, cfg.RegistryDomain, cfg.LegacyRegistryDomains)
1554}
1555
1556// ListAttachments returns attachments for a domain.
1557func (s *Service) ListAttachments(ctx context.Context, domainName string) []domain.Attachment {
1558 return s.getAttachmentsForDomain(ctx, domainName)
1559}
1560
1561// ListOrphanedAttachments returns managed attachment containers no longer configured.
1562func (s *Service) ListOrphanedAttachments(ctx context.Context) ([]domain.CleanupAttachment, error) {
1563 containers, err := s.runtime.ListContainers(ctx, true)
1564 if err != nil {
1565 return nil, fmt.Errorf("failed to list containers with runtime for orphaned attachments: %w", err)
1566 }
1567 return s.orphanedAttachmentsFromContainers(containers), nil
1568}
1569
1570// CleanupOrphanedAttachments optionally stops/removes orphaned attachment containers.
1571// When owner is non-empty, cleanup is scoped to that attachment owner.
1572func (s *Service) CleanupOrphanedAttachments(ctx context.Context, owner string, stop bool) (*domain.CleanupReport, error) {
1573 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1574 zerowrap.FieldLayer: "usecase",
1575 zerowrap.FieldUseCase: "CleanupOrphanedAttachments",
1576 "owner": owner,
1577 })
1578 log := zerowrap.FromCtx(ctx)
1579 report := &domain.CleanupReport{Domain: owner}
1580
1581 containers, err := s.runtime.ListContainers(ctx, true)
1582 if err != nil {
1583 return report, fmt.Errorf("failed to list containers with runtime for orphaned attachment cleanup: %w", err)
1584 }
1585 orphans := filterAttachmentContainersByOwner(s.orphanedAttachmentContainers(containers), owner)
1586 if !stop {
1587 report.PreservedAttachments = cleanupAttachmentsFromContainers(orphans)
1588 if len(report.PreservedAttachments) > 0 {
1589 report.Hints = append(report.Hints, "orphaned attachments preserved; rerun with --stop to stop/remove containers while preserving volumes")
1590 }
1591 return report, nil
1592 }
1593
1594 for _, c := range orphans {
1595 s.stopRouteLogCollectionForCleanup(log, c, report)
1596 s.stopRouteContainerForCleanup(ctx, log, c, report)
1597 if err := s.runtime.RemoveContainer(ctx, c.ID, true); err != nil {
1598 report.PartialFailures = append(report.PartialFailures, domain.CleanupFailure{Action: "remove", Kind: "attachment", ID: c.ID, Name: c.Name, Error: err.Error()})
1599 log.Warn().Err(err).Str(zerowrap.FieldEntityID, c.ID).Msg("failed to remove orphaned attachment")
1600 continue
1601 }
1602 report.RemovedContainers = append(report.RemovedContainers, domain.CleanupContainer{ID: c.ID, Name: c.Name, Image: attachmentImage(c), Status: c.Status})
1603 s.untrackAttachmentContainer(c.Labels[domain.LabelAttachedTo], c.ID)
1604 }
1605 if len(report.RemovedContainers) > 0 {
1606 report.Hints = append(report.Hints, "attachment volumes and data were preserved")
1607 }
1608 if len(report.PartialFailures) > 0 {
1609 report.Warnings = append(report.Warnings, "attachment cleanup completed with partial failures; manual follow-up may be required")
1610 }
1611 return report, nil
1612}
1613
1614func (s *Service) orphanedAttachmentsFromContainers(containers []*domain.Container) []domain.CleanupAttachment {
1615 return cleanupAttachmentsFromContainers(s.orphanedAttachmentContainers(containers))
1616}
1617
1618func (s *Service) orphanedAttachmentContainers(containers []*domain.Container) []*domain.Container {
1619 var out []*domain.Container
1620 for _, c := range containers {
1621 if !isManagedAttachment(c) {
1622 continue
1623 }
1624 owner := c.Labels[domain.LabelAttachedTo]
1625 image := attachmentImage(c)
1626 if owner == "" || image == "" || !s.isAttachmentConfiguredForOwner(owner, image) {
1627 out = append(out, c)
1628 }
1629 }
1630 return out
1631}
1632
1633func filterAttachmentContainersByOwner(containers []*domain.Container, owner string) []*domain.Container {
1634 if owner == "" {
1635 return containers
1636 }
1637 var out []*domain.Container
1638 for _, c := range containers {
1639 if c != nil && c.Labels != nil && strings.EqualFold(c.Labels[domain.LabelAttachedTo], owner) {
1640 out = append(out, c)
1641 }
1642 }
1643 return out
1644}
1645
1646func cleanupAttachmentsFromContainers(containers []*domain.Container) []domain.CleanupAttachment {
1647 attachments := make([]domain.CleanupAttachment, 0, len(containers))
1648 for _, c := range containers {
1649 attachments = append(attachments, domain.CleanupAttachment{
1650 Name: c.Name,
1651 Image: attachmentImage(c),
1652 ContainerID: c.ID,
1653 Status: c.Status,
1654 Owner: c.Labels[domain.LabelAttachedTo],
1655 Reason: "attachment container is no longer configured",
1656 })
1657 }
1658 return attachments
1659}
1660
1661func isManagedAttachment(c *domain.Container) bool {
1662 return c != nil && c.Labels != nil && c.Labels[domain.LabelManaged] == "true" && c.Labels[domain.LabelAttachment] == "true"
1663}
1664
1665func attachmentImage(c *domain.Container) string {
1666 if c == nil {
1667 return ""
1668 }
1669 if c.Labels != nil && c.Labels[domain.LabelImage] != "" {
1670 return c.Labels[domain.LabelImage]
1671 }
1672 return c.Image
1673}
1674
1675func (s *Service) isAttachmentConfiguredForOwner(owner, image string) bool {
1676 return slices.Contains(s.resolveAttachmentsForDomain(owner), image)
1677}
1678
1679func (s *Service) untrackAttachmentContainer(owner, containerID string) {
1680 s.mu.Lock()
1681 defer s.mu.Unlock()
1682 ids := s.attachments[owner]
1683 for i, id := range ids {
1684 if id == containerID {
1685 s.attachments[owner] = slices.Delete(ids, i, i+1)
1686 break
1687 }
1688 }
1689 if len(s.attachments[owner]) == 0 {
1690 delete(s.attachments, owner)
1691 }
1692}
1693
1694// ListNetworks returns Gordon-managed networks.
1695func (s *Service) ListNetworks(ctx context.Context) ([]*domain.NetworkInfo, error) {
1696 s.mu.RLock()
1697 cfg := s.config
1698 s.mu.RUnlock()
1699
1700 networks, err := s.runtime.ListNetworks(ctx)
1701 if err != nil {
1702 return nil, err
1703 }
1704
1705 var filtered []*domain.NetworkInfo
1706 for _, network := range networks {
1707 if strings.HasPrefix(network.Name, cfg.NetworkPrefix+"-") && network.Labels[domain.LabelManaged] == "true" {
1708 filtered = append(filtered, network)
1709 }
1710 }
1711
1712 return filtered, nil
1713}
1714
1715// HealthCheck performs health checks on all containers.
1716func (s *Service) HealthCheck(ctx context.Context) map[string]bool {
1717 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1718 zerowrap.FieldLayer: "usecase",
1719 zerowrap.FieldUseCase: "HealthCheck",
1720 })
1721 log := zerowrap.FromCtx(ctx)
1722
1723 // Copy container IDs under lock, then release before making Docker API calls
1724 s.mu.RLock()
1725 snapshot := make(map[string]string, len(s.containers))
1726 for d, c := range s.containers {
1727 snapshot[d] = c.ID
1728 }
1729 s.mu.RUnlock()
1730
1731 health := make(map[string]bool, len(snapshot))
1732 for d, id := range snapshot {
1733 running, err := s.runtime.IsContainerRunning(ctx, id)
1734 if err != nil {
1735 log.WrapErrWithFields(err, "health check failed", map[string]any{"domain": d, zerowrap.FieldEntityID: id})
1736 health[d] = false
1737 } else {
1738 health[d] = running
1739 }
1740 }
1741 return health
1742}
1743
1744// SyncContainers synchronizes containers with runtime state.
1745func isTrackedManagedContainerStatus(status string) bool {
1746 return status == "" || status == string(domain.ContainerStatusRunning) || status == "restarting"
1747}
1748
1749func (s *Service) SyncContainers(ctx context.Context) error {
1750 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1751 zerowrap.FieldLayer: "usecase",
1752 zerowrap.FieldUseCase: "SyncContainers",
1753 })
1754 log := zerowrap.FromCtx(ctx)
1755
1756 // List containers without holding lock to avoid blocking during the runtime call.
1757 // Use all=true so we can rebuild the running-container maps from a single
1758 // snapshot without a second list call.
1759 allContainers, err := s.runtime.ListContainers(ctx, true)
1760 if err != nil {
1761 return log.WrapErr(err, "failed to list containers")
1762 }
1763
1764 managed := make(map[string]*domain.Container)
1765 attachments := make(map[string][]string)
1766 for _, c := range allContainers {
1767 if !isTrackedManagedContainerStatus(c.Status) {
1768 continue
1769 }
1770 if c.Labels == nil || c.Labels[domain.LabelManaged] != "true" {
1771 continue
1772 }
1773 if c.Labels[domain.LabelAttachment] == "true" {
1774 owner := c.Labels[domain.LabelAttachedTo]
1775 if owner != "" {
1776 attachments[owner] = append(attachments[owner], c.ID)
1777 }
1778 continue
1779 }
1780 if d, ok := c.Labels[domain.LabelDomain]; ok {
1781 managed[d] = c
1782 }
1783 }
1784
1785 s.mu.Lock()
1786 s.containers = managed
1787 s.attachments = attachments
1788 newCount := int64(len(managed))
1789 delta := newCount - s.managedCount
1790 s.managedCount = newCount
1791 s.mu.Unlock()
1792
1793 // Report initial/delta managed container count to OTel UpDownCounter.
1794 if delta != 0 && s.metrics != nil {
1795 s.metrics.ManagedContainers.Add(ctx, delta)
1796 }
1797
1798 log.Info().Int(zerowrap.FieldCount, len(managed)).Msg("container state synchronized")
1799 return nil
1800}
1801
1802// AutoStart starts containers for the provided routes that aren't running.
1803// It skips readiness checks to avoid blocking boot; the background monitor
1804// handles crash recovery.
1805func (s *Service) AutoStart(ctx context.Context, routes []domain.Route) error {
1806 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1807 zerowrap.FieldLayer: "usecase",
1808 zerowrap.FieldUseCase: "AutoStart",
1809 })
1810 log := zerowrap.FromCtx(ctx)
1811
1812 log.Info().Int("route_count", len(routes)).Msg("auto-starting containers for configured routes")
1813
1814 // Filter out routes that already have a running container.
1815 var pending []domain.Route
1816 skipped := 0
1817 for _, route := range routes {
1818 if _, exists := s.Get(ctx, route.Domain); exists {
1819 log.Debug().Str("domain", route.Domain).Msg("container already running, skipping")
1820 skipped++
1821 } else {
1822 pending = append(pending, route)
1823 }
1824 }
1825
1826 if len(pending) == 0 {
1827 log.Info().Int("skipped", skipped).Msg("auto-start completed, all containers already running")
1828 return nil
1829 }
1830
1831 // Deploy all pending routes concurrently with readiness checks skipped.
1832 deployCtx := domain.WithSkipReadiness(ctx)
1833
1834 type result struct {
1835 route domain.Route
1836 err error
1837 }
1838 results := make(chan result, len(pending))
1839 sem := make(chan struct{}, 4) // limit concurrency
1840
1841 for _, route := range pending {
1842 sem <- struct{}{}
1843 go func(r domain.Route) {
1844 defer func() { <-sem }()
1845
1846 log.Info().Str("domain", r.Domain).Str("image", r.Image).Msg("auto-starting container for route")
1847
1848 if _, err := s.Deploy(deployCtx, r); err != nil {
1849 results <- result{route: r, err: err}
1850 return
1851 }
1852 results <- result{route: r}
1853 }(route)
1854 }
1855
1856 var started, errCount int
1857 for i := 0; i < len(pending); i++ {
1858 res := <-results
1859 if res.err != nil {
1860 log.Warn().Err(res.err).Str("domain", res.route.Domain).Msg("failed to auto-start container")
1861 errCount++
1862 } else {
1863 started++
1864 }
1865 }
1866
1867 log.Info().
1868 Int("started", started).
1869 Int("skipped", skipped).
1870 Int("errors", errCount).
1871 Msg("auto-start completed")
1872
1873 if errCount > 0 {
1874 return fmt.Errorf("auto-start completed with %d errors", errCount)
1875 }
1876 return nil
1877}
1878
1879// Shutdown gracefully shuts down all managed containers.
1880func (s *Service) Shutdown(ctx context.Context) error {
1881 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1882 zerowrap.FieldLayer: "usecase",
1883 zerowrap.FieldUseCase: "Shutdown",
1884 })
1885 log := zerowrap.FromCtx(ctx)
1886 log.Info().Msg("shutting down container manager...")
1887
1888 // Wait for any in-flight background cleanup goroutines to finish
1889 s.cleanupWg.Wait()
1890
1891 // Containers are left running across Gordon restarts.
1892 // SyncContainers + AutoStart will pick them back up on next boot.
1893
1894 // Close log writer to stop all log collection
1895 if s.logWriter != nil {
1896 if err := s.logWriter.Close(); err != nil {
1897 log.Warn().Err(err).Msg("failed to close container log writer")
1898 }
1899 }
1900
1901 log.Info().Msg("container manager shutdown complete")
1902 return nil
1903}
1904
1905// WaitForCleanup blocks until all background cleanup goroutines complete.
1906// Intended for use in tests to avoid mock assertion races.
1907func (s *Service) WaitForCleanup() {
1908 s.cleanupWg.Wait()
1909}
1910
1911// StartMonitor begins background monitoring of tracked containers.
1912// The monitor restarts crashed containers and detects crash loops.
1913// Safe to call multiple times; subsequent calls are no-ops.
1914func (s *Service) StartMonitor(ctx context.Context) {
1915 s.mu.Lock()
1916 if s.monitor != nil {
1917 s.mu.Unlock()
1918 return
1919 }
1920 s.monitor = newMonitor(s)
1921 m := s.monitor
1922 m.Start(ctx)
1923 s.mu.Unlock()
1924}
1925
1926// StopMonitor stops the background container monitor.
1927// Safe to call multiple times; subsequent calls are no-ops.
1928func (s *Service) StopMonitor() {
1929 s.mu.Lock()
1930 m := s.monitor
1931 s.monitor = nil
1932 s.mu.Unlock()
1933 if m != nil {
1934 m.Stop()
1935 }
1936}
1937
1938// UpdateConfig updates the service configuration.
1939func (s *Service) UpdateConfig(config Config) {
1940 s.mu.Lock()
1941 s.config = config
1942 s.mu.Unlock()
1943}
1944
1945// UpdateAttachments updates only the attachment configuration in the service.
1946// This is called after a config reload to propagate attachment changes without restart.
1947// The incoming map is deep-copied so external callers cannot mutate service state.
1948func (s *Service) UpdateAttachments(attachments map[string][]string) {
1949 var copied map[string][]string
1950 if attachments != nil {
1951 copied = make(map[string][]string, len(attachments))
1952 for k, v := range attachments {
1953 sl := make([]string, len(v))
1954 copy(sl, v)
1955 copied[k] = sl
1956 }
1957 }
1958 s.mu.Lock()
1959 s.config.Attachments = copied
1960 s.mu.Unlock()
1961}
1962
1963// Helper methods
1964
1965// cleanupFailedContainer stops and removes a container that failed to start properly.
1966func (s *Service) cleanupFailedContainer(ctx context.Context, containerID string) {
1967 cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), failedContainerCleanupTimeout)
1968 defer cancel()
1969
1970 log := zerowrap.FromCtx(cleanupCtx)
1971 if err := s.runtime.StopContainer(cleanupCtx, containerID); err != nil {
1972 log.Warn().Err(err).Str(zerowrap.FieldEntityID, containerID).Msg("failed to stop container after failure")
1973 }
1974 if err := s.runtime.RemoveContainer(cleanupCtx, containerID, true); err != nil {
1975 log.Warn().Err(err).Str(zerowrap.FieldEntityID, containerID).Msg("failed to remove container after failure")
1976 }
1977}
1978
1979// cleanupOldContainer stops and removes an old container after zero-downtime switch.
1980// It also renames the new container to the canonical name.
1981func (s *Service) cleanupOldContainer(ctx context.Context, old *domain.Container, newContainerID, domainName string) {
1982 log := zerowrap.FromCtx(ctx)
1983 log.Info().Str(zerowrap.FieldEntityID, old.ID).Msg("stopping old container after zero-downtime switch")
1984
1985 if s.logWriter != nil {
1986 if err := s.logWriter.StopLogging(old.ID); err != nil {
1987 log.Warn().Err(err).Str(zerowrap.FieldEntityID, old.ID).Msg("failed to stop logging for old container")
1988 }
1989 }
1990
1991 if err := s.runtime.StopContainer(ctx, old.ID); err != nil {
1992 log.Warn().Err(err).Str(zerowrap.FieldEntityID, old.ID).Msg("failed to stop old container")
1993 }
1994
1995 if err := s.runtime.RemoveContainer(ctx, old.ID, true); err != nil {
1996 log.Warn().Err(err).Str(zerowrap.FieldEntityID, old.ID).Msg("failed to remove old container")
1997 }
1998
1999 // Rename new container to canonical name
2000 canonicalName := fmt.Sprintf("gordon-%s", domainName)
2001 if err := s.runtime.RenameContainer(ctx, newContainerID, canonicalName); err != nil {
2002 log.Warn().Err(err).Str("canonical_name", canonicalName).Msg("failed to rename container to canonical name")
2003 }
2004}
2005
2006// startLogCollection starts log collection for a container in the background.
2007// Errors are logged but don't fail the calling operation.
2008func (s *Service) startLogCollection(ctx context.Context, containerID, domainName string) {
2009 if s.logWriter == nil {
2010 return
2011 }
2012
2013 log := zerowrap.FromCtx(ctx)
2014
2015 // Use context.WithoutCancel so the log stream outlives the HTTP request.
2016 // Without this, the Docker log stream closes when the deploy response completes.
2017 bgCtx := context.WithoutCancel(ctx)
2018
2019 logStream, err := s.runtime.GetContainerLogs(bgCtx, containerID, true)
2020 if err != nil {
2021 log.Warn().Err(err).Msg("failed to get container logs for collection")
2022 return
2023 }
2024
2025 if err := s.logWriter.StartLogging(bgCtx, containerID, domainName, logStream); err != nil {
2026 log.Warn().Err(err).Msg("failed to start container log collection")
2027 logStream.Close()
2028 }
2029}
2030
2031func (s *Service) buildValidatedImageRef(ctx context.Context, image string) (string, error) {
2032 imageRef := s.buildImageRef(image)
2033 if err := s.validateImageRef(ctx, imageRef); err != nil {
2034 return "", err
2035 }
2036 return imageRef, nil
2037}
2038
2039func (s *Service) buildImageRef(image string) string {
2040 s.mu.RLock()
2041 cfg := s.config
2042 s.mu.RUnlock()
2043
2044 if !cfg.RegistryAuthEnabled || cfg.RegistryDomain == "" {
2045 return image
2046 }
2047
2048 // Normalize registry domain by trimming any trailing slash
2049 reg := strings.TrimSuffix(cfg.RegistryDomain, "/")
2050
2051 // Check if image already has the registry domain prefix
2052 prefix := reg + "/"
2053 if strings.HasPrefix(image, prefix) {
2054 return image
2055 }
2056
2057 // Check if image already has an explicit registry (host:port/ or host.domain/).
2058 // We need to detect patterns like:
2059 // - "docker.io/library/nginx:latest" (has dot and slash)
2060 // - "localhost:5001/myapp:latest" (has host:port/ pattern)
2061 // - "myregistry:5000/app:v1" (has host:port/ pattern)
2062 // - "[fd00::1]:5000/app:v1" (IPv6 with port)
2063 if hasExplicitRegistry(image) {
2064 return image
2065 }
2066
2067 return fmt.Sprintf("%s/%s", reg, image)
2068}
2069
2070// hasExplicitRegistry checks if an image reference already includes an explicit registry.
2071// Returns true for patterns like "docker.io/image", "localhost:5000/image", "localhost/image",
2072// "registry:8080/image", "[fd00::1]/image", "[fd00::1]:5000/image".
2073func hasExplicitRegistry(image string) bool {
2074 // Find the first slash which separates registry from image name
2075 slashIdx := strings.Index(image, "/")
2076 if slashIdx == -1 {
2077 return false // No slash means no registry prefix (e.g., "myapp:latest")
2078 }
2079
2080 // Extract the part before the first slash (potential registry)
2081 registryPart := image[:slashIdx]
2082
2083 // Check for bracketed IPv6 address (e.g., "[fd00::1]" or "[fd00::1]:5000")
2084 if strings.HasPrefix(registryPart, "[") {
2085 // Look for closing bracket
2086 if closeBracket := strings.Index(registryPart, "]"); closeBracket != -1 {
2087 // Valid bracketed IPv6: either ends at ] or has ]:port
2088 return true
2089 }
2090 }
2091
2092 // Check for localhost (with or without port)
2093 if registryPart == "localhost" {
2094 return true
2095 }
2096
2097 // Check for host:port pattern (e.g., "localhost:5000", "registry:8080")
2098 if colonIdx := strings.LastIndex(registryPart, ":"); colonIdx != -1 {
2099 port := registryPart[colonIdx+1:]
2100 // If everything after colon is digits, it's a port
2101 if len(port) > 0 && isNumeric(port) {
2102 return true
2103 }
2104 }
2105
2106 // Check for domain pattern (has a dot, e.g., "docker.io", "gcr.io")
2107 if strings.Contains(registryPart, ".") {
2108 return true
2109 }
2110
2111 return false
2112}
2113
2114func (s *Service) validateImageRef(ctx context.Context, imageRef string) error {
2115 if !hasExplicitRegistry(imageRef) {
2116 return nil
2117 }
2118 return s.validateExternalImageRef(ctx, imageRef, imageRegistry(imageRef))
2119}
2120
2121func (s *Service) validateImagePullRef(ctx context.Context, imageRef string) error {
2122 return s.validateExternalImageRef(ctx, imageRef, imageRegistryForPolicy(imageRef))
2123}
2124
2125func (s *Service) validateExternalImageRef(ctx context.Context, imageRef, registry string) error {
2126 s.mu.RLock()
2127 cfg := s.config
2128 s.mu.RUnlock()
2129
2130 if registry == "" {
2131 return fmt.Errorf("image %q rejected: invalid image reference", imageRef)
2132 }
2133
2134 if domain.IsGordonRegistryImageRef(imageRef, cfg.RegistryDomain, cfg.LegacyRegistryDomains) || sameRegistry(registry, cfg.RegistryDomain) {
2135 return nil
2136 }
2137 dangerous, err := isDangerousRegistryHost(ctx, imageRegistryHost(registry))
2138 if err != nil {
2139 return fmt.Errorf("image %q rejected: registry %q could not be verified: %w", imageRef, registry, err)
2140 }
2141 if dangerous {
2142 return fmt.Errorf("image %q rejected: registry %q is not allowed", imageRef, registry)
2143 }
2144 if len(cfg.AllowedRegistries) == 0 {
2145 return fmt.Errorf("image %q rejected: registry %q is not in images.allowed_registries", imageRef, registry)
2146 }
2147 for _, allowed := range cfg.AllowedRegistries {
2148 if sameRegistry(registry, allowed) {
2149 if cfg.RequireImageDigest && !isValidSha256DigestRef(imageRef) {
2150 return fmt.Errorf("image %q rejected: digest reference required", imageRef)
2151 }
2152 return nil
2153 }
2154 }
2155 return fmt.Errorf("image %q rejected: registry %q is not in images.allowed_registries", imageRef, registry)
2156}
2157
2158func imageRegistry(imageRef string) string {
2159 slashIdx := strings.Index(imageRef, "/")
2160 if slashIdx == -1 {
2161 return ""
2162 }
2163 return normalizeRegistry(imageRef[:slashIdx])
2164}
2165
2166func imageRegistryForPolicy(imageRef string) string {
2167 if hasExplicitRegistry(imageRef) {
2168 return imageRegistry(imageRef)
2169 }
2170 return "docker.io"
2171}
2172
2173// normalizeRegistry normalizes a registry string for comparison purposes only.
2174// This function is used by sameRegistry to determine if two registry strings
2175// refer to the same registry. The output intentionally strips IPv6 brackets
2176// (e.g., "[fd00::1]:5000" -> "fd00::1:5000") which is acceptable for equality
2177// checks but the result MUST NOT be used to construct or emit valid registry URLs.
2178func normalizeRegistry(registry string) string {
2179 registry = strings.ToLower(strings.TrimSuffix(strings.TrimSpace(registry), "."))
2180 registry = strings.TrimSuffix(registry, "/")
2181 if strings.HasPrefix(registry, "[") {
2182 if end := strings.Index(registry, "]"); end != -1 {
2183 host := strings.Trim(registry[:end+1], "[]")
2184 port := registry[end+1:]
2185 return host + port
2186 }
2187 }
2188 return registry
2189}
2190
2191func imageRegistryHost(registry string) string {
2192 if ip := net.ParseIP(registry); ip != nil {
2193 return registry
2194 }
2195 if h, _, err := net.SplitHostPort(registry); err == nil {
2196 return strings.Trim(h, "[]")
2197 }
2198 if strings.HasPrefix(registry, "[") && strings.Contains(registry, "]") {
2199 return strings.TrimPrefix(strings.Split(registry, "]")[0], "[")
2200 }
2201 if colon := strings.LastIndex(registry, ":"); colon != -1 && isNumeric(registry[colon+1:]) {
2202 return registry[:colon]
2203 }
2204 return registry
2205}
2206
2207func sameRegistry(a, b string) bool {
2208 return normalizeRegistry(a) == normalizeRegistry(b)
2209}
2210
2211func isDangerousRegistryHost(ctx context.Context, host string) (bool, error) {
2212 host = strings.ToLower(strings.TrimSuffix(host, "."))
2213 if host == "localhost" || host == "metadata.google.internal" {
2214 return true, nil
2215 }
2216 if ip := net.ParseIP(host); ip != nil {
2217 return isDangerousRegistryIP(ip), nil
2218 }
2219 addrs, err := net.DefaultResolver.LookupIPAddr(ctx, host)
2220 if err != nil {
2221 return false, err
2222 }
2223 for _, addr := range addrs {
2224 if isDangerousRegistryIP(addr.IP) {
2225 return true, nil
2226 }
2227 }
2228 return false, nil
2229}
2230
2231func isDangerousRegistryIP(ip net.IP) bool {
2232 return ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsUnspecified()
2233}
2234
2235func applySecurityProfile(config *domain.ContainerConfig, cfg Config) {
2236 if strings.EqualFold(cfg.SecurityProfile, "strict") {
2237 config.ReadOnlyRootFS = true
2238 config.CapDrop = []string{"ALL"}
2239 config.CapAdd = []string{"NET_BIND_SERVICE"}
2240 }
2241}
2242
2243// isNumeric checks if a string contains only digits.
2244func isNumeric(s string) bool {
2245 for _, c := range s {
2246 if c < '0' || c > '9' {
2247 return false
2248 }
2249 }
2250 return true
2251}
2252
2253func normalizePullPolicy(policy string) string {
2254 switch strings.ToLower(strings.TrimSpace(policy)) {
2255 case PullPolicyAlways:
2256 return PullPolicyAlways
2257 case PullPolicyIfTagChanged:
2258 return PullPolicyIfTagChanged
2259 case PullPolicyIfNotPresent:
2260 return PullPolicyIfNotPresent
2261 default:
2262 return PullPolicyIfNotPresent
2263 }
2264}
2265
2266func isDigestRef(imageRef string) bool {
2267 return isValidSha256DigestRef(imageRef)
2268}
2269
2270func isValidSha256DigestRef(imageRef string) bool {
2271 _, digest, ok := strings.Cut(imageRef, "@sha256:")
2272 if !ok || len(digest) != 64 {
2273 return false
2274 }
2275 _, err := hex.DecodeString(digest)
2276 return err == nil
2277}
2278
2279func (s *Service) pullRefForDeploy(ctx context.Context, imageRef string) (string, bool) {
2280 if !domain.IsInternalDeploy(ctx) {
2281 return imageRef, false
2282 }
2283 s.mu.RLock()
2284 cfg := s.config
2285 s.mu.RUnlock()
2286 return rewriteToLocalRegistry(imageRef, cfg.RegistryDomain, cfg.LegacyRegistryDomains, cfg.RegistryPort), true
2287}
2288
2289// ensureImage ensures the image is available locally, pulling if needed.
2290// Returns the image reference to use for container operations:
2291// - For digest references (@sha256:...), returns the pullRef since Docker can't tag digests
2292// - For tagged images, returns the original imageRef after tagging the pulled image
2293func (s *Service) ensureImage(ctx context.Context, imageRef string) (string, error) {
2294 ctx, span := tracer.Start(ctx, "container.ensure_image",
2295 trace.WithAttributes(attribute.String("image", imageRef)))
2296 defer span.End()
2297
2298 ctx = zerowrap.CtxWithField(ctx, "image", imageRef)
2299 log := zerowrap.FromCtx(ctx)
2300
2301 // Determine if this is an internal deploy and what reference to use for pulls.
2302 pullRef, isInternal := s.pullRefForDeploy(ctx, imageRef)
2303 if err := s.validateImagePullRef(ctx, imageRef); err != nil {
2304 return "", err
2305 }
2306 if pullRef != imageRef {
2307 log.Info().
2308 Str("original_ref", imageRef).
2309 Str("pull_ref", pullRef).
2310 Msg("internal deploy: using localhost registry for pull")
2311 }
2312
2313 found, err := s.ensureLocalImage(ctx, imageRef, pullRef)
2314 if err != nil {
2315 return "", err
2316 }
2317 if found {
2318 return imageRef, nil
2319 }
2320
2321 // Pull image
2322 log.Info().Msg("pulling image from registry")
2323
2324 if err := s.pullImage(ctx, pullRef, isInternal); err != nil {
2325 return "", err
2326 }
2327
2328 // For digest references, we can't create a tag (Docker doesn't allow it).
2329 // In this case, use the pullRef directly since the image is available under that reference.
2330 if strings.Contains(imageRef, "@sha256:") {
2331 log.Info().
2332 Str("pull_ref", pullRef).
2333 Msg("digest reference: using pull reference for container operations")
2334 return pullRef, nil
2335 }
2336
2337 if err := s.tagImageIfNeeded(ctx, pullRef, imageRef); err != nil {
2338 return "", err
2339 }
2340
2341 // Clean up the temporary pull reference tag to avoid duplicate entries
2342 if pullRef != imageRef {
2343 if err := s.runtime.UntagImage(ctx, pullRef); err != nil {
2344 // Log but don't fail - the canonical tag is already applied
2345 log.Debug().Err(err).Str("pull_ref", pullRef).Msg("failed to remove temporary pull tag")
2346 }
2347 }
2348
2349 log.Info().Msg("image pulled successfully")
2350 return imageRef, nil
2351}
2352
2353func (s *Service) ensureLocalImage(ctx context.Context, imageRef, pullRef string) (bool, error) {
2354 log := zerowrap.FromCtx(ctx)
2355
2356 // For internal deploys (image push events), always pull fresh image
2357 // because the same tag may reference new content (e.g., latest tag updated)
2358 if domain.IsInternalDeploy(ctx) {
2359 log.Info().Msg("internal deploy detected, forcing image pull to ensure latest content")
2360 return false, nil
2361 }
2362
2363 s.mu.RLock()
2364 cfg := s.config
2365 s.mu.RUnlock()
2366
2367 pullPolicy := normalizePullPolicy(cfg.PullPolicy)
2368 switch pullPolicy {
2369 case PullPolicyAlways:
2370 log.Info().Str("pull_policy", pullPolicy).Msg("pull policy forces image pull")
2371 return false, nil
2372 case PullPolicyIfTagChanged:
2373 if !isDigestRef(imageRef) {
2374 log.Info().Str("pull_policy", pullPolicy).Msg("tag reference detected, pulling to check for updates")
2375 return false, nil
2376 }
2377 }
2378
2379 localImages, err := s.runtime.ListImages(ctx)
2380 if err != nil {
2381 log.WrapErr(err, "failed to list local images, will attempt pull")
2382 return false, nil
2383 }
2384
2385 normalizedRef := normalizeImageRef(imageRef)
2386 normalizedPullRef := normalizeImageRef(pullRef)
2387 for _, img := range localImages {
2388 normalizedImage := normalizeImageRef(img)
2389 if normalizedImage == normalizedRef {
2390 log.Info().Msg("image found locally, skipping pull")
2391 return true, nil
2392 }
2393 if normalizedImage == normalizedPullRef {
2394 if err := s.tagImageIfNeeded(ctx, pullRef, imageRef); err != nil {
2395 return false, err
2396 }
2397 // Clean up the temporary pull reference tag
2398 if pullRef != imageRef {
2399 if err := s.runtime.UntagImage(ctx, pullRef); err != nil {
2400 log.Debug().Err(err).Str("pull_ref", pullRef).Msg("failed to remove temporary pull tag")
2401 }
2402 }
2403 log.Info().Msg("image found locally, skipping pull")
2404 return true, nil
2405 }
2406 }
2407
2408 return false, nil
2409}
2410
2411func (s *Service) pullImage(ctx context.Context, pullRef string, isInternal bool) error {
2412 s.mu.RLock()
2413 cfg := s.config
2414 s.mu.RUnlock()
2415
2416 log := zerowrap.FromCtx(ctx)
2417
2418 switch {
2419 case isInternal && cfg.RegistryAuthEnabled:
2420 if cfg.InternalRegistryUsername == "" || cfg.InternalRegistryPassword == "" {
2421 return log.WrapErr(fmt.Errorf("internal registry auth not configured"), "failed to pull image for internal deploy")
2422 }
2423 if err := s.pullImageWithRetry(ctx, func(ctx context.Context) error {
2424 return s.runtime.PullImageWithAuth(ctx, pullRef, cfg.InternalRegistryUsername, cfg.InternalRegistryPassword)
2425 }, isInternal); err != nil {
2426 return log.WrapErr(fmt.Errorf("%w: %w", domain.ErrImagePullFailed, err), "failed to pull image with internal auth")
2427 }
2428 case isInternal:
2429 if err := s.pullImageWithRetry(ctx, func(ctx context.Context) error {
2430 return s.runtime.PullImage(ctx, pullRef)
2431 }, isInternal); err != nil {
2432 return log.WrapErr(fmt.Errorf("%w: %w", domain.ErrImagePullFailed, err), "failed to pull image")
2433 }
2434 case cfg.RegistryAuthEnabled:
2435 if cfg.ServiceTokenUsername == "" || cfg.ServiceToken == "" {
2436 return log.WrapErr(fmt.Errorf("registry service token not configured"), "failed to pull image for registry auth")
2437 }
2438 if err := s.runtime.PullImageWithAuth(ctx, pullRef, cfg.ServiceTokenUsername, cfg.ServiceToken); err != nil {
2439 return log.WrapErr(fmt.Errorf("%w: %w", domain.ErrImagePullFailed, err), "failed to pull image with auth")
2440 }
2441 default:
2442 if err := s.runtime.PullImage(ctx, pullRef); err != nil {
2443 return log.WrapErr(fmt.Errorf("%w: %w", domain.ErrImagePullFailed, err), "failed to pull image")
2444 }
2445 }
2446
2447 return nil
2448}
2449
2450func (s *Service) pullImageWithRetry(ctx context.Context, pullFn func(context.Context) error, isInternal bool) error {
2451 log := zerowrap.FromCtx(ctx)
2452
2453 for attempt := 1; attempt <= internalPullMaxAttempts; attempt++ {
2454 err := pullFn(ctx)
2455 if err == nil {
2456 return nil
2457 }
2458 if !isInternal || !isConnectionRefusedError(err) || attempt == internalPullMaxAttempts {
2459 return fmt.Errorf("internal image pull failed after %d attempts: %w", attempt, err)
2460 }
2461
2462 backoff := time.Duration(attempt) * time.Second
2463 log.Warn().
2464 Err(err).
2465 Int("attempt", attempt).
2466 Dur("backoff", backoff).
2467 Msg("internal image pull failed with connection refused, retrying")
2468
2469 select {
2470 case <-time.After(backoff):
2471 case <-ctx.Done():
2472 return ctx.Err()
2473 }
2474 }
2475 return nil
2476}
2477
2478func (s *Service) tagImageIfNeeded(ctx context.Context, sourceRef, targetRef string) error {
2479 if sourceRef == targetRef {
2480 return nil
2481 }
2482
2483 // Cannot create a tag with a digest reference - Docker/Podman doesn't allow it.
2484 // When using digest references (image@sha256:...), skip tagging as the image
2485 // is already available by its digest.
2486 if strings.Contains(targetRef, "@sha256:") {
2487 return nil
2488 }
2489
2490 log := zerowrap.FromCtx(ctx)
2491 if err := s.runtime.TagImage(ctx, sourceRef, targetRef); err != nil {
2492 return log.WrapErr(err, "failed to tag image from pull reference")
2493 }
2494 return nil
2495}
2496
2497func (s *Service) loadEnvironment(ctx context.Context, preResolved []string, domainName, imageRef string) ([]string, error) {
2498 log := zerowrap.FromCtx(ctx)
2499
2500 var userEnvVars []string
2501 if len(preResolved) > 0 {
2502 userEnvVars = preResolved
2503 } else {
2504 var err error
2505 userEnvVars, err = s.envLoader.LoadEnv(ctx, domainName)
2506 if err != nil {
2507 return nil, log.WrapErr(err, "failed to load environment variables")
2508 }
2509 }
2510
2511 dockerfileEnvVars, err := s.runtime.InspectImageEnv(ctx, imageRef)
2512 if err != nil {
2513 log.WrapErr(err, "failed to inspect image environment")
2514 dockerfileEnvVars = []string{}
2515 }
2516
2517 return mergeEnvironmentVariables(dockerfileEnvVars, userEnvVars), nil
2518}
2519
2520func (s *Service) setupVolumes(ctx context.Context, domainName, imageRef string) (map[string]string, error) {
2521 s.mu.RLock()
2522 cfg := s.config
2523 s.mu.RUnlock()
2524
2525 log := zerowrap.FromCtx(ctx)
2526 volumes := make(map[string]string)
2527
2528 if !cfg.VolumeAutoCreate {
2529 return volumes, nil
2530 }
2531
2532 volumePaths, err := s.runtime.InspectImageVolumes(ctx, imageRef)
2533 if err != nil {
2534 log.WrapErr(err, "failed to inspect image volumes")
2535 return volumes, nil
2536 }
2537
2538 for _, path := range volumePaths {
2539 name := generateVolumeName(cfg.VolumePrefix, domainName, path)
2540
2541 exists, err := s.runtime.VolumeExists(ctx, name)
2542 if err != nil {
2543 log.WrapErrWithFields(err, "failed to check volume", map[string]any{"volume": name})
2544 continue
2545 }
2546
2547 if !exists {
2548 if err := s.runtime.CreateVolume(ctx, name); err != nil {
2549 log.WrapErrWithFields(err, "failed to create volume", map[string]any{"volume": name})
2550 continue
2551 }
2552 log.Info().Str("volume", name).Str(zerowrap.FieldPath, path).Msg("created volume")
2553 }
2554
2555 volumes[path] = name
2556 }
2557
2558 return volumes, nil
2559}
2560
2561func (s *Service) getNetworkForApp(domainName string) string {
2562 var networkIsolation bool
2563 var networkGroups map[string][]string
2564
2565 if s.configProvider != nil {
2566 snap := s.configProvider.GetAttachmentConfig()
2567 networkGroups = snap.NetworkGroups
2568 s.mu.RLock()
2569 networkIsolation = s.config.NetworkIsolation
2570 s.mu.RUnlock()
2571 } else {
2572 s.mu.RLock()
2573 networkIsolation = s.config.NetworkIsolation
2574 networkGroups = s.config.NetworkGroups
2575 s.mu.RUnlock()
2576 }
2577
2578 if !networkIsolation {
2579 return "bridge"
2580 }
2581
2582 for groupName, domains := range networkGroups {
2583 if slices.Contains(domains, domainName) {
2584 return s.generateNetworkName(groupName)
2585 }
2586 }
2587
2588 return s.generateNetworkName(domainName)
2589}
2590
2591func (s *Service) generateNetworkName(identifier string) string {
2592 s.mu.RLock()
2593 cfg := s.config
2594 s.mu.RUnlock()
2595
2596 return fmt.Sprintf("%s-%s", cfg.NetworkPrefix, strings.ReplaceAll(identifier, ".", "-"))
2597}
2598
2599func (s *Service) createNetworkIfNeeded(ctx context.Context, networkName string) error {
2600 if networkName == "bridge" || networkName == "default" {
2601 return nil
2602 }
2603
2604 ctx = zerowrap.CtxWithField(ctx, "network", networkName)
2605 log := zerowrap.FromCtx(ctx)
2606
2607 exists, err := s.runtime.NetworkExists(ctx, networkName)
2608 if err != nil {
2609 return log.WrapErr(err, "failed to check network existence")
2610 }
2611
2612 if !exists {
2613 s.mu.RLock()
2614 internal := s.config.NetworkInternal
2615 s.mu.RUnlock()
2616 if err := s.runtime.CreateNetwork(ctx, networkName, domain.NetworkConfig{Driver: "bridge", Internal: internal}); err != nil {
2617 return log.WrapErr(err, "failed to create network")
2618 }
2619 log.Info().Msg("created network for app isolation")
2620 }
2621
2622 return nil
2623}
2624
2625func (s *Service) cleanupOrphanedContainers(ctx context.Context, domainName string, skipContainerID string) error {
2626 log := zerowrap.FromCtx(ctx)
2627 expectedName := fmt.Sprintf("gordon-%s", domainName)
2628 expectedNewName := expectedName + "-new"
2629 expectedNextName := expectedName + "-next"
2630
2631 allContainers, err := s.runtime.ListContainers(ctx, true)
2632 if err != nil {
2633 return err
2634 }
2635
2636 for _, c := range allContainers {
2637 if (c.Name == expectedName || c.Name == expectedNewName || c.Name == expectedNextName) && c.ID != skipContainerID {
2638 // Without a selected active container, preserve anything that may still
2639 // be serving traffic. When an active container is selected, other active
2640 // temp-name containers are stale candidates left by interrupted deploys.
2641 isActive := c.Status == "running" || c.Status == "restarting"
2642 if isActive && (skipContainerID == "" || c.Name == expectedName) {
2643 log.Debug().
2644 Str(zerowrap.FieldEntityID, c.ID).
2645 Str("container_name", c.Name).
2646 Str(zerowrap.FieldStatus, c.Status).
2647 Msg("skipping active container during orphan cleanup")
2648 continue
2649 }
2650
2651 log.Info().Str(zerowrap.FieldEntityID, c.ID).Str("container_name", c.Name).Str(zerowrap.FieldStatus, c.Status).Msg("found orphaned container, removing")
2652
2653 if err := s.runtime.StopContainer(ctx, c.ID); err != nil {
2654 log.WrapErrWithFields(err, "failed to stop orphaned container", map[string]any{zerowrap.FieldEntityID: c.ID})
2655 }
2656
2657 if err := s.runtime.RemoveContainer(ctx, c.ID, true); err != nil {
2658 log.WrapErrWithFields(err, "failed to remove orphaned container", map[string]any{zerowrap.FieldEntityID: c.ID})
2659 }
2660 }
2661 }
2662
2663 return nil
2664}
2665
2666func (s *Service) cleanupVolumesForDomain(_ context.Context, _ string) error {
2667 return nil
2668}
2669
2670func (s *Service) cleanupNetworkIfEmpty(ctx context.Context, networkName string) error {
2671 if networkName == "bridge" || networkName == "default" {
2672 return nil
2673 }
2674
2675 ctx = zerowrap.CtxWithField(ctx, "network", networkName)
2676 log := zerowrap.FromCtx(ctx)
2677
2678 networks, err := s.runtime.ListNetworks(ctx)
2679 if err != nil {
2680 return log.WrapErr(err, "failed to list networks")
2681 }
2682
2683 for _, n := range networks {
2684 if n.Name == networkName && len(n.Containers) == 0 {
2685 if err := s.runtime.RemoveNetwork(ctx, networkName); err != nil {
2686 return err
2687 }
2688 log.Info().Msg("cleaned up empty network")
2689 break
2690 }
2691 }
2692
2693 return nil
2694}
2695
2696func (s *Service) deployAttachments(ctx context.Context, domainName, networkName string) error {
2697 // Collect attachments from both domain-specific and network group configs
2698 attachments := s.resolveAttachmentsForDomain(domainName)
2699 if len(attachments) == 0 {
2700 return nil
2701 }
2702
2703 log := zerowrap.FromCtx(ctx)
2704 var deployed []string // track successfully deployed attachment IDs for rollback
2705 for _, svc := range attachments {
2706 if err := s.deployAttachedService(ctx, domainName, svc, networkName); err != nil {
2707 log.WrapErrWithFields(err, "failed to deploy attachment", map[string]any{zerowrap.FieldService: svc, "domain": domainName})
2708
2709 // Rollback: clean up already-deployed attachments
2710 for _, id := range deployed {
2711 if stopErr := s.runtime.StopContainer(ctx, id); stopErr != nil {
2712 log.Warn().Err(stopErr).Str("attachment_id", id).Msg("rollback: failed to stop attachment")
2713 }
2714 if rmErr := s.runtime.RemoveContainer(ctx, id, true); rmErr != nil {
2715 log.Warn().Err(rmErr).Str("attachment_id", id).Msg("rollback: failed to remove attachment")
2716 }
2717 }
2718 // Deregister rolled-back attachments
2719 if len(deployed) > 0 {
2720 s.mu.Lock()
2721 remaining := s.attachments[domainName]
2722 filtered := remaining[:0]
2723 rollbackSet := make(map[string]bool, len(deployed))
2724 for _, id := range deployed {
2725 rollbackSet[id] = true
2726 }
2727 for _, id := range remaining {
2728 if !rollbackSet[id] {
2729 filtered = append(filtered, id)
2730 }
2731 }
2732 s.attachments[domainName] = filtered
2733 s.mu.Unlock()
2734 }
2735
2736 return fmt.Errorf("failed to deploy attachment %q (rolled back %d already-deployed)", svc, len(deployed))
2737 }
2738 // Collect IDs of attachments tracked under this domain after successful deploy
2739 s.mu.RLock()
2740 ids := s.attachments[domainName]
2741 if len(ids) > 0 {
2742 latest := ids[len(ids)-1]
2743 deployed = append(deployed, latest)
2744 }
2745 s.mu.RUnlock()
2746 }
2747 return nil
2748}
2749
2750// resolveAttachmentsForDomain returns attachments for a domain by checking:
2751// 1. Direct domain attachments (attachments[domain])
2752// 2. Network group attachments (attachments[group] where domain is in network_groups[group])
2753func (s *Service) resolveAttachmentsForDomain(domainName string) []string {
2754 var attachments map[string][]string
2755 var networkGroups map[string][]string
2756
2757 if s.configProvider != nil {
2758 snap := s.configProvider.GetAttachmentConfig()
2759 attachments = snap.Attachments
2760 networkGroups = snap.NetworkGroups
2761 } else {
2762 s.mu.RLock()
2763 attachments = s.config.Attachments
2764 networkGroups = s.config.NetworkGroups
2765 s.mu.RUnlock()
2766 }
2767
2768 seen := make(map[string]bool)
2769 var result []string
2770
2771 // First, add domain-specific attachments
2772 if domainAttachments, ok := attachments[domainName]; ok {
2773 for _, img := range domainAttachments {
2774 if !seen[img] {
2775 seen[img] = true
2776 result = append(result, img)
2777 }
2778 }
2779 }
2780
2781 // Then, find which network group this domain belongs to and add group attachments
2782 for groupName, domains := range networkGroups {
2783 if slices.Contains(domains, domainName) {
2784 if groupAttachments, ok := attachments[groupName]; ok {
2785 for _, img := range groupAttachments {
2786 if !seen[img] {
2787 seen[img] = true
2788 result = append(result, img)
2789 }
2790 }
2791 }
2792 break // Domain can only be in one network group
2793 }
2794 }
2795
2796 return result
2797}
2798
2799func (s *Service) getAttachmentsForDomain(ctx context.Context, domainName string) []domain.Attachment {
2800 containers, err := s.runtime.ListContainers(ctx, true)
2801 if err != nil {
2802 return nil
2803 }
2804
2805 return s.filterAttachments(ctx, containers, domainName)
2806}
2807
2808// getAllAttachments fetches all containers once and returns attachments grouped by domain.
2809// This avoids N+1 queries when listing multiple routes.
2810func (s *Service) getAllAttachments(ctx context.Context) map[string][]domain.Attachment {
2811 containers, err := s.runtime.ListContainers(ctx, true)
2812 if err != nil {
2813 return nil
2814 }
2815
2816 result := make(map[string][]domain.Attachment)
2817 for _, container := range containers {
2818 if container.Labels == nil {
2819 continue
2820 }
2821 if container.Labels[domain.LabelAttachment] != "true" {
2822 continue
2823 }
2824 ownerDomain := container.Labels[domain.LabelAttachedTo]
2825 if ownerDomain == "" {
2826 continue
2827 }
2828 image := container.Image
2829 serviceName := container.Name
2830 if labelImage, ok := container.Labels[domain.LabelImage]; ok && labelImage != "" {
2831 image = labelImage
2832 serviceName = extractServiceName(labelImage)
2833 }
2834 // Get the container's network for display in the routes table
2835 network := ""
2836 if networkName, err := s.runtime.GetContainerNetwork(ctx, container.ID); err == nil {
2837 network = networkName
2838 }
2839 attachment := domain.Attachment{
2840 Name: serviceName,
2841 Image: s.stripRegistryPrefix(image),
2842 ContainerID: container.ID,
2843 Status: container.Status,
2844 Network: network,
2845 Ports: append([]int(nil), container.Ports...),
2846 }
2847 result[ownerDomain] = append(result[ownerDomain], attachment)
2848 }
2849
2850 return result
2851}
2852
2853// filterAttachments extracts attachments for a specific domain from a container list.
2854func (s *Service) filterAttachments(ctx context.Context, containers []*domain.Container, domainName string) []domain.Attachment {
2855 attachments := make([]domain.Attachment, 0)
2856 for _, container := range containers {
2857 if container.Labels == nil {
2858 continue
2859 }
2860 if container.Labels[domain.LabelAttachment] != "true" {
2861 continue
2862 }
2863 if container.Labels[domain.LabelAttachedTo] != domainName {
2864 continue
2865 }
2866 image := container.Image
2867 serviceName := container.Name
2868 if labelImage, ok := container.Labels[domain.LabelImage]; ok && labelImage != "" {
2869 image = labelImage
2870 serviceName = extractServiceName(labelImage)
2871 }
2872 // Get the container's network for display in the routes table
2873 network := ""
2874 if networkName, err := s.runtime.GetContainerNetwork(ctx, container.ID); err == nil {
2875 network = networkName
2876 }
2877 attachment := domain.Attachment{
2878 Name: serviceName,
2879 Image: s.stripRegistryPrefix(image),
2880 ContainerID: container.ID,
2881 Status: container.Status,
2882 Network: network,
2883 Ports: append([]int(nil), container.Ports...),
2884 }
2885 attachments = append(attachments, attachment)
2886 }
2887
2888 return attachments
2889}
2890
2891func (s *Service) deployAttachedService(ctx context.Context, ownerDomain, serviceImage, networkName string) error {
2892 log := zerowrap.FromCtx(ctx)
2893
2894 // Parse service name from image (e.g., "my-postgres:latest" → "postgres")
2895 serviceName := extractServiceName(serviceImage)
2896 containerName := fmt.Sprintf("gordon-%s-%s", sanitizeName(ownerDomain), serviceName)
2897
2898 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
2899 "attachment": serviceName,
2900 "container_name": containerName,
2901 "owner_domain": ownerDomain,
2902 })
2903
2904 // Check if already running (idempotent)
2905 // Try new name first, then fall back to legacy name for backwards compatibility
2906 existingContainer, err := s.resolveExistingAttachment(ctx, ownerDomain, serviceName)
2907 if err != nil {
2908 return err
2909 }
2910
2911 if existingContainer != nil {
2912 shouldSkip, err := s.handleRunningAttachment(ctx, existingContainer, containerName, serviceImage)
2913 if err != nil {
2914 return err
2915 }
2916 if shouldSkip {
2917 return nil
2918 }
2919 if existingContainer.Status == string(domain.ContainerStatusRunning) {
2920 existingContainer = nil
2921 }
2922 }
2923
2924 if err := s.removeStoppedAttachment(ctx, existingContainer, containerName); err != nil {
2925 return err
2926 }
2927
2928 log.Info().Str(zerowrap.FieldService, serviceImage).Msg("deploying attached service")
2929
2930 config, err := s.buildAttachmentContainerConfig(ctx, ownerDomain, serviceName, serviceImage, containerName, networkName)
2931 if err != nil {
2932 return err
2933 }
2934
2935 container, err := s.runtime.CreateContainer(ctx, config)
2936 if err != nil {
2937 return log.WrapErr(err, "failed to create attachment container")
2938 }
2939
2940 // Start container
2941 if err := s.runtime.StartContainer(ctx, container.ID); err != nil {
2942 s.runtime.RemoveContainer(ctx, container.ID, true)
2943 return log.WrapErr(err, "failed to start attachment container")
2944 }
2945
2946 // Wait for attachment to be ready before proceeding
2947 if err := s.waitForAttachmentReady(ctx, container.ID, config); err != nil {
2948 log.WrapErr(err, "attachment readiness check failed, cleaning up")
2949 if stopErr := s.runtime.StopContainer(ctx, container.ID); stopErr != nil {
2950 log.Warn().Err(stopErr).Str(zerowrap.FieldEntityID, container.ID).Msg("failed to stop unready attachment")
2951 }
2952 s.runtime.RemoveContainer(ctx, container.ID, true)
2953 return fmt.Errorf("attachment %q not ready: %w", serviceName, err)
2954 }
2955 // Track attachment
2956 s.mu.Lock()
2957 s.attachments[ownerDomain] = append(s.attachments[ownerDomain], container.ID)
2958 s.mu.Unlock()
2959
2960 // Start log collection for attachment
2961 s.startLogCollection(ctx, container.ID, containerName)
2962
2963 log.Info().Str(zerowrap.FieldEntityID, container.ID).Msg("attachment deployed successfully")
2964 return nil
2965}
2966
2967func (s *Service) buildAttachmentContainerConfig(ctx context.Context, ownerDomain, serviceName, serviceImage, containerName, networkName string) (*domain.ContainerConfig, error) {
2968 imageRef, err := s.buildValidatedImageRef(ctx, serviceImage)
2969 if err != nil {
2970 return nil, err
2971 }
2972 actualImageRef, err := s.ensureImage(ctx, imageRef)
2973 if err != nil {
2974 return nil, err
2975 }
2976
2977 exposedPorts := s.attachmentExposedPorts(ctx, actualImageRef)
2978 volumes, err := s.attachmentVolumes(ctx, containerName, actualImageRef)
2979 if err != nil {
2980 return nil, err
2981 }
2982 envVars, err := s.attachmentEnv(ctx, containerName, actualImageRef)
2983 if err != nil {
2984 return nil, err
2985 }
2986
2987 s.mu.RLock()
2988 cfg := s.config
2989 s.mu.RUnlock()
2990
2991 config := &domain.ContainerConfig{
2992 Image: actualImageRef,
2993 Name: containerName,
2994 Hostname: serviceName,
2995 Aliases: []string{serviceName},
2996 Ports: exposedPorts,
2997 Env: envVars,
2998 Volumes: volumes,
2999 NetworkMode: networkName,
3000 Labels: map[string]string{
3001 domain.LabelManaged: "true",
3002 domain.LabelAttachment: "true",
3003 domain.LabelAttachedTo: ownerDomain,
3004 domain.LabelEnvHash: hashEnvironment(envVars),
3005 domain.LabelImage: serviceImage,
3006 },
3007 MemoryLimit: cfg.DefaultMemoryLimit,
3008 NanoCPUs: cfg.DefaultNanoCPUs,
3009 PidsLimit: cfg.DefaultPidsLimit,
3010 RestartPolicy: domain.RestartPolicyAlways,
3011 }
3012 applySecurityProfile(config, cfg)
3013 return config, nil
3014}
3015
3016func (s *Service) attachmentExposedPorts(ctx context.Context, imageRef string) []int {
3017 log := zerowrap.FromCtx(ctx)
3018 exposedPorts, err := s.runtime.GetImageExposedPorts(ctx, imageRef)
3019 if err != nil {
3020 log.WrapErr(err, "failed to get exposed ports for attachment, using defaults")
3021 return []int{}
3022 }
3023 return exposedPorts
3024}
3025
3026func (s *Service) attachmentVolumes(ctx context.Context, containerName, imageRef string) (map[string]string, error) {
3027 volumes, err := s.setupVolumes(ctx, containerName, imageRef)
3028 if err != nil {
3029 return nil, fmt.Errorf("failed to setup volumes for attachment %s with image %s: %w", containerName, imageRef, err)
3030 }
3031 return volumes, nil
3032}
3033
3034func (s *Service) attachmentEnv(ctx context.Context, containerName, imageRef string) ([]string, error) {
3035 envVars, err := s.loadEnvironment(ctx, nil, containerName, imageRef)
3036 if err != nil {
3037 return nil, fmt.Errorf("failed to load environment for attachment %s with image %s: %w", containerName, imageRef, err)
3038 }
3039 return envVars, nil
3040}
3041
3042func (s *Service) attachmentEnvDrifted(ctx context.Context, existing *domain.Container, containerName, serviceImage string) (bool, error) {
3043 imageRef, err := s.buildValidatedImageRef(ctx, serviceImage)
3044 if err != nil {
3045 return false, err
3046 }
3047 currentEnv, err := s.loadEnvironment(ctx, nil, containerName, imageRef)
3048 if err != nil {
3049 return false, err
3050 }
3051
3052 currentHash := hashEnvironment(currentEnv)
3053 existingHash, ok := existing.Labels[domain.LabelEnvHash]
3054 if ok && existingHash == currentHash {
3055 return false, nil
3056 }
3057
3058 return true, nil
3059}
3060
3061func (s *Service) handleRunningAttachment(ctx context.Context, existing *domain.Container, containerName, serviceImage string) (bool, error) {
3062 if existing.Status != string(domain.ContainerStatusRunning) {
3063 return false, nil
3064 }
3065
3066 log := zerowrap.FromCtx(ctx)
3067
3068 // Check image drift
3069 existingImage := existing.Labels[domain.LabelImage]
3070 imageDrifted := existingImage != "" && existingImage != serviceImage
3071
3072 // Check env drift
3073 envDrifted, err := s.attachmentEnvDrifted(ctx, existing, containerName, serviceImage)
3074 if err != nil {
3075 return false, log.WrapErr(err, "failed to check attachment env drift")
3076 }
3077
3078 if !envDrifted && !imageDrifted {
3079 log.Debug().Str("container_name", containerName).Msg("attachment already running with current env and image, skipping")
3080 return true, nil
3081 }
3082
3083 if imageDrifted {
3084 log.Info().Str("container_name", containerName).Str("existing_image", existingImage).Str("desired_image", serviceImage).Msg("attachment image changed, recreating")
3085 }
3086 if envDrifted {
3087 log.Info().Str("container_name", containerName).Msg("attachment env changed, recreating")
3088 }
3089
3090 if err := s.runtime.StopContainer(ctx, existing.ID); err != nil {
3091 return false, log.WrapErr(err, "failed to stop attachment for drift update")
3092 }
3093 if err := s.runtime.RemoveContainer(ctx, existing.ID, true); err != nil {
3094 return false, log.WrapErr(err, "failed to remove attachment for drift update")
3095 }
3096
3097 return false, nil
3098}
3099
3100func (s *Service) removeStoppedAttachment(ctx context.Context, existing *domain.Container, containerName string) error {
3101 if existing == nil {
3102 return nil
3103 }
3104
3105 log := zerowrap.FromCtx(ctx)
3106 log.Info().Str("container_name", containerName).Msg("removing stopped attachment container")
3107 if err := s.runtime.RemoveContainer(ctx, existing.ID, true); err != nil {
3108 return log.WrapErr(err, "failed to remove existing attachment container")
3109 }
3110
3111 return nil
3112}
3113
3114func (s *Service) resolveExistingAttachment(ctx context.Context, ownerDomain, serviceName string) (*domain.Container, error) {
3115 log := zerowrap.FromCtx(ctx)
3116 containerName := fmt.Sprintf("gordon-%s-%s", sanitizeName(ownerDomain), serviceName)
3117 existingContainer := s.findContainerByName(ctx, containerName)
3118 if existingContainer != nil {
3119 return existingContainer, nil
3120 }
3121
3122 containerNameLegacy := fmt.Sprintf("gordon-%s-%s", sanitizeNameLegacy(ownerDomain), serviceName)
3123 existingContainer = s.findContainerByName(ctx, containerNameLegacy)
3124 if existingContainer == nil {
3125 return nil, nil
3126 }
3127
3128 log.Info().Str("container_name", containerNameLegacy).Msg("found attachment with legacy naming, will be replaced")
3129 if err := s.runtime.StopContainer(ctx, existingContainer.ID); err != nil {
3130 return nil, log.WrapErr(err, "failed to stop legacy attachment container")
3131 }
3132 if err := s.runtime.RemoveContainer(ctx, existingContainer.ID, true); err != nil {
3133 return nil, log.WrapErr(err, "failed to remove legacy attachment container")
3134 }
3135
3136 return nil, nil
3137}
3138
3139// Utility functions
3140
3141func normalizeImageRef(image string) string {
3142 if isDigestRef(image) {
3143 return image
3144 }
3145
3146 // Extract repository from image reference, handling port numbers correctly.
3147 // Examples:
3148 // nginx:latest -> docker.io/library/nginx
3149 // user/repo:tag -> docker.io/user/repo
3150 // localhost:5000/image:tag -> localhost:5000/image
3151 // registry.example.com/image:tag -> registry.example.com/image
3152 //
3153 // Note: This function assumes valid Docker image references. Edge cases like
3154 // "registry.com:5000" (registry with port but no image name) are not valid
3155 // image references per Docker's naming conventions, which require at least
3156 // one path component after the registry (e.g., "registry.com:5000/image").
3157
3158 // Find the tag separator: the last colon that isn't part of a port number.
3159 // A colon is part of a port if there's no slash after it until the next colon.
3160 repo := image
3161 lastColon := strings.LastIndex(image, ":")
3162 if lastColon != -1 {
3163 afterColon := image[lastColon+1:]
3164 // If there's no slash after the colon, it's the tag separator
3165 if !strings.Contains(afterColon, "/") {
3166 repo = image[:lastColon]
3167 }
3168 }
3169
3170 if !strings.Contains(repo, "/") {
3171 return "docker.io/library/" + repo
3172 }
3173
3174 if strings.Count(repo, "/") == 1 && !strings.Contains(strings.Split(repo, "/")[0], ".") && !strings.Contains(strings.Split(repo, "/")[0], ":") {
3175 return "docker.io/" + repo
3176 }
3177
3178 return repo
3179}
3180
3181func generateVolumeName(prefix, domainName, volumePath string) string {
3182 return fmt.Sprintf("%s-%s-%s",
3183 prefix,
3184 strings.ReplaceAll(domainName, ".", "-"),
3185 strings.ReplaceAll(strings.Trim(volumePath, "/"), "/", "-"))
3186}
3187
3188func mergeEnvironmentVariables(dockerfileEnv, userEnv []string) []string {
3189 envMap := make(map[string]string)
3190
3191 for _, env := range dockerfileEnv {
3192 if k, v, ok := strings.Cut(env, "="); ok {
3193 envMap[k] = v
3194 }
3195 }
3196
3197 for _, env := range userEnv {
3198 if k, v, ok := strings.Cut(env, "="); ok {
3199 envMap[k] = v
3200 }
3201 }
3202
3203 keys := make([]string, 0, len(envMap))
3204 for k := range envMap {
3205 keys = append(keys, k)
3206 }
3207 slices.Sort(keys)
3208
3209 result := make([]string, 0, len(envMap))
3210 for _, k := range keys {
3211 result = append(result, k+"="+envMap[k])
3212 }
3213
3214 return result
3215}
3216
3217// extractServiceName gets service name from image reference.
3218// "my-postgres:latest" → "postgres", "redis:7" → "redis"
3219func extractServiceName(image string) string {
3220 // Remove tag
3221 parts := strings.Split(image, ":")
3222 name := parts[0]
3223
3224 // Remove registry prefix if present
3225 if strings.Contains(name, "/") {
3226 nameParts := strings.Split(name, "/")
3227 name = nameParts[len(nameParts)-1]
3228 }
3229
3230 // Remove common prefixes like "my-"
3231 name = strings.TrimPrefix(name, "my-")
3232
3233 return name
3234}
3235
3236// sanitizeName is a convenience wrapper around domain.SanitizeDomainForContainer.
3237func sanitizeName(d string) string {
3238 return domain.SanitizeDomainForContainer(d)
3239}
3240
3241// sanitizeNameLegacy is a convenience wrapper around domain.SanitizeDomainForContainerLegacy.
3242func sanitizeNameLegacy(d string) string {
3243 return domain.SanitizeDomainForContainerLegacy(d)
3244}
3245
3246func isContainerNotFoundError(err error) bool {
3247 if err == nil {
3248 return false
3249 }
3250 if errors.Is(err, domain.ErrContainerNotFound) {
3251 return true
3252 }
3253 msg := strings.ToLower(err.Error())
3254 return strings.Contains(msg, "no such container") ||
3255 strings.Contains(msg, "no container with name or id") ||
3256 strings.Contains(msg, "container not found")
3257}
3258
3259func isConnectionRefusedError(err error) bool {
3260 if err == nil {
3261 return false
3262 }
3263 return strings.Contains(strings.ToLower(err.Error()), "connection refused")
3264}
3265
3266// findContainerByName finds a container by its name.
3267func (s *Service) findContainerByName(ctx context.Context, name string) *domain.Container {
3268 containers, err := s.runtime.ListContainers(ctx, true)
3269 if err != nil {
3270 return nil
3271 }
3272
3273 for _, c := range containers {
3274 if c.Name == name {
3275 return c
3276 }
3277 }
3278 return nil
3279}
3280
3281func (s *Service) waitForReady(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig) error {
3282 if err := s.pollContainerRunning(ctx, containerID); err != nil {
3283 return err
3284 }
3285
3286 s.mu.RLock()
3287 cfg := s.config
3288 s.mu.RUnlock()
3289
3290 readinessMode := cfg.ReadinessMode
3291 if readinessMode == "" {
3292 readinessMode = "delay"
3293 }
3294
3295 switch readinessMode {
3296 case "delay":
3297 return s.waitForReadyByDelay(ctx, containerID)
3298 case "docker-health":
3299 _, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(ctx, containerID)
3300 if err != nil {
3301 return err
3302 }
3303 if !hasHealthcheck {
3304 return errors.New("no healthcheck detected")
3305 }
3306 return s.waitForHealthy(ctx, containerID, cfg.HealthTimeout)
3307 default: // auto
3308 return s.readinessCascade(ctx, containerID, containerConfig, cfg)
3309 }
3310}
3311
3312// readinessCascade auto-detects the strongest available readiness signal:
3313// 1. Docker healthcheck (if present) → wait for healthy status
3314// 2. HTTP probe (if gordon.health label set) → GET until 2xx/3xx
3315// 3. Default HTTP probe (GET / on exposed port) → wait for 2xx/3xx
3316// 4. TCP probe (if port info available) → connect until accepted
3317// 5. Delay fallback (last resort) → waitForReadyByDelay
3318func (s *Service) readinessCascade(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) error {
3319 log := zerowrap.FromCtx(ctx)
3320
3321 // 1. Docker healthcheck
3322 _, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(ctx, containerID)
3323 if err != nil {
3324 return err
3325 }
3326 if hasHealthcheck {
3327 log.Info().Msg("readiness cascade: using Docker healthcheck")
3328 return s.waitForHealthy(ctx, containerID, cfg.HealthTimeout)
3329 }
3330
3331 // 2. HTTP probe via gordon.health label
3332 if probed, probeErr := s.tryHTTPProbe(ctx, containerID, containerConfig, cfg); probed {
3333 return probeErr
3334 }
3335
3336 // 3. Default HTTP probe on root path (if port info available)
3337 if probed, probeErr := s.tryDefaultHTTPProbe(ctx, containerID, containerConfig, cfg); probed {
3338 return probeErr
3339 }
3340
3341 // 4. TCP probe (if port info available)
3342 if probed, probeErr := s.tryTCPProbe(ctx, containerID, containerConfig, cfg); probed {
3343 return probeErr
3344 }
3345
3346 // 5. Delay fallback
3347 log.Info().Msg("readiness cascade: using delay fallback")
3348 return s.waitForReadyByDelay(ctx, containerID)
3349}
3350
3351// waitForAttachmentReady runs a reduced readiness cascade for attachments:
3352// Docker healthcheck → TCP probe → delay fallback.
3353// HTTP probes are skipped — attachments are typically databases/caches.
3354func (s *Service) waitForAttachmentReady(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig) error {
3355 if err := s.pollContainerRunning(ctx, containerID); err != nil {
3356 return err
3357 }
3358
3359 log := zerowrap.FromCtx(ctx)
3360
3361 s.mu.RLock()
3362 cfg := s.config
3363 s.mu.RUnlock()
3364
3365 // 1. Docker healthcheck (if present)
3366 _, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(ctx, containerID)
3367 if err != nil {
3368 return err
3369 }
3370 if hasHealthcheck {
3371 log.Info().Msg("attachment readiness: using Docker healthcheck")
3372 timeout := cfg.HealthTimeout
3373 if timeout == 0 {
3374 timeout = 90 * time.Second
3375 }
3376 return s.waitForHealthy(ctx, containerID, timeout)
3377 }
3378
3379 // 2. TCP probe (if port info available)
3380 if containerConfig != nil && len(containerConfig.Ports) > 0 {
3381 ip, port, probeErr := s.resolveProbeEndpoint(ctx, containerID, containerConfig)
3382 if probeErr == nil && ip != "" && port > 0 {
3383 addr := fmt.Sprintf("%s:%d", ip, port)
3384 timeout := cfg.AttachmentReadinessTimeout
3385 if timeout == 0 {
3386 timeout = 30 * time.Second
3387 }
3388 log.Info().Str("addr", addr).Dur("timeout", timeout).Msg("attachment readiness: using TCP probe")
3389 return tcpProbe(ctx, addr, timeout)
3390 }
3391 log.Debug().Err(probeErr).Msg("attachment readiness: TCP probe skipped, could not resolve endpoint")
3392 }
3393
3394 // 3. Delay fallback
3395 log.Info().Msg("attachment readiness: using delay fallback")
3396 return s.waitForReadyByDelay(ctx, containerID)
3397}
3398
3399// tryHTTPProbe attempts an HTTP probe if the gordon.health label is set.
3400// Returns (true, err) if the probe was attempted, (false, nil) if skipped.
3401func (s *Service) tryHTTPProbe(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) (bool, error) {
3402 log := zerowrap.FromCtx(ctx)
3403 if containerConfig == nil {
3404 return false, nil
3405 }
3406 healthPath, ok := containerConfig.Labels[domain.LabelHealth]
3407 if !ok || healthPath == "" {
3408 return false, nil
3409 }
3410 ip, port, probeErr := s.resolveProbeEndpoint(ctx, containerID, containerConfig)
3411 if probeErr != nil || ip == "" || port <= 0 {
3412 log.Debug().Err(probeErr).Msg("readiness cascade: HTTP probe skipped, could not resolve container endpoint")
3413 return false, nil
3414 }
3415 url := fmt.Sprintf("http://%s:%d%s", ip, port, healthPath)
3416 timeout := cfg.HTTPProbeTimeout
3417 if timeout == 0 {
3418 timeout = 60 * time.Second
3419 }
3420 log.Info().Str("url", url).Dur("timeout", timeout).Msg("readiness cascade: using HTTP probe")
3421 return true, httpProbe(ctx, url, timeout)
3422}
3423
3424// tryDefaultHTTPProbe attempts an HTTP GET on "/" using the container's
3425// exposed port. This is the default readiness check for web containers —
3426// it verifies the app is actually serving HTTP, not just accepting TCP.
3427// Returns (true, err) if the probe was attempted, (false, nil) if skipped.
3428func (s *Service) tryDefaultHTTPProbe(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) (bool, error) {
3429 log := zerowrap.FromCtx(ctx)
3430 if containerConfig == nil || len(containerConfig.Ports) == 0 {
3431 return false, nil
3432 }
3433 ip, port, probeErr := s.resolveProbeEndpoint(ctx, containerID, containerConfig)
3434 if probeErr != nil || ip == "" || port <= 0 {
3435 log.Debug().Err(probeErr).Msg("readiness cascade: default HTTP probe skipped, could not resolve container endpoint")
3436 return false, nil
3437 }
3438 url := fmt.Sprintf("http://%s:%d/", ip, port)
3439 timeout := cfg.HTTPProbeTimeout
3440 if timeout == 0 {
3441 timeout = 60 * time.Second
3442 }
3443 log.Info().Str("url", url).Dur("timeout", timeout).Msg("readiness cascade: using default HTTP alive probe")
3444 return true, httpAliveProbe(ctx, url, timeout)
3445}
3446
3447// tryTCPProbe attempts a TCP probe if port info is available.
3448// Returns (true, err) if the probe was attempted, (false, nil) if skipped.
3449func (s *Service) tryTCPProbe(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) (bool, error) {
3450 log := zerowrap.FromCtx(ctx)
3451 if containerConfig == nil || len(containerConfig.Ports) == 0 {
3452 return false, nil
3453 }
3454 ip, port, probeErr := s.resolveProbeEndpoint(ctx, containerID, containerConfig)
3455 if probeErr != nil || ip == "" || port <= 0 {
3456 log.Debug().Err(probeErr).Msg("readiness cascade: TCP probe skipped, could not resolve container endpoint")
3457 return false, nil
3458 }
3459 addr := fmt.Sprintf("%s:%d", ip, port)
3460 timeout := cfg.TCPProbeTimeout
3461 if timeout == 0 {
3462 timeout = 30 * time.Second
3463 }
3464 log.Info().Str("addr", addr).Dur("timeout", timeout).Msg("readiness cascade: using TCP probe")
3465 return true, tcpProbe(ctx, addr, timeout)
3466}
3467
3468// resolveContainerEndpoint returns a host-reachable address for probing.
3469// In rootless podman/Docker setups, container internal IPs are not routable
3470// from the host. We use the host port binding (127.0.0.1:<mapped_port>)
3471// which is always reachable. Falls back to the container's internal IP
3472// only if no host port mapping exists (e.g. host-network mode).
3473func (s *Service) resolveProbeEndpoint(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig) (string, int, error) {
3474 log := zerowrap.FromCtx(ctx)
3475
3476 internalPort := 0
3477 if containerConfig != nil {
3478 if labels := containerConfig.Labels; labels != nil {
3479 for _, key := range []string{domain.LabelProxyPort, domain.LabelPort} {
3480 if portStr, ok := labels[key]; ok && portStr != "" {
3481 port, err := strconv.Atoi(portStr)
3482 if err == nil && port > 0 && port <= 65535 {
3483 internalPort = port
3484 break
3485 }
3486 log.Warn().Str("label", key).Str("port_value", portStr).Msg("invalid probe port label value")
3487 }
3488 }
3489 }
3490 }
3491
3492 if internalPort == 0 {
3493 _, resolvedPort, err := s.runtime.GetContainerNetworkInfo(ctx, containerID)
3494 if err != nil {
3495 return "", 0, err
3496 }
3497 internalPort = resolvedPort
3498 }
3499
3500 hostPort, hostErr := s.runtime.GetContainerPort(ctx, containerID, internalPort)
3501 if hostErr == nil && hostPort > 0 {
3502 log.Debug().
3503 Int("internal_port", internalPort).
3504 Int("host_port", hostPort).
3505 Msg("resolved container endpoint via host port binding")
3506 return "127.0.0.1", hostPort, nil
3507 }
3508
3509 ip, _, fallbackErr := s.runtime.GetContainerNetworkInfo(ctx, containerID)
3510 if fallbackErr != nil {
3511 return "", 0, fallbackErr
3512 }
3513 log.Debug().
3514 Str("ip", ip).
3515 Int("port", internalPort).
3516 Msg("resolved container endpoint via internal IP (no host port binding)")
3517 return ip, internalPort, nil
3518}
3519
3520// waitForReadyByDelay waits using the legacy running+delay strategy.
3521func (s *Service) waitForReadyByDelay(ctx context.Context, containerID string) error {
3522 log := zerowrap.FromCtx(ctx)
3523
3524 s.mu.RLock()
3525 cfg := s.config
3526 s.mu.RUnlock()
3527
3528 delay := cfg.ReadinessDelay
3529 if delay == 0 {
3530 delay = 5 * time.Second
3531 }
3532
3533 log.Debug().Dur("delay", delay).Msg("waiting for container readiness")
3534 select {
3535 case <-time.After(delay):
3536 case <-ctx.Done():
3537 return ctx.Err()
3538 }
3539
3540 // Verify still running after delay
3541 running, err := s.runtime.IsContainerRunning(ctx, containerID)
3542 if err != nil {
3543 return err
3544 }
3545 if running {
3546 return nil
3547 }
3548
3549 return s.waitForRecovery(ctx, containerID)
3550}
3551
3552func (s *Service) waitForHealthy(ctx context.Context, containerID string, timeout time.Duration) error {
3553 if timeout == 0 {
3554 timeout = 90 * time.Second
3555 }
3556 waitCtx, cancel := context.WithTimeout(ctx, timeout)
3557 defer cancel()
3558
3559 var lastStatus string
3560 for {
3561 status, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(waitCtx, containerID)
3562 if err != nil {
3563 return err
3564 }
3565 if !hasHealthcheck {
3566 return errors.New("no healthcheck detected")
3567 }
3568 // Treat empty health status as transitional startup. Some runtimes can
3569 // temporarily report an empty status before first probe results.
3570 if status == "" {
3571 status = "starting"
3572 }
3573 if status == "healthy" {
3574 return nil
3575 }
3576 if status == "unhealthy" {
3577 return errors.New("container reported unhealthy")
3578 }
3579 lastStatus = status
3580
3581 select {
3582 case <-time.After(time.Second):
3583 case <-waitCtx.Done():
3584 return fmt.Errorf("container healthcheck timeout after %s (last status: %s)", timeout, lastStatus)
3585 }
3586 }
3587}
3588
3589// pollContainerRunning polls until the container is running (max 30 seconds).
3590func (s *Service) pollContainerRunning(ctx context.Context, containerID string) error {
3591 for i := 0; i < 30; i++ {
3592 running, err := s.runtime.IsContainerRunning(ctx, containerID)
3593 if err != nil {
3594 return err
3595 }
3596 if running {
3597 return nil
3598 }
3599 // Check if the container has already exited (crash) rather than still starting up.
3600 if info, inspectErr := s.runtime.InspectContainer(ctx, containerID); inspectErr == nil && info != nil && info.Status == string(domain.ContainerStatusExited) {
3601 return fmt.Errorf("container exited immediately with code %d: %w", info.ExitCode, domain.ErrContainerExited)
3602 }
3603 if i == 29 {
3604 return fmt.Errorf("container did not start within 30 seconds")
3605 }
3606 select {
3607 case <-time.After(time.Second):
3608 case <-ctx.Done():
3609 return ctx.Err()
3610 }
3611 }
3612 return fmt.Errorf("container did not start within 30 seconds")
3613}
3614
3615// waitForRecovery waits for a container to recover within the recovery window.
3616func (s *Service) waitForRecovery(ctx context.Context, containerID string) error {
3617 log := zerowrap.FromCtx(ctx)
3618 log.Warn().
3619 Dur("recovery_window", readinessRecoveryWindow).
3620 Msg("container not running after readiness delay, waiting for recovery")
3621
3622 deadline := time.Now().Add(readinessRecoveryWindow)
3623 for time.Now().Before(deadline) {
3624 select {
3625 case <-time.After(time.Second):
3626 case <-ctx.Done():
3627 return ctx.Err()
3628 }
3629
3630 running, err := s.runtime.IsContainerRunning(ctx, containerID)
3631 if err != nil {
3632 return err
3633 }
3634 if running {
3635 log.Info().Msg("container recovered during readiness recovery window")
3636 return nil
3637 }
3638 }
3639
3640 return fmt.Errorf("container not running after readiness delay and recovery window")
3641}
3642
3643// publishContainerDeployed publishes a container.deployed event.
3644func (s *Service) publishContainerDeployed(ctx context.Context, domainName, containerID string) {
3645 payload := &domain.ContainerEventPayload{
3646 ContainerID: containerID,
3647 Domain: domainName,
3648 Action: "deployed",
3649 }
3650
3651 if err := s.eventBus.Publish(domain.EventContainerDeployed, payload); err != nil {
3652 log := zerowrap.FromCtx(ctx)
3653 log.Warn().Err(err).Msg("failed to publish container deployed event")
3654 }
3655}
3656
3657// rewriteToRegistryDomain rewrites an image reference to use the configured registry domain.
3658// e.g., "myapp:latest" -> "registry.example.com/myapp:latest"
3659func rewriteToRegistryDomain(imageRef, registryDomain string) string {
3660 if registryDomain == "" {
3661 return imageRef
3662 }
3663
3664 prefix := registryDomain + "/"
3665 if strings.HasPrefix(imageRef, prefix) {
3666 return imageRef
3667 }
3668
3669 return prefix + imageRef
3670}
3671
3672// rewriteToLocalRegistry rewrites Gordon-managed image references to use the
3673// local registry address. Current and legacy Gordon registry domains are
3674// stripped before prefixing localhost:<registryPort>/. Bare refs are prefixed.
3675// External registries are preserved unchanged.
3676func rewriteToLocalRegistry(imageRef, registryDomain string, legacyRegistryDomains []string, registryPort int) string {
3677 if imageRef == "" {
3678 return imageRef
3679 }
3680
3681 localRegistry := fmt.Sprintf("localhost:%d", registryPort)
3682 localPrefix := localRegistry + "/"
3683 if strings.HasPrefix(imageRef, localPrefix) {
3684 return imageRef
3685 }
3686
3687 if domain.IsGordonRegistryImageRef(imageRef, registryDomain, legacyRegistryDomains) {
3688 return localPrefix + domain.StripKnownGordonRegistry(imageRef, registryDomain, legacyRegistryDomains)
3689 }
3690 if hasExplicitRegistry(imageRef) {
3691 return imageRef
3692 }
3693
3694 return localPrefix + imageRef
3695}