1// Package container implements the container management use case.
2package container
3
4import (
5 "bufio"
6 "context"
7 "encoding/binary"
8 "encoding/hex"
9 "errors"
10 "fmt"
11 "io"
12 "maps"
13 "net"
14 "slices"
15 "strconv"
16 "strings"
17 "sync"
18 "time"
19
20 "github.com/bnema/zerowrap"
21 "go.opentelemetry.io/otel"
22 "go.opentelemetry.io/otel/attribute"
23 "go.opentelemetry.io/otel/codes"
24 "go.opentelemetry.io/otel/metric"
25 "go.opentelemetry.io/otel/trace"
26
27 "github.com/bnema/gordon/internal/adapters/out/telemetry"
28 "github.com/bnema/gordon/internal/boundaries/out"
29 "github.com/bnema/gordon/internal/domain"
30)
31
32// Config holds configuration needed by the container service.
33type Config struct {
34 RegistryAuthEnabled bool
35 RegistryDomain string
36 RegistryPort int
37 ServiceTokenUsername string
38 ServiceToken string
39 InternalRegistryUsername string
40 InternalRegistryPassword string
41 PullPolicy string
42 VolumeAutoCreate bool
43 VolumePrefix string
44 VolumePreserve bool
45 NetworkIsolation bool
46 NetworkPrefix string
47 NetworkGroups map[string][]string
48 NetworkInternal bool
49 Attachments map[string][]string
50 AllowedRegistries []string
51 RequireImageDigest bool
52 SecurityProfile string
53 ReadinessDelay time.Duration // Delay after container starts before considering it ready
54 ReadinessMode string // Readiness strategy: auto, docker-health, delay
55 HealthTimeout time.Duration // Max wait for health-based readiness
56 DrainDelay time.Duration // Grace period after cache invalidation before stopping old container
57 DrainDelayConfigured bool // True when deploy.drain_delay was explicitly configured
58 DrainMode string // Drain strategy: auto, inflight, delay
59 DrainTimeout time.Duration // Max wait for in-flight requests to drain
60 StabilizationDelay time.Duration // Post-switch monitoring window (default 2s)
61 TCPProbeTimeout time.Duration // TCP probe timeout (default 30s)
62 HTTPProbeTimeout time.Duration // HTTP probe timeout (default 60s)
63 AttachmentReadinessTimeout time.Duration // Max wait for attachment readiness (default 30s)
64 DefaultMemoryLimit int64 // Default memory limit in bytes for containers (0 = no limit)
65 DefaultNanoCPUs int64 // Default CPU quota in nanoseconds for containers (0 = no limit)
66 DefaultPidsLimit int64 // Default max PIDs for containers (0 = no limit)
67}
68
69var tracer = otel.Tracer("gordon.container")
70
71const (
72 PullPolicyAlways = "always"
73 PullPolicyIfNotPresent = "if-not-present"
74 PullPolicyIfTagChanged = "if-tag-changed"
75
76 // readinessRecoveryWindow is an additional grace window used when a container
77 // is briefly not running at the end of readiness delay. This avoids false
78 // negatives during short startup flaps.
79 readinessRecoveryWindow = 30 * time.Second
80 internalPullMaxAttempts = 3
81 deployFailureLogLimit = 20
82)
83
84// Service implements the ContainerService interface.
85type Service struct {
86 runtime out.ContainerRuntime
87 envLoader out.EnvLoader
88 eventBus out.EventPublisher
89 logWriter out.ContainerLogWriter
90 cacheInvalidator out.ProxyCacheInvalidator
91 drainWaiter out.ProxyDrainWaiter
92 config Config
93 configProvider AttachmentConfigProvider // live config reads for attachments/networks (may be nil)
94 metrics *telemetry.Metrics
95 containers map[string]*domain.Container
96 attachments map[string][]string // ownerDomain → []containerIDs
97 managedCount int64 // tracks UpDownCounter value for delta computation
98 mu sync.RWMutex
99 deployMu sync.Map // per-domain deploy locks (domain → *domainDeployLock)
100 cleanupWg sync.WaitGroup // tracks background old-container cleanup goroutines
101 monitor *Monitor
102}
103
104// domainDeployLock is a context-aware mutex using a buffered channel.
105// It allows Lock() to be interrupted by context cancellation.
106type domainDeployLock struct {
107 ch chan struct{}
108}
109
110// newDomainDeployLock creates a new lock that is initially unlocked.
111func newDomainDeployLock() *domainDeployLock {
112 l := &domainDeployLock{ch: make(chan struct{}, 1)}
113 l.ch <- struct{}{}
114 return l
115}
116
117// Lock acquires the lock or returns an error if the context is cancelled.
118func (l *domainDeployLock) Lock(ctx context.Context) error {
119 // Fast path: honor already-canceled contexts before blocking.
120 if err := ctx.Err(); err != nil {
121 return err
122 }
123 select {
124 case <-ctx.Done():
125 return ctx.Err()
126 case <-l.ch:
127 // Re-check context after acquisition: if canceled while we were
128 // in the select, release the token so the lock isn't consumed.
129 if err := ctx.Err(); err != nil {
130 l.ch <- struct{}{}
131 return err
132 }
133 return nil
134 }
135}
136
137// Unlock releases the lock.
138func (l *domainDeployLock) Unlock() {
139 select {
140 case l.ch <- struct{}{}:
141 default:
142 // This should never happen if Lock/Unlock are used correctly.
143 // Panic to detect double-unlocks or other misuse early.
144 panic("domainDeployLock: double unlock detected")
145 }
146}
147
148// NewService creates a new container service.
149func NewService(
150 runtime out.ContainerRuntime,
151 envLoader out.EnvLoader,
152 eventBus out.EventPublisher,
153 logWriter out.ContainerLogWriter,
154 config Config,
155 configProvider AttachmentConfigProvider,
156) *Service {
157 return &Service{
158 runtime: runtime,
159 envLoader: envLoader,
160 eventBus: eventBus,
161 logWriter: logWriter,
162 config: config,
163 configProvider: configProvider,
164 containers: make(map[string]*domain.Container),
165 attachments: make(map[string][]string),
166 }
167}
168
169// SetMetrics sets the telemetry metrics for the container service.
170func (s *Service) SetMetrics(m *telemetry.Metrics) {
171 s.metrics = m
172}
173
174// SetProxyCacheInvalidator sets the proxy cache invalidator for synchronous
175// cache invalidation during zero-downtime deployments. This must be called
176// after construction because the proxy service is created after the container
177// service during application initialization.
178func (s *Service) SetProxyCacheInvalidator(inv out.ProxyCacheInvalidator) {
179 s.mu.Lock()
180 s.cacheInvalidator = inv
181 s.mu.Unlock()
182}
183
184// SetProxyDrainWaiter sets the proxy in-flight drain waiter used during
185// zero-downtime replacement before stopping the old container.
186func (s *Service) SetProxyDrainWaiter(waiter out.ProxyDrainWaiter) {
187 s.mu.Lock()
188 s.drainWaiter = waiter
189 s.mu.Unlock()
190}
191
192// domainDeployMu returns a per-domain lock for serializing deploy operations.
193func (s *Service) domainDeployMu(domain string) *domainDeployLock {
194 v, _ := s.deployMu.LoadOrStore(domain, newDomainDeployLock())
195 // Type assertion is safe: LoadOrStore always stores *domainDeployLock values,
196 // and any concurrent LoadOrStore calls will also store *domainDeployLock.
197 return v.(*domainDeployLock)
198}
199
200// acquireDomainDeployLock acquires the per-domain deploy lock with context cancellation support.
201// Returns an unlock function that must be called to release the lock.
202func (s *Service) acquireDomainDeployLock(ctx context.Context, domain string) (func(), error) {
203 lock := s.domainDeployMu(domain)
204 if err := lock.Lock(ctx); err != nil {
205 return nil, err
206 }
207 return func() { lock.Unlock() }, nil
208}
209
210// buildContainerConfig constructs the container configuration for deployment.
211func (s *Service) deploymentContainerName(containerDomain string, existing *domain.Container) string {
212 canonicalName := fmt.Sprintf("gordon-%s", containerDomain)
213 if existing == nil {
214 return canonicalName
215 }
216
217 // Alternate temporary names if the tracked container was left with a temp suffix.
218 // This prevents name collisions after interrupted zero-downtime deploys.
219 newName := canonicalName + "-new"
220 nextName := canonicalName + "-next"
221 switch existing.Name {
222 case newName:
223 return nextName
224 case nextName:
225 return newName
226 default:
227 return newName
228 }
229}
230
231// containerConfigInput groups the parameters for building a container configuration.
232type containerConfigInput struct {
233 Domain string
234 Image string
235 ImageRef string
236 ExposedPorts []int
237 EnvVars []string
238 EnvHash string
239 Volumes map[string]string
240 NetworkName string
241 ImageLabels map[string]string
242 Existing *domain.Container
243}
244
245func (s *Service) buildContainerConfig(in containerConfigInput) *domain.ContainerConfig {
246 s.mu.RLock()
247 cfg := s.config
248 s.mu.RUnlock()
249
250 containerName := s.deploymentContainerName(in.Domain, in.Existing)
251 labels := map[string]string{
252 domain.LabelDomain: in.Domain,
253 domain.LabelEnvHash: in.EnvHash,
254 domain.LabelImage: in.Image,
255 domain.LabelManaged: "true",
256 domain.LabelRoute: in.Domain,
257 }
258
259 // Propagate proxy/port labels from image so readiness probes can find them
260 for _, key := range []string{domain.LabelProxyPort, domain.LabelPort, domain.LabelHealth} {
261 if v, ok := in.ImageLabels[key]; ok && v != "" {
262 labels[key] = v
263 }
264 }
265
266 // Ensure the label-specified proxy port is included in exposed ports
267 // so Docker creates a host port binding for it.
268 ports := in.ExposedPorts
269 if portStr, ok := labels[domain.LabelProxyPort]; ok {
270 if port, err := strconv.Atoi(portStr); err == nil && port > 0 && port <= 65535 {
271 if !slices.Contains(ports, port) {
272 ports = append(ports, port)
273 }
274 }
275 }
276
277 containerConfig := &domain.ContainerConfig{
278 Image: in.ImageRef,
279 Name: containerName,
280 Ports: ports,
281 Env: in.EnvVars,
282 Volumes: in.Volumes,
283 NetworkMode: in.NetworkName,
284 Hostname: in.Domain,
285 Labels: labels,
286 AutoRemove: false,
287 MemoryLimit: cfg.DefaultMemoryLimit,
288 NanoCPUs: cfg.DefaultNanoCPUs,
289 PidsLimit: cfg.DefaultPidsLimit,
290 }
291 applySecurityProfile(containerConfig, cfg)
292 return containerConfig
293}
294
295// Deploy creates and starts a container for the given route.
296// Implements zero-downtime deployment: new container starts before old one stops.
297func (s *Service) Deploy(ctx context.Context, route domain.Route) (*domain.Container, error) {
298 ctx, span := tracer.Start(ctx, "container.deploy",
299 trace.WithAttributes(
300 attribute.String("domain", route.Domain),
301 attribute.String("image", route.Image),
302 ))
303 defer span.End()
304
305 deployStart := time.Now()
306
307 // Serialize deploys for the same domain to prevent race conditions
308 // (e.g. multiple image.pushed events + explicit deploy call from CLI).
309 unlock, err := s.acquireDomainDeployLock(ctx, route.Domain)
310 if err != nil {
311 return nil, err
312 }
313 defer unlock()
314
315 // Enrich context with use case fields for all downstream logs
316 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
317 zerowrap.FieldLayer: "usecase",
318 zerowrap.FieldUseCase: "Deploy",
319 "domain": route.Domain,
320 })
321 log := zerowrap.FromCtx(ctx)
322
323 // Record deploy metrics and trace status
324 defer s.recordDeployMetrics(ctx, span, route, deployStart, &err)
325
326 existing, hasExisting := s.resolveExistingContainer(ctx, route.Domain)
327
328 resources, err := s.prepareDeployResources(ctx, route, existing)
329 if err != nil {
330 if deployErr := s.wrapPullDeployFailure(route, existing, err); deployErr != nil {
331 err = deployErr
332 return nil, err
333 }
334 return nil, err
335 }
336
337 // Skip redundant deploy: if the existing container is already running
338 // the exact same image (by Docker image ID), return it immediately.
339 // This prevents the double-deploy caused by the event-based deploy
340 // (triggered by image.pushed) racing with the explicit CLI deploy call.
341 if hasExisting {
342 existingForSkip := existing
343 if existingForSkip.ImageID == "" && existingForSkip.Image != "" && normalizeImageRef(existingForSkip.Image) == normalizeImageRef(route.Image) {
344 existingForSkip = s.containerForRedundantCheck(ctx, existingForSkip)
345 }
346 if existingForSkip.ImageID != "" {
347 if skip, container := s.skipRedundantDeploy(ctx, existingForSkip, resources.actualImageRef, resources.envHash); skip {
348 return container, nil
349 }
350 }
351 }
352
353 newContainer, err := s.createStartedContainer(ctx, route, existing, resources)
354 if err != nil {
355 return nil, err
356 }
357
358 invalidated := s.activateDeployedContainer(ctx, route.Domain, newContainer)
359
360 // Post-switch stabilization: verify new container stays running
361 if hasExisting {
362 stable, stabilizeErr := s.stabilizeNewContainer(ctx, route.Domain, newContainer, existing)
363 if stabilizeErr != nil {
364 // Both old and new containers are dead; assign to named return so
365 // the deferred recordDeployMetrics and span see the failure.
366 err = stabilizeErr
367 return nil, err
368 }
369 if !stable {
370 // Rollback performed — old container is restored
371 return existing, nil
372 }
373 }
374
375 // Finalize old container in the background — the new container is already
376 // serving traffic, so there's no reason to block the deploy response while
377 // waiting for the old container to stop (which can take 20s if the app
378 // doesn't handle SIGTERM).
379 s.cleanupWg.Add(1)
380 go func() {
381 defer s.cleanupWg.Done()
382 s.finalizePreviousContainer(context.WithoutCancel(ctx), route.Domain, existing, hasExisting, invalidated, newContainer.ID)
383 }()
384
385 // Start container log collection (non-blocking, errors don't fail deployment)
386 s.startLogCollection(ctx, newContainer.ID, route.Domain)
387
388 log.Info().
389 Str("image", route.Image).
390 Str(zerowrap.FieldEntityID, newContainer.ID).
391 Ints("ports", newContainer.Ports).
392 Str("network", resources.networkName).
393 Bool("zero_downtime", hasExisting).
394 Msg("container deployed successfully")
395
396 return newContainer, nil
397}
398
399// recordDeployMetrics records span error status and deploy metrics at the end of a Deploy call.
400// It is called via defer and receives a pointer to the named return error so it can observe
401// the final error value after all deferred functions have run.
402func (s *Service) recordDeployMetrics(ctx context.Context, span trace.Span, route domain.Route, start time.Time, errPtr *error) {
403 err := *errPtr
404 if err != nil {
405 span.RecordError(err)
406 span.SetStatus(codes.Error, err.Error())
407 }
408 if s.metrics == nil {
409 return
410 }
411 attrs := metric.WithAttributes(
412 attribute.String("domain", route.Domain),
413 attribute.String("image", route.Image),
414 )
415 s.metrics.DeployTotal.Add(ctx, 1, attrs)
416 s.metrics.DeployDuration.Record(ctx, time.Since(start).Seconds(), attrs)
417 if err != nil {
418 s.metrics.DeployErrors.Add(ctx, 1, attrs)
419 }
420}
421
422// skipRedundantDeploy checks whether the existing container is already running
423// the same image (by Docker image ID) as the one we are about to deploy.
424// When a push triggers both an event-based deploy and an explicit CLI deploy,
425// the second one arrives after the first has already completed; this avoids
426// a full redundant create-start-readiness cycle.
427func (s *Service) skipRedundantDeploy(ctx context.Context, existing *domain.Container, actualImageRef, envHash string) (bool, *domain.Container) {
428 log := zerowrap.FromCtx(ctx)
429
430 newImageID, err := s.runtime.GetImageID(ctx, actualImageRef)
431 if err != nil {
432 log.Debug().Err(err).Msg("cannot resolve image ID for redundancy check, proceeding with deploy")
433 return false, nil
434 }
435
436 if existing.ImageID != newImageID {
437 return false, nil
438 }
439
440 // Verify the container is actually still running before skipping.
441 running, err := s.runtime.IsContainerRunning(ctx, existing.ID)
442 if err != nil || !running {
443 return false, nil
444 }
445
446 if existing.Labels != nil {
447 existingEnvHash, hasEnvHash := existing.Labels[domain.LabelEnvHash]
448 if !hasEnvHash || existingEnvHash != envHash {
449 log.Info().
450 Str("container_id", existing.ID).
451 Bool("has_env_hash", hasEnvHash).
452 Msg("env changed, proceeding with deploy despite same image")
453 return false, nil
454 }
455 } else {
456 return false, nil
457 }
458
459 log.Info().
460 Str("container_id", existing.ID).
461 Str("image_id", newImageID).
462 Msg("skipping redundant deploy: container already running this image")
463
464 return true, existing
465}
466
467func (s *Service) containerForRedundantCheck(ctx context.Context, existing *domain.Container) *domain.Container {
468 if existing == nil || existing.ImageID != "" || existing.ID == "" {
469 return existing
470 }
471
472 log := zerowrap.FromCtx(ctx)
473 inspected, err := s.runtime.InspectContainer(ctx, existing.ID)
474 if err != nil {
475 log.Debug().Err(err).Str("container_id", existing.ID).Msg("cannot inspect existing container for redundancy check")
476 return existing
477 }
478 if inspected == nil || inspected.ImageID == "" {
479 return existing
480 }
481
482 existingCopy := *existing
483 existingCopy.ImageID = inspected.ImageID
484 return &existingCopy
485}
486
487type deployResources struct {
488 networkName string
489 actualImageRef string
490 exposedPorts []int
491 imageLabels map[string]string
492 envVars []string
493 envHash string
494 volumes map[string]string
495}
496
497// resolveExistingContainer returns the currently running container for a domain.
498// It first checks the in-memory map (fast path). If not found, it queries the
499// runtime for a running container with matching name and managed label. This
500// handles cases where Gordon restarted and in-memory state is stale.
501//
502// The slow path checks canonical, -new, and -next names because a previous
503// deploy may have been interrupted (e.g. eventbus timeout, Gordon restart)
504// leaving the active container under a temp name. Canonical is preferred when
505// multiple matches exist.
506func (s *Service) resolveExistingContainer(ctx context.Context, domainName string) (*domain.Container, bool) {
507 // Fast path: check in-memory state
508 s.mu.RLock()
509 container, ok := s.containers[domainName]
510 s.mu.RUnlock()
511 if ok {
512 return container, true
513 }
514
515 // Slow path: query runtime for running containers
516 log := zerowrap.FromCtx(ctx)
517 canonicalName := fmt.Sprintf("gordon-%s", domainName)
518 candidateNames := map[string]bool{
519 canonicalName: true,
520 canonicalName + "-new": true,
521 canonicalName + "-next": true,
522 }
523
524 running, err := s.runtime.ListContainers(ctx, false)
525 if err != nil {
526 log.Warn().Err(err).Msg("failed to list running containers for existing container resolution")
527 return nil, false
528 }
529
530 // Prefer canonical name; fall back to any running managed temp container.
531 var best *domain.Container
532 for _, c := range running {
533 if !candidateNames[c.Name] || c.Labels[domain.LabelManaged] != "true" {
534 continue
535 }
536 if best == nil || c.Name == canonicalName {
537 best = c
538 }
539 }
540
541 if best != nil {
542 log.Info().
543 Str("container_id", best.ID).
544 Str("container_name", best.Name).
545 Msg("resolved existing container from runtime (in-memory state was stale)")
546
547 // Update in-memory state so subsequent lookups are fast
548 s.mu.Lock()
549 if _, alreadyTracked := s.containers[domainName]; !alreadyTracked {
550 s.managedCount++
551 }
552 s.containers[domainName] = best
553 s.mu.Unlock()
554
555 return best, true
556 }
557
558 return nil, false
559}
560
561func (s *Service) prepareDeployResources(ctx context.Context, route domain.Route, existing *domain.Container) (*deployResources, error) {
562 log := zerowrap.FromCtx(ctx)
563
564 existingID := ""
565 if existing != nil {
566 existingID = existing.ID
567 }
568 if err := s.cleanupOrphanedContainers(ctx, route.Domain, existingID); err != nil {
569 log.WrapErr(err, "failed to cleanup orphaned containers")
570 }
571
572 imageRef, err := s.buildValidatedImageRef(ctx, route.Image)
573 if err != nil {
574 return nil, err
575 }
576 actualImageRef, err := s.ensureImage(ctx, imageRef)
577 if err != nil {
578 return nil, err
579 }
580
581 networkName := s.getNetworkForApp(route.Domain)
582 if err := s.createNetworkIfNeeded(ctx, networkName); err != nil {
583 return nil, log.WrapErr(err, "failed to create network")
584 }
585 if err := s.deployAttachments(ctx, route.Domain, networkName); err != nil {
586 return nil, log.WrapErr(err, "failed to deploy attachments")
587 }
588
589 exposedPorts, err := s.runtime.GetImageExposedPorts(ctx, actualImageRef)
590 if err != nil {
591 log.WrapErr(err, "failed to get exposed ports, using defaults")
592 exposedPorts = []int{80, 8080, 3000}
593 }
594
595 imageLabels, err := s.runtime.GetImageLabels(ctx, actualImageRef)
596 if err != nil {
597 log.Warn().Err(err).Msg("failed to get image labels, skipping label propagation")
598 imageLabels = nil
599 }
600
601 envVars, err := s.loadEnvironment(ctx, route.Env, route.Domain, actualImageRef)
602 if err != nil {
603 return nil, err
604 }
605 envHash := hashEnvironment(envVars)
606
607 volumes, err := s.setupVolumes(ctx, route.Domain, actualImageRef)
608 if err != nil {
609 return nil, err
610 }
611
612 return &deployResources{
613 networkName: networkName,
614 actualImageRef: actualImageRef,
615 exposedPorts: exposedPorts,
616 imageLabels: imageLabels,
617 envVars: envVars,
618 envHash: envHash,
619 volumes: volumes,
620 }, nil
621}
622
623func (s *Service) createStartedContainer(ctx context.Context, route domain.Route, existing *domain.Container, resources *deployResources) (*domain.Container, error) {
624 ctx, span := tracer.Start(ctx, "container.create_and_start")
625 defer span.End()
626
627 log := zerowrap.FromCtx(ctx)
628
629 containerConfig := s.buildContainerConfig(containerConfigInput{
630 Domain: route.Domain,
631 Image: route.Image,
632 ImageRef: resources.actualImageRef,
633 ExposedPorts: resources.exposedPorts,
634 EnvVars: resources.envVars,
635 EnvHash: resources.envHash,
636 Volumes: resources.volumes,
637 NetworkName: resources.networkName,
638 ImageLabels: resources.imageLabels,
639 Existing: existing,
640 })
641
642 newContainer, err := s.runtime.CreateContainer(ctx, containerConfig)
643 if err != nil {
644 return nil, log.WrapErr(err, "failed to create container")
645 }
646 if err := s.runtime.StartContainer(ctx, newContainer.ID); err != nil {
647 s.runtime.RemoveContainer(ctx, newContainer.ID, true)
648 return nil, log.WrapErr(err, "failed to start container")
649 }
650 if !domain.IsSkipReadiness(ctx) {
651 if err := s.waitForReady(ctx, newContainer.ID, containerConfig); err != nil {
652 deployErr := s.newDeployFailure(newContainer, "container failed readiness check", err, s.captureRecentContainerLogs(ctx, newContainer.ID))
653 s.cleanupFailedContainer(ctx, newContainer.ID)
654 return nil, deployErr
655 }
656 }
657
658 inspected, err := s.runtime.InspectContainer(ctx, newContainer.ID)
659 if err != nil {
660 s.cleanupFailedContainer(ctx, newContainer.ID)
661 return nil, log.WrapErr(err, "failed to inspect started container")
662 }
663
664 return inspected, nil
665}
666
667func (s *Service) newDeployFailure(container *domain.Container, cause string, err error, logs []string) error {
668 deployErr := &domain.DeployFailureError{
669 Summary: "failed to deploy",
670 Cause: cause,
671 Hint: inferDeployFailureHint(err),
672 Logs: logs,
673 Err: err,
674 }
675 if container != nil {
676 deployErr.ContainerName = container.Name
677 deployErr.ContainerID = container.ID
678 }
679
680 return deployErr
681}
682
683func (s *Service) wrapPullDeployFailure(route domain.Route, existing *domain.Container, err error) error {
684 if !isPullFailure(err) {
685 return nil
686 }
687
688 return s.newDeployFailure(&domain.Container{Name: s.deploymentContainerName(route.Domain, existing)}, "failed to pull image", err, nil)
689}
690
691func (s *Service) captureRecentContainerLogs(ctx context.Context, containerID string) []string {
692 if ctx.Err() != nil {
693 return nil
694 }
695
696 logStream, err := s.runtime.GetContainerLogs(ctx, containerID, false)
697 if err != nil {
698 return nil
699 }
700 defer func() {
701 _ = logStream.Close()
702 }()
703
704 logs, err := parseDockerLogLines(logStream, deployFailureLogLimit)
705 if err != nil {
706 return nil
707 }
708
709 return logs
710}
711
712func parseDockerLogLines(r io.Reader, maxLines int) ([]string, error) {
713 reader := bufio.NewReader(r)
714 collector := newLogLineCollector(maxLines)
715
716 header, err := reader.Peek(8)
717 if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, bufio.ErrBufferFull) {
718 return nil, err
719 }
720
721 if len(header) >= 8 && looksLikeDockerLogStream(header) {
722 return parseDockerFramedLogLines(reader, collector)
723 }
724
725 return parsePlainLogLines(reader, collector)
726}
727
728func parseDockerFramedLogLines(reader *bufio.Reader, collector *logLineCollector) ([]string, error) {
729 for {
730 frameHeader := make([]byte, 8)
731 if _, err := io.ReadFull(reader, frameHeader); err != nil {
732 if isLogStreamEOF(err) {
733 collector.flush()
734 return collector.lines, nil
735 }
736 return nil, err
737 }
738
739 size := int(binary.BigEndian.Uint32(frameHeader[4:8]))
740 if size <= 0 {
741 continue
742 }
743
744 payload, err := readLogPayload(reader, size)
745 if err != nil {
746 collector.addChunk(string(payload))
747 collector.flush()
748 if isLogStreamEOF(err) {
749 return collector.lines, nil
750 }
751 return nil, err
752 }
753
754 collector.addChunk(string(payload))
755 }
756}
757
758func parsePlainLogLines(reader *bufio.Reader, collector *logLineCollector) ([]string, error) {
759
760 scanner := bufio.NewScanner(reader)
761 for scanner.Scan() {
762 collector.addLine(scanner.Text())
763 }
764 if err := scanner.Err(); err != nil {
765 return nil, err
766 }
767
768 collector.flush()
769 return collector.lines, nil
770}
771
772func readLogPayload(reader *bufio.Reader, size int) ([]byte, error) {
773 payload := make([]byte, size)
774 _, err := io.ReadFull(reader, payload)
775 return payload, err
776}
777
778func isLogStreamEOF(err error) bool {
779 return errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF)
780}
781
782func looksLikeDockerLogStream(data []byte) bool {
783 return len(data) >= 8 && data[1] == 0 && data[2] == 0 && data[3] == 0
784}
785
786type logLineCollector struct {
787 lines []string
788 pending string
789 maxLines int
790}
791
792func newLogLineCollector(maxLines int) *logLineCollector {
793 return &logLineCollector{maxLines: maxLines}
794}
795
796func (c *logLineCollector) addChunk(chunk string) {
797 for _, part := range strings.SplitAfter(chunk, "\n") {
798 c.pending += part
799 if strings.HasSuffix(part, "\n") {
800 c.flush()
801 }
802 }
803}
804
805func (c *logLineCollector) addLine(line string) {
806 line = strings.TrimRight(line, "\r")
807 if line == "" {
808 return
809 }
810 c.lines = append(c.lines, line)
811 if c.maxLines > 0 && len(c.lines) > c.maxLines {
812 c.lines = c.lines[len(c.lines)-c.maxLines:]
813 }
814}
815
816func (c *logLineCollector) flush() {
817 if c.pending == "" {
818 return
819 }
820 c.addLine(strings.TrimRight(c.pending, "\r\n"))
821 c.pending = ""
822}
823
824func inferDeployFailureHint(err error) string {
825 if err == nil {
826 return ""
827 }
828
829 msg := strings.ToLower(err.Error())
830 switch {
831 case strings.Contains(msg, "healthcheck timeout"):
832 return "check startup logs and confirm the health endpoint becomes ready within the configured timeout"
833 case strings.Contains(msg, "unhealthy"):
834 return "inspect recent app logs and verify the healthcheck command reflects actual readiness"
835 case strings.Contains(msg, "no healthcheck detected"):
836 return "add a Docker healthcheck or switch readiness mode"
837 case isPullFailure(err), strings.Contains(msg, "access denied"), strings.Contains(msg, "unauthorized"):
838 return "verify registry auth and confirm the image exists at the requested tag"
839 default:
840 return ""
841 }
842}
843
844func isPullFailure(err error) bool {
845 if err == nil {
846 return false
847 }
848
849 if errors.Is(err, domain.ErrImagePullFailed) {
850 return true
851 }
852
853 msg := strings.ToLower(err.Error())
854 return strings.Contains(msg, "failed to pull image") ||
855 strings.Contains(msg, "pull access denied")
856}
857
858func (s *Service) activateDeployedContainer(ctx context.Context, domainName string, container *domain.Container) bool {
859 s.mu.Lock()
860 _, wasTracked := s.containers[domainName]
861 s.containers[domainName] = container
862 if !wasTracked {
863 s.managedCount++
864 }
865 s.mu.Unlock()
866
867 // Track managed container count (only increment for new domains, not replacements)
868 if !wasTracked && s.metrics != nil {
869 s.metrics.ManagedContainers.Add(ctx, 1)
870 }
871
872 s.publishContainerDeployed(ctx, domainName, container.ID)
873
874 s.mu.RLock()
875 inv := s.cacheInvalidator
876 s.mu.RUnlock()
877 if inv != nil {
878 inv.InvalidateTarget(ctx, domainName)
879 return true
880 }
881
882 return false
883}
884
885// stabilizeNewContainer monitors the new container briefly after traffic switch.
886// If it crashes during this window, rolls back to old container.
887// Returns true if stabilization succeeded, false if rollback was performed.
888// Returns an error only when both new and old containers are dead.
889func (s *Service) stabilizeNewContainer(ctx context.Context, domainName string, newContainer, oldContainer *domain.Container) (bool, error) {
890 log := zerowrap.FromCtx(ctx)
891
892 if oldContainer == nil {
893 return true, nil
894 }
895
896 s.mu.RLock()
897 delay := s.config.StabilizationDelay
898 s.mu.RUnlock()
899 if delay == 0 {
900 delay = 2 * time.Second
901 }
902
903 // Brief stabilization: verify new container is still running after delay
904 select {
905 case <-time.After(delay):
906 case <-ctx.Done():
907 return true, nil
908 }
909
910 running, err := s.runtime.IsContainerRunning(ctx, newContainer.ID)
911 if err != nil || !running {
912 log.Error().
913 Str("new_container", newContainer.ID).
914 Str("old_container", oldContainer.ID).
915 Msg("new container crashed during stabilization, rolling back to old")
916
917 // Verify old container is still running before restoring it.
918 // If both old and new are dead (e.g., OOM), returning a dead
919 // container as "existing" would leave the domain in a broken state.
920 oldRunning, oldErr := s.runtime.IsContainerRunning(ctx, oldContainer.ID)
921 if oldErr != nil || !oldRunning {
922 log.Error().
923 Str("old_container", oldContainer.ID).
924 Msg("old container is also not running, cannot rollback")
925
926 // Cleanup failed new container
927 if stopErr := s.runtime.StopContainer(ctx, newContainer.ID); stopErr != nil {
928 log.WrapErrWithFields(stopErr, "failed to stop failed new container during rollback", map[string]any{zerowrap.FieldEntityID: newContainer.ID})
929 }
930 if removeErr := s.runtime.RemoveContainer(ctx, newContainer.ID, true); removeErr != nil {
931 log.WrapErrWithFields(removeErr, "failed to remove failed new container during rollback", map[string]any{zerowrap.FieldEntityID: newContainer.ID})
932 }
933
934 return false, fmt.Errorf("stabilization failed: new container crashed and old container %s is also not running", oldContainer.ID)
935 }
936
937 // Rollback: restore old container as tracked
938 s.mu.Lock()
939 s.containers[domainName] = oldContainer
940 s.mu.Unlock()
941
942 // Re-invalidate proxy cache to point back to old
943 s.mu.RLock()
944 inv := s.cacheInvalidator
945 s.mu.RUnlock()
946 if inv != nil {
947 inv.InvalidateTarget(ctx, domainName)
948 }
949
950 // Cleanup failed new container
951 if stopErr := s.runtime.StopContainer(ctx, newContainer.ID); stopErr != nil {
952 log.WrapErrWithFields(stopErr, "failed to stop failed new container during rollback", map[string]any{zerowrap.FieldEntityID: newContainer.ID})
953 }
954 if removeErr := s.runtime.RemoveContainer(ctx, newContainer.ID, true); removeErr != nil {
955 log.WrapErrWithFields(removeErr, "failed to remove failed new container during rollback", map[string]any{zerowrap.FieldEntityID: newContainer.ID})
956 }
957
958 return false, nil
959 }
960
961 return true, nil
962}
963
964func (s *Service) finalizePreviousContainer(ctx context.Context, domainName string, existing *domain.Container, hasExisting, invalidated bool, newContainerID string) {
965 if !hasExisting {
966 return
967 }
968
969 if invalidated {
970 s.waitForDrain(ctx, existing.ID)
971 }
972
973 s.cleanupOldContainer(ctx, existing, newContainerID, domainName)
974}
975
976func (s *Service) waitForDrain(ctx context.Context, oldContainerID string) {
977 log := zerowrap.FromCtx(ctx)
978
979 s.mu.RLock()
980 cfg := s.config
981 waiter := s.drainWaiter
982 s.mu.RUnlock()
983
984 drainMode := cfg.DrainMode
985 if drainMode == "" {
986 drainMode = "delay"
987 }
988
989 shouldUseInFlight := (drainMode == "inflight" || drainMode == "auto") && waiter != nil
990 if shouldUseInFlight {
991 timeout := cfg.DrainTimeout
992 if timeout == 0 {
993 timeout = 30 * time.Second
994 }
995 drained := waiter.WaitForNoInFlight(ctx, oldContainerID, timeout)
996 if !drained {
997 log.Warn().
998 Str("old_container_id", oldContainerID).
999 Dur("drain_timeout", timeout).
1000 Msg("drain wait timed out; old container may still have in-flight traffic")
1001 }
1002 return
1003 }
1004
1005 drainDelay := 2 * time.Second
1006 if cfg.DrainDelayConfigured {
1007 drainDelay = cfg.DrainDelay
1008 }
1009 if drainDelay <= 0 {
1010 return
1011 }
1012 select {
1013 case <-time.After(drainDelay):
1014 case <-ctx.Done():
1015 }
1016}
1017
1018// restartContainerWithRecovery restarts a container, reconciling stale state if needed.
1019// It returns the (possibly refreshed) container and updated attachment IDs.
1020func (s *Service) restartContainerWithRecovery(ctx context.Context, domainName string, container *domain.Container, attachmentIDs []string) (*domain.Container, []string, error) {
1021 log := zerowrap.FromCtx(ctx)
1022
1023 if err := s.runtime.RestartContainer(ctx, container.ID); err != nil {
1024 if !isContainerNotFoundError(err) {
1025 return nil, nil, log.WrapErr(err, "failed to restart container")
1026 }
1027
1028 // In-memory container ID can become stale after external runtime changes.
1029 // Re-sync and retry once with the latest tracked container.
1030 log.Warn().
1031 Err(err).
1032 Str(zerowrap.FieldEntityID, container.ID).
1033 Msg("tracked container missing during restart, attempting state reconciliation")
1034
1035 if syncErr := s.SyncContainers(ctx); syncErr != nil {
1036 log.Warn().Err(syncErr).Msg("failed to sync container state during restart recovery")
1037 }
1038
1039 s.mu.RLock()
1040 refreshed, refreshedExists := s.containers[domainName]
1041 attachmentIDs = append([]string{}, s.attachments[domainName]...)
1042 s.mu.RUnlock()
1043
1044 if !refreshedExists || refreshed == nil {
1045 return nil, nil, domain.ErrContainerNotFound
1046 }
1047 if err := s.runtime.RestartContainer(ctx, refreshed.ID); err != nil {
1048 return nil, nil, log.WrapErr(err, "failed to restart container after state reconciliation")
1049 }
1050 return refreshed, attachmentIDs, nil
1051 }
1052
1053 return container, attachmentIDs, nil
1054}
1055
1056// Restart restarts a running container for the given domain.
1057func (s *Service) Restart(ctx context.Context, domainName string, withAttachments bool) error {
1058 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1059 zerowrap.FieldLayer: "usecase",
1060 zerowrap.FieldUseCase: "Restart",
1061 "domain": domainName,
1062 })
1063 log := zerowrap.FromCtx(ctx)
1064
1065 s.mu.RLock()
1066 container, exists := s.containers[domainName]
1067 attachments := s.attachments[domainName]
1068 var attachmentIDs []string
1069 if len(attachments) > 0 {
1070 attachmentIDs = make([]string, len(attachments))
1071 copy(attachmentIDs, attachments)
1072 }
1073 s.mu.RUnlock()
1074
1075 if !exists || container == nil {
1076 return domain.ErrContainerNotFound
1077 }
1078
1079 // When attachments are requested, sync runtime state first to get accurate
1080 // attachment tracking, then check if configured attachments are deployed.
1081 // If fewer attachment containers are tracked than configured, fail early with
1082 // guidance rather than restarting the main container into a broken state
1083 // (e.g., missing database hostname).
1084 if withAttachments {
1085 if syncErr := s.SyncContainers(ctx); syncErr != nil {
1086 log.Warn().Err(syncErr).Msg("failed to sync container state before attachment check")
1087 }
1088
1089 // Re-read attachment IDs after sync
1090 s.mu.RLock()
1091 if freshAttachments := s.attachments[domainName]; len(freshAttachments) > 0 {
1092 attachmentIDs = make([]string, len(freshAttachments))
1093 copy(attachmentIDs, freshAttachments)
1094 } else {
1095 attachmentIDs = nil
1096 }
1097 s.mu.RUnlock()
1098
1099 configuredAttachments := s.resolveAttachmentsForDomain(domainName)
1100 if len(configuredAttachments) > 0 && len(attachmentIDs) < len(configuredAttachments) {
1101 missing := len(configuredAttachments) - len(attachmentIDs)
1102 return fmt.Errorf("%w: domain %q has %d configured attachment(s) but only %d deployed (%d missing)",
1103 domain.ErrAttachmentNotDeployed, domainName, len(configuredAttachments), len(attachmentIDs), missing)
1104 }
1105 }
1106
1107 // Restart the main container (with stale-state recovery)
1108 container, attachmentIDs, err := s.restartContainerWithRecovery(ctx, domainName, container, attachmentIDs)
1109 if err != nil {
1110 return err
1111 }
1112 log.Info().Str(zerowrap.FieldEntityID, container.ID).Msg("container restarted")
1113
1114 // Record restart metric
1115 if s.metrics != nil {
1116 s.metrics.ContainerRestarts.Add(ctx, 1, metric.WithAttributes(
1117 attribute.String("domain", domainName),
1118 attribute.String("source", "api"),
1119 ))
1120 }
1121
1122 // Restart attachments if requested
1123 if withAttachments && len(attachmentIDs) > 0 {
1124 for _, attachID := range attachmentIDs {
1125 if err := s.runtime.RestartContainer(ctx, attachID); err != nil {
1126 log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to restart attachment")
1127 continue // Don't fail the whole operation for one attachment
1128 }
1129 log.Info().Str("attachment_id", attachID).Msg("attachment restarted")
1130 }
1131 }
1132
1133 return nil
1134}
1135
1136// Stop stops a running container.
1137func (s *Service) Stop(ctx context.Context, containerID string) error {
1138 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1139 zerowrap.FieldLayer: "usecase",
1140 zerowrap.FieldUseCase: "Stop",
1141 zerowrap.FieldEntityID: containerID,
1142 })
1143 log := zerowrap.FromCtx(ctx)
1144
1145 // Stop log collection before stopping container
1146 if s.logWriter != nil {
1147 if err := s.logWriter.StopLogging(containerID); err != nil {
1148 log.Warn().Err(err).Msg("failed to stop container log collection")
1149 }
1150 }
1151
1152 if err := s.runtime.StopContainer(ctx, containerID); err != nil {
1153 return log.WrapErr(err, "failed to stop container")
1154 }
1155
1156 log.Info().Msg("container stopped")
1157 return nil
1158}
1159
1160// Remove removes a container.
1161func (s *Service) Remove(ctx context.Context, containerID string, force bool) error {
1162 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1163 zerowrap.FieldLayer: "usecase",
1164 zerowrap.FieldUseCase: "Remove",
1165 zerowrap.FieldEntityID: containerID,
1166 })
1167 log := zerowrap.FromCtx(ctx)
1168
1169 // Stop log collection before removing container
1170 if s.logWriter != nil {
1171 if err := s.logWriter.StopLogging(containerID); err != nil {
1172 log.Warn().Err(err).Msg("failed to stop container log collection")
1173 }
1174 }
1175
1176 // Find domain and attachment IDs for cleanup
1177 containerDomain, attachmentIDs, cfg := s.findContainerContext(containerID)
1178
1179 if err := s.runtime.RemoveContainer(ctx, containerID, force); err != nil {
1180 return log.WrapErr(err, "failed to remove container")
1181 }
1182
1183 // Cleanup volumes
1184 if containerDomain != "" && !cfg.VolumePreserve {
1185 if err := s.cleanupVolumesForDomain(ctx, containerDomain); err != nil {
1186 log.WrapErrWithFields(err, "failed to cleanup volumes", map[string]any{"domain": containerDomain})
1187 }
1188 }
1189
1190 // Clean up attachment containers
1191 s.removeAttachments(ctx, attachmentIDs)
1192
1193 // Remove from tracking
1194 s.mu.Lock()
1195 var removedDomain string
1196 for d, c := range s.containers {
1197 if c.ID == containerID {
1198 delete(s.containers, d)
1199 removedDomain = d
1200 s.managedCount--
1201 log.Info().Str("domain", d).Msg("container removed")
1202 break
1203 }
1204 }
1205 if removedDomain != "" {
1206 delete(s.attachments, removedDomain)
1207 }
1208 s.mu.Unlock()
1209
1210 // Decrement managed container count
1211 if removedDomain != "" && s.metrics != nil {
1212 s.metrics.ManagedContainers.Add(ctx, -1)
1213 }
1214
1215 // Note: We intentionally do NOT clean up deployMu entries to avoid a race condition:
1216 // If Remove deletes the mutex entry while a concurrent Deploy is acquiring the lock,
1217 // the Deploy would create a fresh mutex, breaking serialization and allowing concurrent
1218 // deploys to the same domain. The memory footprint is acceptable (one small struct per
1219 // domain ever deployed), and this is the safer choice for correctness.
1220
1221 // Cleanup network
1222 if removedDomain != "" && cfg.NetworkIsolation {
1223 networkName := s.getNetworkForApp(removedDomain)
1224 if err := s.cleanupNetworkIfEmpty(ctx, networkName); err != nil {
1225 log.WrapErrWithFields(err, "failed to cleanup network", map[string]any{"network": networkName})
1226 }
1227 }
1228
1229 return nil
1230}
1231
1232// findContainerContext looks up the domain, attachment IDs, and config snapshot for a container.
1233func (s *Service) findContainerContext(containerID string) (string, []string, Config) {
1234 s.mu.RLock()
1235 defer s.mu.RUnlock()
1236 cfg := s.config
1237 var containerDomain string
1238 for d, c := range s.containers {
1239 if c.ID == containerID {
1240 containerDomain = d
1241 break
1242 }
1243 }
1244 var attachmentIDs []string
1245 if containerDomain != "" {
1246 attachmentIDs = append(attachmentIDs, s.attachments[containerDomain]...)
1247 }
1248 return containerDomain, attachmentIDs, cfg
1249}
1250
1251// removeAttachments stops and removes attachment containers best-effort.
1252func (s *Service) removeAttachments(ctx context.Context, attachmentIDs []string) {
1253 log := zerowrap.FromCtx(ctx)
1254 for _, attachID := range attachmentIDs {
1255 if s.logWriter != nil {
1256 if err := s.logWriter.StopLogging(attachID); err != nil {
1257 log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to stop attachment log collection")
1258 }
1259 }
1260 if err := s.runtime.StopContainer(ctx, attachID); err != nil {
1261 log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to stop attachment container")
1262 }
1263 if err := s.runtime.RemoveContainer(ctx, attachID, true); err != nil {
1264 log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to remove attachment container")
1265 }
1266 }
1267}
1268
1269// Get retrieves a container by domain.
1270func (s *Service) Get(_ context.Context, domainName string) (*domain.Container, bool) {
1271 s.mu.RLock()
1272 defer s.mu.RUnlock()
1273 container, exists := s.containers[domainName]
1274 return container, exists
1275}
1276
1277// List returns all managed containers.
1278func (s *Service) List(_ context.Context) map[string]*domain.Container {
1279 s.mu.RLock()
1280 defer s.mu.RUnlock()
1281
1282 result := make(map[string]*domain.Container, len(s.containers))
1283 maps.Copy(result, s.containers)
1284 return result
1285}
1286
1287// ListRoutesWithDetails returns routes with network and attachment info.
1288// Note: Container map is copied under lock, then external calls are made without lock.
1289// If containers are removed between copy and runtime calls, errors are handled gracefully
1290// (network becomes empty string). This trade-off avoids holding locks during I/O.
1291func (s *Service) ListRoutesWithDetails(ctx context.Context) []domain.RouteInfo {
1292 s.mu.RLock()
1293 containers := make(map[string]*domain.Container, len(s.containers))
1294 maps.Copy(containers, s.containers)
1295 s.mu.RUnlock()
1296
1297 // Fetch all attachments once to avoid N+1 queries
1298 attachmentsByDomain := s.getAllAttachments(ctx)
1299
1300 results := make([]domain.RouteInfo, 0, len(containers))
1301 for domainName, container := range containers {
1302 network := ""
1303 image := ""
1304 containerID := ""
1305 status := ""
1306 if container != nil {
1307 containerID = container.ID
1308 status = container.Status
1309 image = container.Image
1310 if container.Labels != nil {
1311 if labelImage, ok := container.Labels[domain.LabelImage]; ok && labelImage != "" {
1312 image = labelImage
1313 }
1314 }
1315 // Strip registry domain prefix from image for cleaner display
1316 image = s.stripRegistryPrefix(image)
1317 if networkName, err := s.runtime.GetContainerNetwork(ctx, container.ID); err == nil {
1318 network = networkName
1319 }
1320 }
1321
1322 results = append(results, domain.RouteInfo{
1323 Domain: domainName,
1324 Image: image,
1325 ContainerID: containerID,
1326 ContainerStatus: status,
1327 Network: network,
1328 Attachments: attachmentsByDomain[domainName],
1329 })
1330 }
1331
1332 return results
1333}
1334
1335// stripRegistryPrefix removes the configured registry domain prefix from an image reference.
1336// For example, "reg.example.com/myapp:latest" becomes "myapp:latest" if registry domain is "reg.example.com".
1337func (s *Service) stripRegistryPrefix(image string) string {
1338 s.mu.RLock()
1339 cfg := s.config
1340 s.mu.RUnlock()
1341
1342 if cfg.RegistryDomain == "" {
1343 return image
1344 }
1345 prefix := strings.TrimSuffix(cfg.RegistryDomain, "/") + "/"
1346 if strings.HasPrefix(image, prefix) {
1347 return strings.TrimPrefix(image, prefix)
1348 }
1349 return image
1350}
1351
1352// ListAttachments returns attachments for a domain.
1353func (s *Service) ListAttachments(ctx context.Context, domainName string) []domain.Attachment {
1354 return s.getAttachmentsForDomain(ctx, domainName)
1355}
1356
1357// ListNetworks returns Gordon-managed networks.
1358func (s *Service) ListNetworks(ctx context.Context) ([]*domain.NetworkInfo, error) {
1359 s.mu.RLock()
1360 cfg := s.config
1361 s.mu.RUnlock()
1362
1363 networks, err := s.runtime.ListNetworks(ctx)
1364 if err != nil {
1365 return nil, err
1366 }
1367
1368 var filtered []*domain.NetworkInfo
1369 for _, network := range networks {
1370 if strings.HasPrefix(network.Name, cfg.NetworkPrefix+"-") && network.Labels[domain.LabelManaged] == "true" {
1371 filtered = append(filtered, network)
1372 }
1373 }
1374
1375 return filtered, nil
1376}
1377
1378// HealthCheck performs health checks on all containers.
1379func (s *Service) HealthCheck(ctx context.Context) map[string]bool {
1380 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1381 zerowrap.FieldLayer: "usecase",
1382 zerowrap.FieldUseCase: "HealthCheck",
1383 })
1384 log := zerowrap.FromCtx(ctx)
1385
1386 // Copy container IDs under lock, then release before making Docker API calls
1387 s.mu.RLock()
1388 snapshot := make(map[string]string, len(s.containers))
1389 for d, c := range s.containers {
1390 snapshot[d] = c.ID
1391 }
1392 s.mu.RUnlock()
1393
1394 health := make(map[string]bool, len(snapshot))
1395 for d, id := range snapshot {
1396 running, err := s.runtime.IsContainerRunning(ctx, id)
1397 if err != nil {
1398 log.WrapErrWithFields(err, "health check failed", map[string]any{"domain": d, zerowrap.FieldEntityID: id})
1399 health[d] = false
1400 } else {
1401 health[d] = running
1402 }
1403 }
1404 return health
1405}
1406
1407// SyncContainers synchronizes containers with runtime state.
1408func (s *Service) SyncContainers(ctx context.Context) error {
1409 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1410 zerowrap.FieldLayer: "usecase",
1411 zerowrap.FieldUseCase: "SyncContainers",
1412 })
1413 log := zerowrap.FromCtx(ctx)
1414
1415 // List containers without holding lock to avoid blocking during Docker API call
1416 allContainers, err := s.runtime.ListContainers(ctx, false)
1417 if err != nil {
1418 return log.WrapErr(err, "failed to list containers")
1419 }
1420
1421 managed := make(map[string]*domain.Container)
1422 attachments := make(map[string][]string)
1423 for _, c := range allContainers {
1424 if c.Labels == nil || c.Labels[domain.LabelManaged] != "true" {
1425 continue
1426 }
1427 if c.Labels[domain.LabelAttachment] == "true" {
1428 owner := c.Labels[domain.LabelAttachedTo]
1429 if owner != "" {
1430 attachments[owner] = append(attachments[owner], c.ID)
1431 }
1432 continue
1433 }
1434 if d, ok := c.Labels[domain.LabelDomain]; ok {
1435 managed[d] = c
1436 }
1437 }
1438
1439 s.mu.Lock()
1440 s.containers = managed
1441 s.attachments = attachments
1442 newCount := int64(len(managed))
1443 delta := newCount - s.managedCount
1444 s.managedCount = newCount
1445 s.mu.Unlock()
1446
1447 // Report initial/delta managed container count to OTel UpDownCounter.
1448 if delta != 0 && s.metrics != nil {
1449 s.metrics.ManagedContainers.Add(ctx, delta)
1450 }
1451
1452 log.Info().Int(zerowrap.FieldCount, len(managed)).Msg("container state synchronized")
1453 return nil
1454}
1455
1456// AutoStart starts containers for the provided routes that aren't running.
1457// It skips readiness checks to avoid blocking boot; the background monitor
1458// handles crash recovery.
1459func (s *Service) AutoStart(ctx context.Context, routes []domain.Route) error {
1460 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1461 zerowrap.FieldLayer: "usecase",
1462 zerowrap.FieldUseCase: "AutoStart",
1463 })
1464 log := zerowrap.FromCtx(ctx)
1465
1466 log.Info().Int("route_count", len(routes)).Msg("auto-starting containers for configured routes")
1467
1468 // Filter out routes that already have a running container.
1469 var pending []domain.Route
1470 skipped := 0
1471 for _, route := range routes {
1472 if _, exists := s.Get(ctx, route.Domain); exists {
1473 log.Debug().Str("domain", route.Domain).Msg("container already running, skipping")
1474 skipped++
1475 } else {
1476 pending = append(pending, route)
1477 }
1478 }
1479
1480 if len(pending) == 0 {
1481 log.Info().Int("skipped", skipped).Msg("auto-start completed, all containers already running")
1482 return nil
1483 }
1484
1485 // Deploy all pending routes concurrently with readiness checks skipped.
1486 deployCtx := domain.WithSkipReadiness(ctx)
1487
1488 type result struct {
1489 route domain.Route
1490 err error
1491 }
1492 results := make(chan result, len(pending))
1493 sem := make(chan struct{}, 4) // limit concurrency
1494
1495 for _, route := range pending {
1496 sem <- struct{}{}
1497 go func(r domain.Route) {
1498 defer func() { <-sem }()
1499
1500 log.Info().Str("domain", r.Domain).Str("image", r.Image).Msg("auto-starting container for route")
1501
1502 if _, err := s.Deploy(deployCtx, r); err != nil {
1503 results <- result{route: r, err: err}
1504 return
1505 }
1506 results <- result{route: r}
1507 }(route)
1508 }
1509
1510 var started, errCount int
1511 for i := 0; i < len(pending); i++ {
1512 res := <-results
1513 if res.err != nil {
1514 log.Warn().Err(res.err).Str("domain", res.route.Domain).Msg("failed to auto-start container")
1515 errCount++
1516 } else {
1517 started++
1518 }
1519 }
1520
1521 log.Info().
1522 Int("started", started).
1523 Int("skipped", skipped).
1524 Int("errors", errCount).
1525 Msg("auto-start completed")
1526
1527 if errCount > 0 {
1528 return fmt.Errorf("auto-start completed with %d errors", errCount)
1529 }
1530 return nil
1531}
1532
1533// Shutdown gracefully shuts down all managed containers.
1534func (s *Service) Shutdown(ctx context.Context) error {
1535 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1536 zerowrap.FieldLayer: "usecase",
1537 zerowrap.FieldUseCase: "Shutdown",
1538 })
1539 log := zerowrap.FromCtx(ctx)
1540 log.Info().Msg("shutting down container manager...")
1541
1542 // Wait for any in-flight background cleanup goroutines to finish
1543 s.cleanupWg.Wait()
1544
1545 // Containers are left running across Gordon restarts.
1546 // SyncContainers + AutoStart will pick them back up on next boot.
1547
1548 // Close log writer to stop all log collection
1549 if s.logWriter != nil {
1550 if err := s.logWriter.Close(); err != nil {
1551 log.Warn().Err(err).Msg("failed to close container log writer")
1552 }
1553 }
1554
1555 log.Info().Msg("container manager shutdown complete")
1556 return nil
1557}
1558
1559// WaitForCleanup blocks until all background cleanup goroutines complete.
1560// Intended for use in tests to avoid mock assertion races.
1561func (s *Service) WaitForCleanup() {
1562 s.cleanupWg.Wait()
1563}
1564
1565// StartMonitor begins background monitoring of tracked containers.
1566// The monitor restarts crashed containers and detects crash loops.
1567// Safe to call multiple times; subsequent calls are no-ops.
1568func (s *Service) StartMonitor(ctx context.Context) {
1569 s.mu.Lock()
1570 if s.monitor != nil {
1571 s.mu.Unlock()
1572 return
1573 }
1574 s.monitor = newMonitor(s)
1575 m := s.monitor
1576 m.Start(ctx)
1577 s.mu.Unlock()
1578}
1579
1580// StopMonitor stops the background container monitor.
1581// Safe to call multiple times; subsequent calls are no-ops.
1582func (s *Service) StopMonitor() {
1583 s.mu.Lock()
1584 m := s.monitor
1585 s.monitor = nil
1586 s.mu.Unlock()
1587 if m != nil {
1588 m.Stop()
1589 }
1590}
1591
1592// UpdateConfig updates the service configuration.
1593func (s *Service) UpdateConfig(config Config) {
1594 s.mu.Lock()
1595 s.config = config
1596 s.mu.Unlock()
1597}
1598
1599// UpdateAttachments updates only the attachment configuration in the service.
1600// This is called after a config reload to propagate attachment changes without restart.
1601// The incoming map is deep-copied so external callers cannot mutate service state.
1602func (s *Service) UpdateAttachments(attachments map[string][]string) {
1603 var copied map[string][]string
1604 if attachments != nil {
1605 copied = make(map[string][]string, len(attachments))
1606 for k, v := range attachments {
1607 sl := make([]string, len(v))
1608 copy(sl, v)
1609 copied[k] = sl
1610 }
1611 }
1612 s.mu.Lock()
1613 s.config.Attachments = copied
1614 s.mu.Unlock()
1615}
1616
1617// Helper methods
1618
1619// cleanupFailedContainer stops and removes a container that failed to start properly.
1620func (s *Service) cleanupFailedContainer(ctx context.Context, containerID string) {
1621 log := zerowrap.FromCtx(ctx)
1622 if err := s.runtime.StopContainer(ctx, containerID); err != nil {
1623 log.Warn().Err(err).Str(zerowrap.FieldEntityID, containerID).Msg("failed to stop container after failure")
1624 }
1625 if err := s.runtime.RemoveContainer(ctx, containerID, true); err != nil {
1626 log.Warn().Err(err).Str(zerowrap.FieldEntityID, containerID).Msg("failed to remove container after failure")
1627 }
1628}
1629
1630// cleanupOldContainer stops and removes an old container after zero-downtime switch.
1631// It also renames the new container to the canonical name.
1632func (s *Service) cleanupOldContainer(ctx context.Context, old *domain.Container, newContainerID, domainName string) {
1633 log := zerowrap.FromCtx(ctx)
1634 log.Info().Str(zerowrap.FieldEntityID, old.ID).Msg("stopping old container after zero-downtime switch")
1635
1636 if s.logWriter != nil {
1637 if err := s.logWriter.StopLogging(old.ID); err != nil {
1638 log.Warn().Err(err).Str(zerowrap.FieldEntityID, old.ID).Msg("failed to stop logging for old container")
1639 }
1640 }
1641
1642 if err := s.runtime.StopContainer(ctx, old.ID); err != nil {
1643 log.Warn().Err(err).Str(zerowrap.FieldEntityID, old.ID).Msg("failed to stop old container")
1644 }
1645
1646 if err := s.runtime.RemoveContainer(ctx, old.ID, true); err != nil {
1647 log.Warn().Err(err).Str(zerowrap.FieldEntityID, old.ID).Msg("failed to remove old container")
1648 }
1649
1650 // Rename new container to canonical name
1651 canonicalName := fmt.Sprintf("gordon-%s", domainName)
1652 if err := s.runtime.RenameContainer(ctx, newContainerID, canonicalName); err != nil {
1653 log.Warn().Err(err).Str("canonical_name", canonicalName).Msg("failed to rename container to canonical name")
1654 }
1655}
1656
1657// startLogCollection starts log collection for a container in the background.
1658// Errors are logged but don't fail the calling operation.
1659func (s *Service) startLogCollection(ctx context.Context, containerID, domainName string) {
1660 if s.logWriter == nil {
1661 return
1662 }
1663
1664 log := zerowrap.FromCtx(ctx)
1665
1666 // Use context.WithoutCancel so the log stream outlives the HTTP request.
1667 // Without this, the Docker log stream closes when the deploy response completes.
1668 bgCtx := context.WithoutCancel(ctx)
1669
1670 logStream, err := s.runtime.GetContainerLogs(bgCtx, containerID, true)
1671 if err != nil {
1672 log.Warn().Err(err).Msg("failed to get container logs for collection")
1673 return
1674 }
1675
1676 if err := s.logWriter.StartLogging(bgCtx, containerID, domainName, logStream); err != nil {
1677 log.Warn().Err(err).Msg("failed to start container log collection")
1678 logStream.Close()
1679 }
1680}
1681
1682func (s *Service) buildValidatedImageRef(ctx context.Context, image string) (string, error) {
1683 imageRef := s.buildImageRef(image)
1684 if err := s.validateImageRef(ctx, imageRef); err != nil {
1685 return "", err
1686 }
1687 return imageRef, nil
1688}
1689
1690func (s *Service) buildImageRef(image string) string {
1691 s.mu.RLock()
1692 cfg := s.config
1693 s.mu.RUnlock()
1694
1695 if !cfg.RegistryAuthEnabled || cfg.RegistryDomain == "" {
1696 return image
1697 }
1698
1699 // Normalize registry domain by trimming any trailing slash
1700 reg := strings.TrimSuffix(cfg.RegistryDomain, "/")
1701
1702 // Check if image already has the registry domain prefix
1703 prefix := reg + "/"
1704 if strings.HasPrefix(image, prefix) {
1705 return image
1706 }
1707
1708 // Check if image already has an explicit registry (host:port/ or host.domain/).
1709 // We need to detect patterns like:
1710 // - "docker.io/library/nginx:latest" (has dot and slash)
1711 // - "localhost:5001/myapp:latest" (has host:port/ pattern)
1712 // - "myregistry:5000/app:v1" (has host:port/ pattern)
1713 // - "[fd00::1]:5000/app:v1" (IPv6 with port)
1714 if hasExplicitRegistry(image) {
1715 return image
1716 }
1717
1718 return fmt.Sprintf("%s/%s", reg, image)
1719}
1720
1721// hasExplicitRegistry checks if an image reference already includes an explicit registry.
1722// Returns true for patterns like "docker.io/image", "localhost:5000/image", "localhost/image",
1723// "registry:8080/image", "[fd00::1]/image", "[fd00::1]:5000/image".
1724func hasExplicitRegistry(image string) bool {
1725 // Find the first slash which separates registry from image name
1726 slashIdx := strings.Index(image, "/")
1727 if slashIdx == -1 {
1728 return false // No slash means no registry prefix (e.g., "myapp:latest")
1729 }
1730
1731 // Extract the part before the first slash (potential registry)
1732 registryPart := image[:slashIdx]
1733
1734 // Check for bracketed IPv6 address (e.g., "[fd00::1]" or "[fd00::1]:5000")
1735 if strings.HasPrefix(registryPart, "[") {
1736 // Look for closing bracket
1737 if closeBracket := strings.Index(registryPart, "]"); closeBracket != -1 {
1738 // Valid bracketed IPv6: either ends at ] or has ]:port
1739 return true
1740 }
1741 }
1742
1743 // Check for localhost (with or without port)
1744 if registryPart == "localhost" {
1745 return true
1746 }
1747
1748 // Check for host:port pattern (e.g., "localhost:5000", "registry:8080")
1749 if colonIdx := strings.LastIndex(registryPart, ":"); colonIdx != -1 {
1750 port := registryPart[colonIdx+1:]
1751 // If everything after colon is digits, it's a port
1752 if len(port) > 0 && isNumeric(port) {
1753 return true
1754 }
1755 }
1756
1757 // Check for domain pattern (has a dot, e.g., "docker.io", "gcr.io")
1758 if strings.Contains(registryPart, ".") {
1759 return true
1760 }
1761
1762 return false
1763}
1764
1765func (s *Service) validateImageRef(ctx context.Context, imageRef string) error {
1766 if !hasExplicitRegistry(imageRef) {
1767 return nil
1768 }
1769 return s.validateExternalImageRef(ctx, imageRef, imageRegistry(imageRef))
1770}
1771
1772func (s *Service) validateImagePullRef(ctx context.Context, imageRef string) error {
1773 return s.validateExternalImageRef(ctx, imageRef, imageRegistryForPolicy(imageRef))
1774}
1775
1776func (s *Service) validateExternalImageRef(ctx context.Context, imageRef, registry string) error {
1777 s.mu.RLock()
1778 cfg := s.config
1779 s.mu.RUnlock()
1780
1781 if registry == "" {
1782 return fmt.Errorf("image %q rejected: invalid image reference", imageRef)
1783 }
1784
1785 if sameRegistry(registry, cfg.RegistryDomain) {
1786 return nil
1787 }
1788 dangerous, err := isDangerousRegistryHost(ctx, imageRegistryHost(registry))
1789 if err != nil {
1790 return fmt.Errorf("image %q rejected: registry %q could not be verified: %w", imageRef, registry, err)
1791 }
1792 if dangerous {
1793 return fmt.Errorf("image %q rejected: registry %q is not allowed", imageRef, registry)
1794 }
1795 if len(cfg.AllowedRegistries) == 0 {
1796 return fmt.Errorf("image %q rejected: registry %q is not in images.allowed_registries", imageRef, registry)
1797 }
1798 for _, allowed := range cfg.AllowedRegistries {
1799 if sameRegistry(registry, allowed) {
1800 if cfg.RequireImageDigest && !isValidSha256DigestRef(imageRef) {
1801 return fmt.Errorf("image %q rejected: digest reference required", imageRef)
1802 }
1803 return nil
1804 }
1805 }
1806 return fmt.Errorf("image %q rejected: registry %q is not in images.allowed_registries", imageRef, registry)
1807}
1808
1809func imageRegistry(imageRef string) string {
1810 slashIdx := strings.Index(imageRef, "/")
1811 if slashIdx == -1 {
1812 return ""
1813 }
1814 return normalizeRegistry(imageRef[:slashIdx])
1815}
1816
1817func imageRegistryForPolicy(imageRef string) string {
1818 if hasExplicitRegistry(imageRef) {
1819 return imageRegistry(imageRef)
1820 }
1821 return "docker.io"
1822}
1823
1824// normalizeRegistry normalizes a registry string for comparison purposes only.
1825// This function is used by sameRegistry to determine if two registry strings
1826// refer to the same registry. The output intentionally strips IPv6 brackets
1827// (e.g., "[fd00::1]:5000" -> "fd00::1:5000") which is acceptable for equality
1828// checks but the result MUST NOT be used to construct or emit valid registry URLs.
1829func normalizeRegistry(registry string) string {
1830 registry = strings.ToLower(strings.TrimSuffix(strings.TrimSpace(registry), "."))
1831 registry = strings.TrimSuffix(registry, "/")
1832 if strings.HasPrefix(registry, "[") {
1833 if end := strings.Index(registry, "]"); end != -1 {
1834 host := strings.Trim(registry[:end+1], "[]")
1835 port := registry[end+1:]
1836 return host + port
1837 }
1838 }
1839 return registry
1840}
1841
1842func imageRegistryHost(registry string) string {
1843 if ip := net.ParseIP(registry); ip != nil {
1844 return registry
1845 }
1846 if h, _, err := net.SplitHostPort(registry); err == nil {
1847 return strings.Trim(h, "[]")
1848 }
1849 if strings.HasPrefix(registry, "[") && strings.Contains(registry, "]") {
1850 return strings.TrimPrefix(strings.Split(registry, "]")[0], "[")
1851 }
1852 if colon := strings.LastIndex(registry, ":"); colon != -1 && isNumeric(registry[colon+1:]) {
1853 return registry[:colon]
1854 }
1855 return registry
1856}
1857
1858func sameRegistry(a, b string) bool {
1859 return normalizeRegistry(a) == normalizeRegistry(b)
1860}
1861
1862func isDangerousRegistryHost(ctx context.Context, host string) (bool, error) {
1863 host = strings.ToLower(strings.TrimSuffix(host, "."))
1864 if host == "localhost" || host == "metadata.google.internal" {
1865 return true, nil
1866 }
1867 if ip := net.ParseIP(host); ip != nil {
1868 return isDangerousRegistryIP(ip), nil
1869 }
1870 addrs, err := net.DefaultResolver.LookupIPAddr(ctx, host)
1871 if err != nil {
1872 return false, err
1873 }
1874 for _, addr := range addrs {
1875 if isDangerousRegistryIP(addr.IP) {
1876 return true, nil
1877 }
1878 }
1879 return false, nil
1880}
1881
1882func isDangerousRegistryIP(ip net.IP) bool {
1883 return ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsUnspecified()
1884}
1885
1886func applySecurityProfile(config *domain.ContainerConfig, cfg Config) {
1887 if strings.EqualFold(cfg.SecurityProfile, "strict") {
1888 config.ReadOnlyRootFS = true
1889 config.CapDrop = []string{"ALL"}
1890 config.CapAdd = []string{"NET_BIND_SERVICE"}
1891 }
1892}
1893
1894// isNumeric checks if a string contains only digits.
1895func isNumeric(s string) bool {
1896 for _, c := range s {
1897 if c < '0' || c > '9' {
1898 return false
1899 }
1900 }
1901 return true
1902}
1903
1904func normalizePullPolicy(policy string) string {
1905 switch strings.ToLower(strings.TrimSpace(policy)) {
1906 case PullPolicyAlways:
1907 return PullPolicyAlways
1908 case PullPolicyIfTagChanged:
1909 return PullPolicyIfTagChanged
1910 case PullPolicyIfNotPresent:
1911 return PullPolicyIfNotPresent
1912 default:
1913 return PullPolicyIfNotPresent
1914 }
1915}
1916
1917func isDigestRef(imageRef string) bool {
1918 return isValidSha256DigestRef(imageRef)
1919}
1920
1921func isValidSha256DigestRef(imageRef string) bool {
1922 _, digest, ok := strings.Cut(imageRef, "@sha256:")
1923 if !ok || len(digest) != 64 {
1924 return false
1925 }
1926 _, err := hex.DecodeString(digest)
1927 return err == nil
1928}
1929
1930func (s *Service) pullRefForDeploy(ctx context.Context, imageRef string) (string, bool) {
1931 if !domain.IsInternalDeploy(ctx) {
1932 return imageRef, false
1933 }
1934 s.mu.RLock()
1935 cfg := s.config
1936 s.mu.RUnlock()
1937 return rewriteToLocalRegistry(imageRef, cfg.RegistryDomain, cfg.RegistryPort), true
1938}
1939
1940// ensureImage ensures the image is available locally, pulling if needed.
1941// Returns the image reference to use for container operations:
1942// - For digest references (@sha256:...), returns the pullRef since Docker can't tag digests
1943// - For tagged images, returns the original imageRef after tagging the pulled image
1944func (s *Service) ensureImage(ctx context.Context, imageRef string) (string, error) {
1945 ctx, span := tracer.Start(ctx, "container.ensure_image",
1946 trace.WithAttributes(attribute.String("image", imageRef)))
1947 defer span.End()
1948
1949 ctx = zerowrap.CtxWithField(ctx, "image", imageRef)
1950 log := zerowrap.FromCtx(ctx)
1951
1952 // Determine if this is an internal deploy and what reference to use for pulls.
1953 pullRef, isInternal := s.pullRefForDeploy(ctx, imageRef)
1954 if err := s.validateImagePullRef(ctx, imageRef); err != nil {
1955 return "", err
1956 }
1957 if pullRef != imageRef {
1958 log.Info().
1959 Str("original_ref", imageRef).
1960 Str("pull_ref", pullRef).
1961 Msg("internal deploy: using localhost registry for pull")
1962 }
1963
1964 found, err := s.ensureLocalImage(ctx, imageRef, pullRef)
1965 if err != nil {
1966 return "", err
1967 }
1968 if found {
1969 return imageRef, nil
1970 }
1971
1972 // Pull image
1973 log.Info().Msg("pulling image from registry")
1974
1975 if err := s.pullImage(ctx, pullRef, isInternal); err != nil {
1976 return "", err
1977 }
1978
1979 // For digest references, we can't create a tag (Docker doesn't allow it).
1980 // In this case, use the pullRef directly since the image is available under that reference.
1981 if strings.Contains(imageRef, "@sha256:") {
1982 log.Info().
1983 Str("pull_ref", pullRef).
1984 Msg("digest reference: using pull reference for container operations")
1985 return pullRef, nil
1986 }
1987
1988 if err := s.tagImageIfNeeded(ctx, pullRef, imageRef); err != nil {
1989 return "", err
1990 }
1991
1992 // Clean up the temporary pull reference tag to avoid duplicate entries
1993 if pullRef != imageRef {
1994 if err := s.runtime.UntagImage(ctx, pullRef); err != nil {
1995 // Log but don't fail - the canonical tag is already applied
1996 log.Debug().Err(err).Str("pull_ref", pullRef).Msg("failed to remove temporary pull tag")
1997 }
1998 }
1999
2000 log.Info().Msg("image pulled successfully")
2001 return imageRef, nil
2002}
2003
2004func (s *Service) ensureLocalImage(ctx context.Context, imageRef, pullRef string) (bool, error) {
2005 log := zerowrap.FromCtx(ctx)
2006
2007 // For internal deploys (image push events), always pull fresh image
2008 // because the same tag may reference new content (e.g., latest tag updated)
2009 if domain.IsInternalDeploy(ctx) {
2010 log.Info().Msg("internal deploy detected, forcing image pull to ensure latest content")
2011 return false, nil
2012 }
2013
2014 s.mu.RLock()
2015 cfg := s.config
2016 s.mu.RUnlock()
2017
2018 pullPolicy := normalizePullPolicy(cfg.PullPolicy)
2019 switch pullPolicy {
2020 case PullPolicyAlways:
2021 log.Info().Str("pull_policy", pullPolicy).Msg("pull policy forces image pull")
2022 return false, nil
2023 case PullPolicyIfTagChanged:
2024 if !isDigestRef(imageRef) {
2025 log.Info().Str("pull_policy", pullPolicy).Msg("tag reference detected, pulling to check for updates")
2026 return false, nil
2027 }
2028 }
2029
2030 localImages, err := s.runtime.ListImages(ctx)
2031 if err != nil {
2032 log.WrapErr(err, "failed to list local images, will attempt pull")
2033 return false, nil
2034 }
2035
2036 normalizedRef := normalizeImageRef(imageRef)
2037 normalizedPullRef := normalizeImageRef(pullRef)
2038 for _, img := range localImages {
2039 normalizedImage := normalizeImageRef(img)
2040 if normalizedImage == normalizedRef {
2041 log.Info().Msg("image found locally, skipping pull")
2042 return true, nil
2043 }
2044 if normalizedImage == normalizedPullRef {
2045 if err := s.tagImageIfNeeded(ctx, pullRef, imageRef); err != nil {
2046 return false, err
2047 }
2048 // Clean up the temporary pull reference tag
2049 if pullRef != imageRef {
2050 if err := s.runtime.UntagImage(ctx, pullRef); err != nil {
2051 log.Debug().Err(err).Str("pull_ref", pullRef).Msg("failed to remove temporary pull tag")
2052 }
2053 }
2054 log.Info().Msg("image found locally, skipping pull")
2055 return true, nil
2056 }
2057 }
2058
2059 return false, nil
2060}
2061
2062func (s *Service) pullImage(ctx context.Context, pullRef string, isInternal bool) error {
2063 s.mu.RLock()
2064 cfg := s.config
2065 s.mu.RUnlock()
2066
2067 log := zerowrap.FromCtx(ctx)
2068
2069 switch {
2070 case isInternal && cfg.RegistryAuthEnabled:
2071 if cfg.InternalRegistryUsername == "" || cfg.InternalRegistryPassword == "" {
2072 return log.WrapErr(fmt.Errorf("internal registry auth not configured"), "failed to pull image for internal deploy")
2073 }
2074 if err := s.pullImageWithRetry(ctx, func(ctx context.Context) error {
2075 return s.runtime.PullImageWithAuth(ctx, pullRef, cfg.InternalRegistryUsername, cfg.InternalRegistryPassword)
2076 }, isInternal); err != nil {
2077 return log.WrapErr(fmt.Errorf("%w: %w", domain.ErrImagePullFailed, err), "failed to pull image with internal auth")
2078 }
2079 case isInternal:
2080 if err := s.pullImageWithRetry(ctx, func(ctx context.Context) error {
2081 return s.runtime.PullImage(ctx, pullRef)
2082 }, isInternal); err != nil {
2083 return log.WrapErr(fmt.Errorf("%w: %w", domain.ErrImagePullFailed, err), "failed to pull image")
2084 }
2085 case cfg.RegistryAuthEnabled:
2086 if cfg.ServiceTokenUsername == "" || cfg.ServiceToken == "" {
2087 return log.WrapErr(fmt.Errorf("registry service token not configured"), "failed to pull image for registry auth")
2088 }
2089 if err := s.runtime.PullImageWithAuth(ctx, pullRef, cfg.ServiceTokenUsername, cfg.ServiceToken); err != nil {
2090 return log.WrapErr(fmt.Errorf("%w: %w", domain.ErrImagePullFailed, err), "failed to pull image with auth")
2091 }
2092 default:
2093 if err := s.runtime.PullImage(ctx, pullRef); err != nil {
2094 return log.WrapErr(fmt.Errorf("%w: %w", domain.ErrImagePullFailed, err), "failed to pull image")
2095 }
2096 }
2097
2098 return nil
2099}
2100
2101func (s *Service) pullImageWithRetry(ctx context.Context, pullFn func(context.Context) error, isInternal bool) error {
2102 log := zerowrap.FromCtx(ctx)
2103
2104 for attempt := 1; attempt <= internalPullMaxAttempts; attempt++ {
2105 err := pullFn(ctx)
2106 if err == nil {
2107 return nil
2108 }
2109 if !isInternal || !isConnectionRefusedError(err) || attempt == internalPullMaxAttempts {
2110 return fmt.Errorf("internal image pull failed after %d attempts: %w", attempt, err)
2111 }
2112
2113 backoff := time.Duration(attempt) * time.Second
2114 log.Warn().
2115 Err(err).
2116 Int("attempt", attempt).
2117 Dur("backoff", backoff).
2118 Msg("internal image pull failed with connection refused, retrying")
2119
2120 select {
2121 case <-time.After(backoff):
2122 case <-ctx.Done():
2123 return ctx.Err()
2124 }
2125 }
2126 return nil
2127}
2128
2129func (s *Service) tagImageIfNeeded(ctx context.Context, sourceRef, targetRef string) error {
2130 if sourceRef == targetRef {
2131 return nil
2132 }
2133
2134 // Cannot create a tag with a digest reference - Docker/Podman doesn't allow it.
2135 // When using digest references (image@sha256:...), skip tagging as the image
2136 // is already available by its digest.
2137 if strings.Contains(targetRef, "@sha256:") {
2138 return nil
2139 }
2140
2141 log := zerowrap.FromCtx(ctx)
2142 if err := s.runtime.TagImage(ctx, sourceRef, targetRef); err != nil {
2143 return log.WrapErr(err, "failed to tag image from pull reference")
2144 }
2145 return nil
2146}
2147
2148func (s *Service) loadEnvironment(ctx context.Context, preResolved []string, domainName, imageRef string) ([]string, error) {
2149 log := zerowrap.FromCtx(ctx)
2150
2151 var userEnvVars []string
2152 if len(preResolved) > 0 {
2153 userEnvVars = preResolved
2154 } else {
2155 var err error
2156 userEnvVars, err = s.envLoader.LoadEnv(ctx, domainName)
2157 if err != nil {
2158 return nil, log.WrapErr(err, "failed to load environment variables")
2159 }
2160 }
2161
2162 dockerfileEnvVars, err := s.runtime.InspectImageEnv(ctx, imageRef)
2163 if err != nil {
2164 log.WrapErr(err, "failed to inspect image environment")
2165 dockerfileEnvVars = []string{}
2166 }
2167
2168 return mergeEnvironmentVariables(dockerfileEnvVars, userEnvVars), nil
2169}
2170
2171func (s *Service) setupVolumes(ctx context.Context, domainName, imageRef string) (map[string]string, error) {
2172 s.mu.RLock()
2173 cfg := s.config
2174 s.mu.RUnlock()
2175
2176 log := zerowrap.FromCtx(ctx)
2177 volumes := make(map[string]string)
2178
2179 if !cfg.VolumeAutoCreate {
2180 return volumes, nil
2181 }
2182
2183 volumePaths, err := s.runtime.InspectImageVolumes(ctx, imageRef)
2184 if err != nil {
2185 log.WrapErr(err, "failed to inspect image volumes")
2186 return volumes, nil
2187 }
2188
2189 for _, path := range volumePaths {
2190 name := generateVolumeName(cfg.VolumePrefix, domainName, path)
2191
2192 exists, err := s.runtime.VolumeExists(ctx, name)
2193 if err != nil {
2194 log.WrapErrWithFields(err, "failed to check volume", map[string]any{"volume": name})
2195 continue
2196 }
2197
2198 if !exists {
2199 if err := s.runtime.CreateVolume(ctx, name); err != nil {
2200 log.WrapErrWithFields(err, "failed to create volume", map[string]any{"volume": name})
2201 continue
2202 }
2203 log.Info().Str("volume", name).Str(zerowrap.FieldPath, path).Msg("created volume")
2204 }
2205
2206 volumes[path] = name
2207 }
2208
2209 return volumes, nil
2210}
2211
2212func (s *Service) getNetworkForApp(domainName string) string {
2213 var networkIsolation bool
2214 var networkGroups map[string][]string
2215
2216 if s.configProvider != nil {
2217 snap := s.configProvider.GetAttachmentConfig()
2218 networkGroups = snap.NetworkGroups
2219 s.mu.RLock()
2220 networkIsolation = s.config.NetworkIsolation
2221 s.mu.RUnlock()
2222 } else {
2223 s.mu.RLock()
2224 networkIsolation = s.config.NetworkIsolation
2225 networkGroups = s.config.NetworkGroups
2226 s.mu.RUnlock()
2227 }
2228
2229 if !networkIsolation {
2230 return "bridge"
2231 }
2232
2233 for groupName, domains := range networkGroups {
2234 if slices.Contains(domains, domainName) {
2235 return s.generateNetworkName(groupName)
2236 }
2237 }
2238
2239 return s.generateNetworkName(domainName)
2240}
2241
2242func (s *Service) generateNetworkName(identifier string) string {
2243 s.mu.RLock()
2244 cfg := s.config
2245 s.mu.RUnlock()
2246
2247 return fmt.Sprintf("%s-%s", cfg.NetworkPrefix, strings.ReplaceAll(identifier, ".", "-"))
2248}
2249
2250func (s *Service) createNetworkIfNeeded(ctx context.Context, networkName string) error {
2251 if networkName == "bridge" || networkName == "default" {
2252 return nil
2253 }
2254
2255 ctx = zerowrap.CtxWithField(ctx, "network", networkName)
2256 log := zerowrap.FromCtx(ctx)
2257
2258 exists, err := s.runtime.NetworkExists(ctx, networkName)
2259 if err != nil {
2260 return log.WrapErr(err, "failed to check network existence")
2261 }
2262
2263 if !exists {
2264 s.mu.RLock()
2265 internal := s.config.NetworkInternal
2266 s.mu.RUnlock()
2267 if err := s.runtime.CreateNetwork(ctx, networkName, domain.NetworkConfig{Driver: "bridge", Internal: internal}); err != nil {
2268 return log.WrapErr(err, "failed to create network")
2269 }
2270 log.Info().Msg("created network for app isolation")
2271 }
2272
2273 return nil
2274}
2275
2276func (s *Service) cleanupOrphanedContainers(ctx context.Context, domainName string, skipContainerID string) error {
2277 log := zerowrap.FromCtx(ctx)
2278 expectedName := fmt.Sprintf("gordon-%s", domainName)
2279 expectedNewName := expectedName + "-new"
2280 expectedNextName := expectedName + "-next"
2281
2282 allContainers, err := s.runtime.ListContainers(ctx, true)
2283 if err != nil {
2284 return err
2285 }
2286
2287 for _, c := range allContainers {
2288 if (c.Name == expectedName || c.Name == expectedNewName || c.Name == expectedNextName) && c.ID != skipContainerID {
2289 // Skip any running container regardless of name — a running
2290 // temp container may be actively serving traffic while the
2291 // new container stabilizes, or may be from a concurrent deploy.
2292 if c.Status == "running" {
2293 log.Debug().
2294 Str(zerowrap.FieldEntityID, c.ID).
2295 Str("container_name", c.Name).
2296 Str(zerowrap.FieldStatus, c.Status).
2297 Msg("skipping running container during orphan cleanup")
2298 continue
2299 }
2300
2301 log.Info().Str(zerowrap.FieldEntityID, c.ID).Str("container_name", c.Name).Str(zerowrap.FieldStatus, c.Status).Msg("found orphaned container, removing")
2302
2303 if err := s.runtime.StopContainer(ctx, c.ID); err != nil {
2304 log.WrapErrWithFields(err, "failed to stop orphaned container", map[string]any{zerowrap.FieldEntityID: c.ID})
2305 }
2306
2307 if err := s.runtime.RemoveContainer(ctx, c.ID, true); err != nil {
2308 log.WrapErrWithFields(err, "failed to remove orphaned container", map[string]any{zerowrap.FieldEntityID: c.ID})
2309 }
2310 }
2311 }
2312
2313 return nil
2314}
2315
2316func (s *Service) cleanupVolumesForDomain(_ context.Context, _ string) error {
2317 return nil
2318}
2319
2320func (s *Service) cleanupNetworkIfEmpty(ctx context.Context, networkName string) error {
2321 if networkName == "bridge" || networkName == "default" {
2322 return nil
2323 }
2324
2325 ctx = zerowrap.CtxWithField(ctx, "network", networkName)
2326 log := zerowrap.FromCtx(ctx)
2327
2328 networks, err := s.runtime.ListNetworks(ctx)
2329 if err != nil {
2330 return log.WrapErr(err, "failed to list networks")
2331 }
2332
2333 for _, n := range networks {
2334 if n.Name == networkName && len(n.Containers) == 0 {
2335 if err := s.runtime.RemoveNetwork(ctx, networkName); err != nil {
2336 return err
2337 }
2338 log.Info().Msg("cleaned up empty network")
2339 break
2340 }
2341 }
2342
2343 return nil
2344}
2345
2346func (s *Service) deployAttachments(ctx context.Context, domainName, networkName string) error {
2347 // Collect attachments from both domain-specific and network group configs
2348 attachments := s.resolveAttachmentsForDomain(domainName)
2349 if len(attachments) == 0 {
2350 return nil
2351 }
2352
2353 log := zerowrap.FromCtx(ctx)
2354 var deployed []string // track successfully deployed attachment IDs for rollback
2355 for _, svc := range attachments {
2356 if err := s.deployAttachedService(ctx, domainName, svc, networkName); err != nil {
2357 log.WrapErrWithFields(err, "failed to deploy attachment", map[string]any{zerowrap.FieldService: svc, "domain": domainName})
2358
2359 // Rollback: clean up already-deployed attachments
2360 for _, id := range deployed {
2361 if stopErr := s.runtime.StopContainer(ctx, id); stopErr != nil {
2362 log.Warn().Err(stopErr).Str("attachment_id", id).Msg("rollback: failed to stop attachment")
2363 }
2364 if rmErr := s.runtime.RemoveContainer(ctx, id, true); rmErr != nil {
2365 log.Warn().Err(rmErr).Str("attachment_id", id).Msg("rollback: failed to remove attachment")
2366 }
2367 }
2368 // Deregister rolled-back attachments
2369 if len(deployed) > 0 {
2370 s.mu.Lock()
2371 remaining := s.attachments[domainName]
2372 filtered := remaining[:0]
2373 rollbackSet := make(map[string]bool, len(deployed))
2374 for _, id := range deployed {
2375 rollbackSet[id] = true
2376 }
2377 for _, id := range remaining {
2378 if !rollbackSet[id] {
2379 filtered = append(filtered, id)
2380 }
2381 }
2382 s.attachments[domainName] = filtered
2383 s.mu.Unlock()
2384 }
2385
2386 return fmt.Errorf("failed to deploy attachment %q (rolled back %d already-deployed)", svc, len(deployed))
2387 }
2388 // Collect IDs of attachments tracked under this domain after successful deploy
2389 s.mu.RLock()
2390 ids := s.attachments[domainName]
2391 if len(ids) > 0 {
2392 latest := ids[len(ids)-1]
2393 deployed = append(deployed, latest)
2394 }
2395 s.mu.RUnlock()
2396 }
2397 return nil
2398}
2399
2400// resolveAttachmentsForDomain returns attachments for a domain by checking:
2401// 1. Direct domain attachments (attachments[domain])
2402// 2. Network group attachments (attachments[group] where domain is in network_groups[group])
2403func (s *Service) resolveAttachmentsForDomain(domainName string) []string {
2404 var attachments map[string][]string
2405 var networkGroups map[string][]string
2406
2407 if s.configProvider != nil {
2408 snap := s.configProvider.GetAttachmentConfig()
2409 attachments = snap.Attachments
2410 networkGroups = snap.NetworkGroups
2411 } else {
2412 s.mu.RLock()
2413 attachments = s.config.Attachments
2414 networkGroups = s.config.NetworkGroups
2415 s.mu.RUnlock()
2416 }
2417
2418 seen := make(map[string]bool)
2419 var result []string
2420
2421 // First, add domain-specific attachments
2422 if domainAttachments, ok := attachments[domainName]; ok {
2423 for _, img := range domainAttachments {
2424 if !seen[img] {
2425 seen[img] = true
2426 result = append(result, img)
2427 }
2428 }
2429 }
2430
2431 // Then, find which network group this domain belongs to and add group attachments
2432 for groupName, domains := range networkGroups {
2433 if slices.Contains(domains, domainName) {
2434 if groupAttachments, ok := attachments[groupName]; ok {
2435 for _, img := range groupAttachments {
2436 if !seen[img] {
2437 seen[img] = true
2438 result = append(result, img)
2439 }
2440 }
2441 }
2442 break // Domain can only be in one network group
2443 }
2444 }
2445
2446 return result
2447}
2448
2449func (s *Service) getAttachmentsForDomain(ctx context.Context, domainName string) []domain.Attachment {
2450 containers, err := s.runtime.ListContainers(ctx, true)
2451 if err != nil {
2452 return nil
2453 }
2454
2455 return s.filterAttachments(ctx, containers, domainName)
2456}
2457
2458// getAllAttachments fetches all containers once and returns attachments grouped by domain.
2459// This avoids N+1 queries when listing multiple routes.
2460func (s *Service) getAllAttachments(ctx context.Context) map[string][]domain.Attachment {
2461 containers, err := s.runtime.ListContainers(ctx, true)
2462 if err != nil {
2463 return nil
2464 }
2465
2466 result := make(map[string][]domain.Attachment)
2467 for _, container := range containers {
2468 if container.Labels == nil {
2469 continue
2470 }
2471 if container.Labels[domain.LabelAttachment] != "true" {
2472 continue
2473 }
2474 ownerDomain := container.Labels[domain.LabelAttachedTo]
2475 if ownerDomain == "" {
2476 continue
2477 }
2478 image := container.Image
2479 serviceName := container.Name
2480 if labelImage, ok := container.Labels[domain.LabelImage]; ok && labelImage != "" {
2481 image = labelImage
2482 serviceName = extractServiceName(labelImage)
2483 }
2484 // Get the container's network for display in the routes table
2485 network := ""
2486 if networkName, err := s.runtime.GetContainerNetwork(ctx, container.ID); err == nil {
2487 network = networkName
2488 }
2489 attachment := domain.Attachment{
2490 Name: serviceName,
2491 Image: s.stripRegistryPrefix(image),
2492 ContainerID: container.ID,
2493 Status: container.Status,
2494 Network: network,
2495 Ports: append([]int(nil), container.Ports...),
2496 }
2497 result[ownerDomain] = append(result[ownerDomain], attachment)
2498 }
2499
2500 return result
2501}
2502
2503// filterAttachments extracts attachments for a specific domain from a container list.
2504func (s *Service) filterAttachments(ctx context.Context, containers []*domain.Container, domainName string) []domain.Attachment {
2505 attachments := make([]domain.Attachment, 0)
2506 for _, container := range containers {
2507 if container.Labels == nil {
2508 continue
2509 }
2510 if container.Labels[domain.LabelAttachment] != "true" {
2511 continue
2512 }
2513 if container.Labels[domain.LabelAttachedTo] != domainName {
2514 continue
2515 }
2516 image := container.Image
2517 serviceName := container.Name
2518 if labelImage, ok := container.Labels[domain.LabelImage]; ok && labelImage != "" {
2519 image = labelImage
2520 serviceName = extractServiceName(labelImage)
2521 }
2522 // Get the container's network for display in the routes table
2523 network := ""
2524 if networkName, err := s.runtime.GetContainerNetwork(ctx, container.ID); err == nil {
2525 network = networkName
2526 }
2527 attachment := domain.Attachment{
2528 Name: serviceName,
2529 Image: s.stripRegistryPrefix(image),
2530 ContainerID: container.ID,
2531 Status: container.Status,
2532 Network: network,
2533 Ports: append([]int(nil), container.Ports...),
2534 }
2535 attachments = append(attachments, attachment)
2536 }
2537
2538 return attachments
2539}
2540
2541func (s *Service) deployAttachedService(ctx context.Context, ownerDomain, serviceImage, networkName string) error {
2542 log := zerowrap.FromCtx(ctx)
2543
2544 // Parse service name from image (e.g., "my-postgres:latest" → "postgres")
2545 serviceName := extractServiceName(serviceImage)
2546 containerName := fmt.Sprintf("gordon-%s-%s", sanitizeName(ownerDomain), serviceName)
2547
2548 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
2549 "attachment": serviceName,
2550 "container_name": containerName,
2551 "owner_domain": ownerDomain,
2552 })
2553
2554 // Check if already running (idempotent)
2555 // Try new name first, then fall back to legacy name for backwards compatibility
2556 existingContainer, err := s.resolveExistingAttachment(ctx, ownerDomain, serviceName)
2557 if err != nil {
2558 return err
2559 }
2560
2561 if existingContainer != nil {
2562 shouldSkip, err := s.handleRunningAttachment(ctx, existingContainer, containerName, serviceImage)
2563 if err != nil {
2564 return err
2565 }
2566 if shouldSkip {
2567 return nil
2568 }
2569 if existingContainer.Status == string(domain.ContainerStatusRunning) {
2570 existingContainer = nil
2571 }
2572 }
2573
2574 if err := s.removeStoppedAttachment(ctx, existingContainer, containerName); err != nil {
2575 return err
2576 }
2577
2578 log.Info().Str(zerowrap.FieldService, serviceImage).Msg("deploying attached service")
2579
2580 // Ensure image (canonical ref is returned; internal pulls may use the local registry).
2581 imageRef, err := s.buildValidatedImageRef(ctx, serviceImage)
2582 if err != nil {
2583 return err
2584 }
2585 actualImageRef, err := s.ensureImage(ctx, imageRef)
2586 if err != nil {
2587 return err
2588 }
2589
2590 exposedPorts := s.attachmentExposedPorts(ctx, actualImageRef)
2591 volumes, err := s.attachmentVolumes(ctx, containerName, actualImageRef)
2592 if err != nil {
2593 return err
2594 }
2595 envVars, err := s.attachmentEnv(ctx, containerName, actualImageRef)
2596 if err != nil {
2597 return err
2598 }
2599 envHash := hashEnvironment(envVars)
2600
2601 s.mu.RLock()
2602 cfg := s.config
2603 s.mu.RUnlock()
2604
2605 // Create container with attachment labels (use actual ref, track original in labels)
2606 config := &domain.ContainerConfig{
2607 Image: actualImageRef,
2608 Name: containerName,
2609 Hostname: serviceName, // Internal DNS: postgres, redis, etc.
2610 Aliases: []string{serviceName},
2611 Ports: exposedPorts,
2612 Env: envVars,
2613 Volumes: volumes,
2614 NetworkMode: networkName, // Same network as main app
2615 Labels: map[string]string{
2616 domain.LabelManaged: "true",
2617 domain.LabelAttachment: "true",
2618 domain.LabelAttachedTo: ownerDomain,
2619 domain.LabelEnvHash: envHash,
2620 domain.LabelImage: serviceImage,
2621 },
2622 MemoryLimit: cfg.DefaultMemoryLimit,
2623 NanoCPUs: cfg.DefaultNanoCPUs,
2624 PidsLimit: cfg.DefaultPidsLimit,
2625 }
2626 applySecurityProfile(config, cfg)
2627
2628 container, err := s.runtime.CreateContainer(ctx, config)
2629 if err != nil {
2630 return log.WrapErr(err, "failed to create attachment container")
2631 }
2632
2633 // Start container
2634 if err := s.runtime.StartContainer(ctx, container.ID); err != nil {
2635 s.runtime.RemoveContainer(ctx, container.ID, true)
2636 return log.WrapErr(err, "failed to start attachment container")
2637 }
2638
2639 // Wait for attachment to be ready before proceeding
2640 if err := s.waitForAttachmentReady(ctx, container.ID, config); err != nil {
2641 log.WrapErr(err, "attachment readiness check failed, cleaning up")
2642 if stopErr := s.runtime.StopContainer(ctx, container.ID); stopErr != nil {
2643 log.Warn().Err(stopErr).Str(zerowrap.FieldEntityID, container.ID).Msg("failed to stop unready attachment")
2644 }
2645 s.runtime.RemoveContainer(ctx, container.ID, true)
2646 return fmt.Errorf("attachment %q not ready: %w", serviceName, err)
2647 }
2648
2649 // Track attachment
2650 s.mu.Lock()
2651 s.attachments[ownerDomain] = append(s.attachments[ownerDomain], container.ID)
2652 s.mu.Unlock()
2653
2654 // Start log collection for attachment
2655 s.startLogCollection(ctx, container.ID, containerName)
2656
2657 log.Info().Str(zerowrap.FieldEntityID, container.ID).Msg("attachment deployed successfully")
2658 return nil
2659}
2660
2661func (s *Service) attachmentExposedPorts(ctx context.Context, imageRef string) []int {
2662 log := zerowrap.FromCtx(ctx)
2663 exposedPorts, err := s.runtime.GetImageExposedPorts(ctx, imageRef)
2664 if err != nil {
2665 log.WrapErr(err, "failed to get exposed ports for attachment, using defaults")
2666 return []int{}
2667 }
2668 return exposedPorts
2669}
2670
2671func (s *Service) attachmentVolumes(ctx context.Context, containerName, imageRef string) (map[string]string, error) {
2672 volumes, err := s.setupVolumes(ctx, containerName, imageRef)
2673 if err != nil {
2674 return nil, fmt.Errorf("failed to setup volumes for attachment %s with image %s: %w", containerName, imageRef, err)
2675 }
2676 return volumes, nil
2677}
2678
2679func (s *Service) attachmentEnv(ctx context.Context, containerName, imageRef string) ([]string, error) {
2680 envVars, err := s.loadEnvironment(ctx, nil, containerName, imageRef)
2681 if err != nil {
2682 return nil, fmt.Errorf("failed to load environment for attachment %s with image %s: %w", containerName, imageRef, err)
2683 }
2684 return envVars, nil
2685}
2686
2687func (s *Service) attachmentEnvDrifted(ctx context.Context, existing *domain.Container, containerName, serviceImage string) (bool, error) {
2688 imageRef, err := s.buildValidatedImageRef(ctx, serviceImage)
2689 if err != nil {
2690 return false, err
2691 }
2692 currentEnv, err := s.loadEnvironment(ctx, nil, containerName, imageRef)
2693 if err != nil {
2694 return false, err
2695 }
2696
2697 currentHash := hashEnvironment(currentEnv)
2698 existingHash, ok := existing.Labels[domain.LabelEnvHash]
2699 if ok && existingHash == currentHash {
2700 return false, nil
2701 }
2702
2703 return true, nil
2704}
2705
2706func (s *Service) handleRunningAttachment(ctx context.Context, existing *domain.Container, containerName, serviceImage string) (bool, error) {
2707 if existing.Status != string(domain.ContainerStatusRunning) {
2708 return false, nil
2709 }
2710
2711 log := zerowrap.FromCtx(ctx)
2712
2713 // Check image drift
2714 existingImage := existing.Labels[domain.LabelImage]
2715 imageDrifted := existingImage != "" && existingImage != serviceImage
2716
2717 // Check env drift
2718 envDrifted, err := s.attachmentEnvDrifted(ctx, existing, containerName, serviceImage)
2719 if err != nil {
2720 return false, log.WrapErr(err, "failed to check attachment env drift")
2721 }
2722
2723 if !envDrifted && !imageDrifted {
2724 log.Debug().Str("container_name", containerName).Msg("attachment already running with current env and image, skipping")
2725 return true, nil
2726 }
2727
2728 if imageDrifted {
2729 log.Info().Str("container_name", containerName).Str("existing_image", existingImage).Str("desired_image", serviceImage).Msg("attachment image changed, recreating")
2730 }
2731 if envDrifted {
2732 log.Info().Str("container_name", containerName).Msg("attachment env changed, recreating")
2733 }
2734
2735 if err := s.runtime.StopContainer(ctx, existing.ID); err != nil {
2736 return false, log.WrapErr(err, "failed to stop attachment for drift update")
2737 }
2738 if err := s.runtime.RemoveContainer(ctx, existing.ID, true); err != nil {
2739 return false, log.WrapErr(err, "failed to remove attachment for drift update")
2740 }
2741
2742 return false, nil
2743}
2744
2745func (s *Service) removeStoppedAttachment(ctx context.Context, existing *domain.Container, containerName string) error {
2746 if existing == nil {
2747 return nil
2748 }
2749
2750 log := zerowrap.FromCtx(ctx)
2751 log.Info().Str("container_name", containerName).Msg("removing stopped attachment container")
2752 if err := s.runtime.RemoveContainer(ctx, existing.ID, true); err != nil {
2753 return log.WrapErr(err, "failed to remove existing attachment container")
2754 }
2755
2756 return nil
2757}
2758
2759func (s *Service) resolveExistingAttachment(ctx context.Context, ownerDomain, serviceName string) (*domain.Container, error) {
2760 log := zerowrap.FromCtx(ctx)
2761 containerName := fmt.Sprintf("gordon-%s-%s", sanitizeName(ownerDomain), serviceName)
2762 existingContainer := s.findContainerByName(ctx, containerName)
2763 if existingContainer != nil {
2764 return existingContainer, nil
2765 }
2766
2767 containerNameLegacy := fmt.Sprintf("gordon-%s-%s", sanitizeNameLegacy(ownerDomain), serviceName)
2768 existingContainer = s.findContainerByName(ctx, containerNameLegacy)
2769 if existingContainer == nil {
2770 return nil, nil
2771 }
2772
2773 log.Info().Str("container_name", containerNameLegacy).Msg("found attachment with legacy naming, will be replaced")
2774 if err := s.runtime.StopContainer(ctx, existingContainer.ID); err != nil {
2775 return nil, log.WrapErr(err, "failed to stop legacy attachment container")
2776 }
2777 if err := s.runtime.RemoveContainer(ctx, existingContainer.ID, true); err != nil {
2778 return nil, log.WrapErr(err, "failed to remove legacy attachment container")
2779 }
2780
2781 return nil, nil
2782}
2783
2784// Utility functions
2785
2786func normalizeImageRef(image string) string {
2787 if isDigestRef(image) {
2788 return image
2789 }
2790
2791 // Extract repository from image reference, handling port numbers correctly.
2792 // Examples:
2793 // nginx:latest -> docker.io/library/nginx
2794 // user/repo:tag -> docker.io/user/repo
2795 // localhost:5000/image:tag -> localhost:5000/image
2796 // registry.example.com/image:tag -> registry.example.com/image
2797 //
2798 // Note: This function assumes valid Docker image references. Edge cases like
2799 // "registry.com:5000" (registry with port but no image name) are not valid
2800 // image references per Docker's naming conventions, which require at least
2801 // one path component after the registry (e.g., "registry.com:5000/image").
2802
2803 // Find the tag separator: the last colon that isn't part of a port number.
2804 // A colon is part of a port if there's no slash after it until the next colon.
2805 repo := image
2806 lastColon := strings.LastIndex(image, ":")
2807 if lastColon != -1 {
2808 afterColon := image[lastColon+1:]
2809 // If there's no slash after the colon, it's the tag separator
2810 if !strings.Contains(afterColon, "/") {
2811 repo = image[:lastColon]
2812 }
2813 }
2814
2815 if !strings.Contains(repo, "/") {
2816 return "docker.io/library/" + repo
2817 }
2818
2819 if strings.Count(repo, "/") == 1 && !strings.Contains(strings.Split(repo, "/")[0], ".") && !strings.Contains(strings.Split(repo, "/")[0], ":") {
2820 return "docker.io/" + repo
2821 }
2822
2823 return repo
2824}
2825
2826func generateVolumeName(prefix, domainName, volumePath string) string {
2827 return fmt.Sprintf("%s-%s-%s",
2828 prefix,
2829 strings.ReplaceAll(domainName, ".", "-"),
2830 strings.ReplaceAll(strings.Trim(volumePath, "/"), "/", "-"))
2831}
2832
2833func mergeEnvironmentVariables(dockerfileEnv, userEnv []string) []string {
2834 envMap := make(map[string]string)
2835
2836 for _, env := range dockerfileEnv {
2837 if k, v, ok := strings.Cut(env, "="); ok {
2838 envMap[k] = v
2839 }
2840 }
2841
2842 for _, env := range userEnv {
2843 if k, v, ok := strings.Cut(env, "="); ok {
2844 envMap[k] = v
2845 }
2846 }
2847
2848 keys := make([]string, 0, len(envMap))
2849 for k := range envMap {
2850 keys = append(keys, k)
2851 }
2852 slices.Sort(keys)
2853
2854 result := make([]string, 0, len(envMap))
2855 for _, k := range keys {
2856 result = append(result, k+"="+envMap[k])
2857 }
2858
2859 return result
2860}
2861
2862// extractServiceName gets service name from image reference.
2863// "my-postgres:latest" → "postgres", "redis:7" → "redis"
2864func extractServiceName(image string) string {
2865 // Remove tag
2866 parts := strings.Split(image, ":")
2867 name := parts[0]
2868
2869 // Remove registry prefix if present
2870 if strings.Contains(name, "/") {
2871 nameParts := strings.Split(name, "/")
2872 name = nameParts[len(nameParts)-1]
2873 }
2874
2875 // Remove common prefixes like "my-"
2876 name = strings.TrimPrefix(name, "my-")
2877
2878 return name
2879}
2880
2881// sanitizeName is a convenience wrapper around domain.SanitizeDomainForContainer.
2882func sanitizeName(d string) string {
2883 return domain.SanitizeDomainForContainer(d)
2884}
2885
2886// sanitizeNameLegacy is a convenience wrapper around domain.SanitizeDomainForContainerLegacy.
2887func sanitizeNameLegacy(d string) string {
2888 return domain.SanitizeDomainForContainerLegacy(d)
2889}
2890
2891func isContainerNotFoundError(err error) bool {
2892 if err == nil {
2893 return false
2894 }
2895 if errors.Is(err, domain.ErrContainerNotFound) {
2896 return true
2897 }
2898 msg := strings.ToLower(err.Error())
2899 return strings.Contains(msg, "no such container") ||
2900 strings.Contains(msg, "no container with name or id") ||
2901 strings.Contains(msg, "container not found")
2902}
2903
2904func isConnectionRefusedError(err error) bool {
2905 if err == nil {
2906 return false
2907 }
2908 return strings.Contains(strings.ToLower(err.Error()), "connection refused")
2909}
2910
2911// findContainerByName finds a container by its name.
2912func (s *Service) findContainerByName(ctx context.Context, name string) *domain.Container {
2913 containers, err := s.runtime.ListContainers(ctx, true)
2914 if err != nil {
2915 return nil
2916 }
2917
2918 for _, c := range containers {
2919 if c.Name == name {
2920 return c
2921 }
2922 }
2923 return nil
2924}
2925
2926func (s *Service) waitForReady(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig) error {
2927 if err := s.pollContainerRunning(ctx, containerID); err != nil {
2928 return err
2929 }
2930
2931 s.mu.RLock()
2932 cfg := s.config
2933 s.mu.RUnlock()
2934
2935 readinessMode := cfg.ReadinessMode
2936 if readinessMode == "" {
2937 readinessMode = "delay"
2938 }
2939
2940 switch readinessMode {
2941 case "delay":
2942 return s.waitForReadyByDelay(ctx, containerID)
2943 case "docker-health":
2944 _, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(ctx, containerID)
2945 if err != nil {
2946 return err
2947 }
2948 if !hasHealthcheck {
2949 return errors.New("no healthcheck detected")
2950 }
2951 return s.waitForHealthy(ctx, containerID, cfg.HealthTimeout)
2952 default: // auto
2953 return s.readinessCascade(ctx, containerID, containerConfig, cfg)
2954 }
2955}
2956
2957// readinessCascade auto-detects the strongest available readiness signal:
2958// 1. Docker healthcheck (if present) → wait for healthy status
2959// 2. HTTP probe (if gordon.health label set) → GET until 2xx/3xx
2960// 3. Default HTTP probe (GET / on exposed port) → wait for 2xx/3xx
2961// 4. TCP probe (if port info available) → connect until accepted
2962// 5. Delay fallback (last resort) → waitForReadyByDelay
2963func (s *Service) readinessCascade(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) error {
2964 log := zerowrap.FromCtx(ctx)
2965
2966 // 1. Docker healthcheck
2967 _, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(ctx, containerID)
2968 if err != nil {
2969 return err
2970 }
2971 if hasHealthcheck {
2972 log.Info().Msg("readiness cascade: using Docker healthcheck")
2973 return s.waitForHealthy(ctx, containerID, cfg.HealthTimeout)
2974 }
2975
2976 // 2. HTTP probe via gordon.health label
2977 if probed, probeErr := s.tryHTTPProbe(ctx, containerID, containerConfig, cfg); probed {
2978 return probeErr
2979 }
2980
2981 // 3. Default HTTP probe on root path (if port info available)
2982 if probed, probeErr := s.tryDefaultHTTPProbe(ctx, containerID, containerConfig, cfg); probed {
2983 return probeErr
2984 }
2985
2986 // 4. TCP probe (if port info available)
2987 if probed, probeErr := s.tryTCPProbe(ctx, containerID, containerConfig, cfg); probed {
2988 return probeErr
2989 }
2990
2991 // 5. Delay fallback
2992 log.Info().Msg("readiness cascade: using delay fallback")
2993 return s.waitForReadyByDelay(ctx, containerID)
2994}
2995
2996// waitForAttachmentReady runs a reduced readiness cascade for attachments:
2997// Docker healthcheck → TCP probe → delay fallback.
2998// HTTP probes are skipped — attachments are typically databases/caches.
2999func (s *Service) waitForAttachmentReady(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig) error {
3000 if err := s.pollContainerRunning(ctx, containerID); err != nil {
3001 return err
3002 }
3003
3004 log := zerowrap.FromCtx(ctx)
3005
3006 s.mu.RLock()
3007 cfg := s.config
3008 s.mu.RUnlock()
3009
3010 // 1. Docker healthcheck (if present)
3011 _, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(ctx, containerID)
3012 if err != nil {
3013 return err
3014 }
3015 if hasHealthcheck {
3016 log.Info().Msg("attachment readiness: using Docker healthcheck")
3017 timeout := cfg.HealthTimeout
3018 if timeout == 0 {
3019 timeout = 90 * time.Second
3020 }
3021 return s.waitForHealthy(ctx, containerID, timeout)
3022 }
3023
3024 // 2. TCP probe (if port info available)
3025 if containerConfig != nil && len(containerConfig.Ports) > 0 {
3026 ip, port, probeErr := s.resolveProbeEndpoint(ctx, containerID, containerConfig)
3027 if probeErr == nil && ip != "" && port > 0 {
3028 addr := fmt.Sprintf("%s:%d", ip, port)
3029 timeout := cfg.AttachmentReadinessTimeout
3030 if timeout == 0 {
3031 timeout = 30 * time.Second
3032 }
3033 log.Info().Str("addr", addr).Dur("timeout", timeout).Msg("attachment readiness: using TCP probe")
3034 return tcpProbe(ctx, addr, timeout)
3035 }
3036 log.Debug().Err(probeErr).Msg("attachment readiness: TCP probe skipped, could not resolve endpoint")
3037 }
3038
3039 // 3. Delay fallback
3040 log.Info().Msg("attachment readiness: using delay fallback")
3041 return s.waitForReadyByDelay(ctx, containerID)
3042}
3043
3044// tryHTTPProbe attempts an HTTP probe if the gordon.health label is set.
3045// Returns (true, err) if the probe was attempted, (false, nil) if skipped.
3046func (s *Service) tryHTTPProbe(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) (bool, error) {
3047 log := zerowrap.FromCtx(ctx)
3048 if containerConfig == nil {
3049 return false, nil
3050 }
3051 healthPath, ok := containerConfig.Labels[domain.LabelHealth]
3052 if !ok || healthPath == "" {
3053 return false, nil
3054 }
3055 ip, port, probeErr := s.resolveProbeEndpoint(ctx, containerID, containerConfig)
3056 if probeErr != nil || ip == "" || port <= 0 {
3057 log.Debug().Err(probeErr).Msg("readiness cascade: HTTP probe skipped, could not resolve container endpoint")
3058 return false, nil
3059 }
3060 url := fmt.Sprintf("http://%s:%d%s", ip, port, healthPath)
3061 timeout := cfg.HTTPProbeTimeout
3062 if timeout == 0 {
3063 timeout = 60 * time.Second
3064 }
3065 log.Info().Str("url", url).Dur("timeout", timeout).Msg("readiness cascade: using HTTP probe")
3066 return true, httpProbe(ctx, url, timeout)
3067}
3068
3069// tryDefaultHTTPProbe attempts an HTTP GET on "/" using the container's
3070// exposed port. This is the default readiness check for web containers —
3071// it verifies the app is actually serving HTTP, not just accepting TCP.
3072// Returns (true, err) if the probe was attempted, (false, nil) if skipped.
3073func (s *Service) tryDefaultHTTPProbe(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) (bool, error) {
3074 log := zerowrap.FromCtx(ctx)
3075 if containerConfig == nil || len(containerConfig.Ports) == 0 {
3076 return false, nil
3077 }
3078 ip, port, probeErr := s.resolveProbeEndpoint(ctx, containerID, containerConfig)
3079 if probeErr != nil || ip == "" || port <= 0 {
3080 log.Debug().Err(probeErr).Msg("readiness cascade: default HTTP probe skipped, could not resolve container endpoint")
3081 return false, nil
3082 }
3083 url := fmt.Sprintf("http://%s:%d/", ip, port)
3084 timeout := cfg.HTTPProbeTimeout
3085 if timeout == 0 {
3086 timeout = 60 * time.Second
3087 }
3088 log.Info().Str("url", url).Dur("timeout", timeout).Msg("readiness cascade: using default HTTP alive probe")
3089 return true, httpAliveProbe(ctx, url, timeout)
3090}
3091
3092// tryTCPProbe attempts a TCP probe if port info is available.
3093// Returns (true, err) if the probe was attempted, (false, nil) if skipped.
3094func (s *Service) tryTCPProbe(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) (bool, error) {
3095 log := zerowrap.FromCtx(ctx)
3096 if containerConfig == nil || len(containerConfig.Ports) == 0 {
3097 return false, nil
3098 }
3099 ip, port, probeErr := s.resolveProbeEndpoint(ctx, containerID, containerConfig)
3100 if probeErr != nil || ip == "" || port <= 0 {
3101 log.Debug().Err(probeErr).Msg("readiness cascade: TCP probe skipped, could not resolve container endpoint")
3102 return false, nil
3103 }
3104 addr := fmt.Sprintf("%s:%d", ip, port)
3105 timeout := cfg.TCPProbeTimeout
3106 if timeout == 0 {
3107 timeout = 30 * time.Second
3108 }
3109 log.Info().Str("addr", addr).Dur("timeout", timeout).Msg("readiness cascade: using TCP probe")
3110 return true, tcpProbe(ctx, addr, timeout)
3111}
3112
3113// resolveContainerEndpoint returns a host-reachable address for probing.
3114// In rootless podman/Docker setups, container internal IPs are not routable
3115// from the host. We use the host port binding (127.0.0.1:<mapped_port>)
3116// which is always reachable. Falls back to the container's internal IP
3117// only if no host port mapping exists (e.g. host-network mode).
3118func (s *Service) resolveProbeEndpoint(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig) (string, int, error) {
3119 log := zerowrap.FromCtx(ctx)
3120
3121 internalPort := 0
3122 if containerConfig != nil {
3123 if labels := containerConfig.Labels; labels != nil {
3124 for _, key := range []string{domain.LabelProxyPort, domain.LabelPort} {
3125 if portStr, ok := labels[key]; ok && portStr != "" {
3126 port, err := strconv.Atoi(portStr)
3127 if err == nil && port > 0 && port <= 65535 {
3128 internalPort = port
3129 break
3130 }
3131 log.Warn().Str("label", key).Str("port_value", portStr).Msg("invalid probe port label value")
3132 }
3133 }
3134 }
3135 }
3136
3137 if internalPort == 0 {
3138 _, resolvedPort, err := s.runtime.GetContainerNetworkInfo(ctx, containerID)
3139 if err != nil {
3140 return "", 0, err
3141 }
3142 internalPort = resolvedPort
3143 }
3144
3145 hostPort, hostErr := s.runtime.GetContainerPort(ctx, containerID, internalPort)
3146 if hostErr == nil && hostPort > 0 {
3147 log.Debug().
3148 Int("internal_port", internalPort).
3149 Int("host_port", hostPort).
3150 Msg("resolved container endpoint via host port binding")
3151 return "127.0.0.1", hostPort, nil
3152 }
3153
3154 ip, _, fallbackErr := s.runtime.GetContainerNetworkInfo(ctx, containerID)
3155 if fallbackErr != nil {
3156 return "", 0, fallbackErr
3157 }
3158 log.Debug().
3159 Str("ip", ip).
3160 Int("port", internalPort).
3161 Msg("resolved container endpoint via internal IP (no host port binding)")
3162 return ip, internalPort, nil
3163}
3164
3165// waitForReadyByDelay waits using the legacy running+delay strategy.
3166func (s *Service) waitForReadyByDelay(ctx context.Context, containerID string) error {
3167 log := zerowrap.FromCtx(ctx)
3168
3169 s.mu.RLock()
3170 cfg := s.config
3171 s.mu.RUnlock()
3172
3173 delay := cfg.ReadinessDelay
3174 if delay == 0 {
3175 delay = 5 * time.Second
3176 }
3177
3178 log.Debug().Dur("delay", delay).Msg("waiting for container readiness")
3179 select {
3180 case <-time.After(delay):
3181 case <-ctx.Done():
3182 return ctx.Err()
3183 }
3184
3185 // Verify still running after delay
3186 running, err := s.runtime.IsContainerRunning(ctx, containerID)
3187 if err != nil {
3188 return err
3189 }
3190 if running {
3191 return nil
3192 }
3193
3194 return s.waitForRecovery(ctx, containerID)
3195}
3196
3197func (s *Service) waitForHealthy(ctx context.Context, containerID string, timeout time.Duration) error {
3198 if timeout == 0 {
3199 timeout = 90 * time.Second
3200 }
3201 waitCtx, cancel := context.WithTimeout(ctx, timeout)
3202 defer cancel()
3203
3204 var lastStatus string
3205 for {
3206 status, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(waitCtx, containerID)
3207 if err != nil {
3208 return err
3209 }
3210 if !hasHealthcheck {
3211 return errors.New("no healthcheck detected")
3212 }
3213 // Treat empty health status as transitional startup. Some runtimes can
3214 // temporarily report an empty status before first probe results.
3215 if status == "" {
3216 status = "starting"
3217 }
3218 if status == "healthy" {
3219 return nil
3220 }
3221 if status == "unhealthy" {
3222 return errors.New("container reported unhealthy")
3223 }
3224 lastStatus = status
3225
3226 select {
3227 case <-time.After(time.Second):
3228 case <-waitCtx.Done():
3229 return fmt.Errorf("container healthcheck timeout after %s (last status: %s)", timeout, lastStatus)
3230 }
3231 }
3232}
3233
3234// pollContainerRunning polls until the container is running (max 30 seconds).
3235func (s *Service) pollContainerRunning(ctx context.Context, containerID string) error {
3236 for i := 0; i < 30; i++ {
3237 running, err := s.runtime.IsContainerRunning(ctx, containerID)
3238 if err != nil {
3239 return err
3240 }
3241 if running {
3242 return nil
3243 }
3244 // Check if the container has already exited (crash) rather than still starting up.
3245 if info, inspectErr := s.runtime.InspectContainer(ctx, containerID); inspectErr == nil && info != nil && info.Status == string(domain.ContainerStatusExited) {
3246 return fmt.Errorf("container exited immediately with code %d: %w", info.ExitCode, domain.ErrContainerExited)
3247 }
3248 if i == 29 {
3249 return fmt.Errorf("container did not start within 30 seconds")
3250 }
3251 select {
3252 case <-time.After(time.Second):
3253 case <-ctx.Done():
3254 return ctx.Err()
3255 }
3256 }
3257 return fmt.Errorf("container did not start within 30 seconds")
3258}
3259
3260// waitForRecovery waits for a container to recover within the recovery window.
3261func (s *Service) waitForRecovery(ctx context.Context, containerID string) error {
3262 log := zerowrap.FromCtx(ctx)
3263 log.Warn().
3264 Dur("recovery_window", readinessRecoveryWindow).
3265 Msg("container not running after readiness delay, waiting for recovery")
3266
3267 deadline := time.Now().Add(readinessRecoveryWindow)
3268 for time.Now().Before(deadline) {
3269 select {
3270 case <-time.After(time.Second):
3271 case <-ctx.Done():
3272 return ctx.Err()
3273 }
3274
3275 running, err := s.runtime.IsContainerRunning(ctx, containerID)
3276 if err != nil {
3277 return err
3278 }
3279 if running {
3280 log.Info().Msg("container recovered during readiness recovery window")
3281 return nil
3282 }
3283 }
3284
3285 return fmt.Errorf("container not running after readiness delay and recovery window")
3286}
3287
3288// publishContainerDeployed publishes a container.deployed event.
3289func (s *Service) publishContainerDeployed(ctx context.Context, domainName, containerID string) {
3290 payload := &domain.ContainerEventPayload{
3291 ContainerID: containerID,
3292 Domain: domainName,
3293 Action: "deployed",
3294 }
3295
3296 if err := s.eventBus.Publish(domain.EventContainerDeployed, payload); err != nil {
3297 log := zerowrap.FromCtx(ctx)
3298 log.Warn().Err(err).Msg("failed to publish container deployed event")
3299 }
3300}
3301
3302// rewriteToRegistryDomain rewrites an image reference to use the configured registry domain.
3303// e.g., "myapp:latest" -> "registry.example.com/myapp:latest"
3304func rewriteToRegistryDomain(imageRef, registryDomain string) string {
3305 if registryDomain == "" {
3306 return imageRef
3307 }
3308
3309 prefix := registryDomain + "/"
3310 if strings.HasPrefix(imageRef, prefix) {
3311 return imageRef
3312 }
3313
3314 return prefix + imageRef
3315}
3316
3317// rewriteToLocalRegistry rewrites an image reference to use the local registry address.
3318// e.g., "registry.example.com/myapp:latest" -> "localhost:5000/myapp:latest"
3319func rewriteToLocalRegistry(imageRef, registryDomain string, registryPort int) string {
3320 if imageRef == "" {
3321 return imageRef
3322 }
3323
3324 localRegistry := fmt.Sprintf("localhost:%d", registryPort)
3325 localPrefix := localRegistry + "/"
3326 imageRef = strings.TrimPrefix(imageRef, localPrefix)
3327
3328 registryDomain = strings.TrimSuffix(registryDomain, "/")
3329 if registryDomain != "" {
3330 prefix := registryDomain + "/"
3331 imageRef = strings.TrimPrefix(imageRef, prefix)
3332 }
3333
3334 return localPrefix + imageRef
3335}