1// Package container implements the container management use case.
2package container
3
4import (
5 "bufio"
6 "context"
7 "encoding/binary"
8 "errors"
9 "fmt"
10 "io"
11 "maps"
12 "slices"
13 "strconv"
14 "strings"
15 "sync"
16 "time"
17
18 "github.com/bnema/zerowrap"
19 "go.opentelemetry.io/otel"
20 "go.opentelemetry.io/otel/attribute"
21 "go.opentelemetry.io/otel/codes"
22 "go.opentelemetry.io/otel/metric"
23 "go.opentelemetry.io/otel/trace"
24
25 "github.com/bnema/gordon/internal/adapters/out/telemetry"
26 "github.com/bnema/gordon/internal/boundaries/out"
27 "github.com/bnema/gordon/internal/domain"
28)
29
30// Config holds configuration needed by the container service.
31type Config struct {
32 RegistryAuthEnabled bool
33 RegistryDomain string
34 RegistryPort int
35 ServiceTokenUsername string
36 ServiceToken string
37 InternalRegistryUsername string
38 InternalRegistryPassword string
39 PullPolicy string
40 VolumeAutoCreate bool
41 VolumePrefix string
42 VolumePreserve bool
43 NetworkIsolation bool
44 NetworkPrefix string
45 NetworkGroups map[string][]string
46 Attachments map[string][]string
47 ReadinessDelay time.Duration // Delay after container starts before considering it ready
48 ReadinessMode string // Readiness strategy: auto, docker-health, delay
49 HealthTimeout time.Duration // Max wait for health-based readiness
50 DrainDelay time.Duration // Grace period after cache invalidation before stopping old container
51 DrainDelayConfigured bool // True when deploy.drain_delay was explicitly configured
52 DrainMode string // Drain strategy: auto, inflight, delay
53 DrainTimeout time.Duration // Max wait for in-flight requests to drain
54 StabilizationDelay time.Duration // Post-switch monitoring window (default 2s)
55 TCPProbeTimeout time.Duration // TCP probe timeout (default 30s)
56 HTTPProbeTimeout time.Duration // HTTP probe timeout (default 60s)
57 AttachmentReadinessTimeout time.Duration // Max wait for attachment readiness (default 30s)
58 DefaultMemoryLimit int64 // Default memory limit in bytes for containers (0 = no limit)
59 DefaultNanoCPUs int64 // Default CPU quota in nanoseconds for containers (0 = no limit)
60 DefaultPidsLimit int64 // Default max PIDs for containers (0 = no limit)
61}
62
63var tracer = otel.Tracer("gordon.container")
64
65const (
66 PullPolicyAlways = "always"
67 PullPolicyIfNotPresent = "if-not-present"
68 PullPolicyIfTagChanged = "if-tag-changed"
69
70 // readinessRecoveryWindow is an additional grace window used when a container
71 // is briefly not running at the end of readiness delay. This avoids false
72 // negatives during short startup flaps.
73 readinessRecoveryWindow = 30 * time.Second
74 internalPullMaxAttempts = 3
75 deployFailureLogLimit = 20
76)
77
78// Service implements the ContainerService interface.
79type Service struct {
80 runtime out.ContainerRuntime
81 envLoader out.EnvLoader
82 eventBus out.EventPublisher
83 logWriter out.ContainerLogWriter
84 cacheInvalidator out.ProxyCacheInvalidator
85 drainWaiter out.ProxyDrainWaiter
86 config Config
87 configProvider AttachmentConfigProvider // live config reads for attachments/networks (may be nil)
88 metrics *telemetry.Metrics
89 containers map[string]*domain.Container
90 attachments map[string][]string // ownerDomain → []containerIDs
91 managedCount int64 // tracks UpDownCounter value for delta computation
92 mu sync.RWMutex
93 deployMu sync.Map // per-domain deploy locks (domain → *domainDeployLock)
94 cleanupWg sync.WaitGroup // tracks background old-container cleanup goroutines
95 monitor *Monitor
96}
97
98// domainDeployLock is a context-aware mutex using a buffered channel.
99// It allows Lock() to be interrupted by context cancellation.
100type domainDeployLock struct {
101 ch chan struct{}
102}
103
104// newDomainDeployLock creates a new lock that is initially unlocked.
105func newDomainDeployLock() *domainDeployLock {
106 l := &domainDeployLock{ch: make(chan struct{}, 1)}
107 l.ch <- struct{}{}
108 return l
109}
110
111// Lock acquires the lock or returns an error if the context is cancelled.
112func (l *domainDeployLock) Lock(ctx context.Context) error {
113 // Fast path: honor already-canceled contexts before blocking.
114 if err := ctx.Err(); err != nil {
115 return err
116 }
117 select {
118 case <-ctx.Done():
119 return ctx.Err()
120 case <-l.ch:
121 // Re-check context after acquisition: if canceled while we were
122 // in the select, release the token so the lock isn't consumed.
123 if err := ctx.Err(); err != nil {
124 l.ch <- struct{}{}
125 return err
126 }
127 return nil
128 }
129}
130
131// Unlock releases the lock.
132func (l *domainDeployLock) Unlock() {
133 select {
134 case l.ch <- struct{}{}:
135 default:
136 // This should never happen if Lock/Unlock are used correctly.
137 // Panic to detect double-unlocks or other misuse early.
138 panic("domainDeployLock: double unlock detected")
139 }
140}
141
142// NewService creates a new container service.
143func NewService(
144 runtime out.ContainerRuntime,
145 envLoader out.EnvLoader,
146 eventBus out.EventPublisher,
147 logWriter out.ContainerLogWriter,
148 config Config,
149 configProvider AttachmentConfigProvider,
150) *Service {
151 return &Service{
152 runtime: runtime,
153 envLoader: envLoader,
154 eventBus: eventBus,
155 logWriter: logWriter,
156 config: config,
157 configProvider: configProvider,
158 containers: make(map[string]*domain.Container),
159 attachments: make(map[string][]string),
160 }
161}
162
163// SetMetrics sets the telemetry metrics for the container service.
164func (s *Service) SetMetrics(m *telemetry.Metrics) {
165 s.metrics = m
166}
167
168// SetProxyCacheInvalidator sets the proxy cache invalidator for synchronous
169// cache invalidation during zero-downtime deployments. This must be called
170// after construction because the proxy service is created after the container
171// service during application initialization.
172func (s *Service) SetProxyCacheInvalidator(inv out.ProxyCacheInvalidator) {
173 s.mu.Lock()
174 s.cacheInvalidator = inv
175 s.mu.Unlock()
176}
177
178// SetProxyDrainWaiter sets the proxy in-flight drain waiter used during
179// zero-downtime replacement before stopping the old container.
180func (s *Service) SetProxyDrainWaiter(waiter out.ProxyDrainWaiter) {
181 s.mu.Lock()
182 s.drainWaiter = waiter
183 s.mu.Unlock()
184}
185
186// domainDeployMu returns a per-domain lock for serializing deploy operations.
187func (s *Service) domainDeployMu(domain string) *domainDeployLock {
188 v, _ := s.deployMu.LoadOrStore(domain, newDomainDeployLock())
189 // Type assertion is safe: LoadOrStore always stores *domainDeployLock values,
190 // and any concurrent LoadOrStore calls will also store *domainDeployLock.
191 return v.(*domainDeployLock)
192}
193
194// acquireDomainDeployLock acquires the per-domain deploy lock with context cancellation support.
195// Returns an unlock function that must be called to release the lock.
196func (s *Service) acquireDomainDeployLock(ctx context.Context, domain string) (func(), error) {
197 lock := s.domainDeployMu(domain)
198 if err := lock.Lock(ctx); err != nil {
199 return nil, err
200 }
201 return func() { lock.Unlock() }, nil
202}
203
204// buildContainerConfig constructs the container configuration for deployment.
205func (s *Service) deploymentContainerName(containerDomain string, existing *domain.Container) string {
206 canonicalName := fmt.Sprintf("gordon-%s", containerDomain)
207 if existing == nil {
208 return canonicalName
209 }
210
211 // Alternate temporary names if the tracked container was left with a temp suffix.
212 // This prevents name collisions after interrupted zero-downtime deploys.
213 newName := canonicalName + "-new"
214 nextName := canonicalName + "-next"
215 switch existing.Name {
216 case newName:
217 return nextName
218 case nextName:
219 return newName
220 default:
221 return newName
222 }
223}
224
225// containerConfigInput groups the parameters for building a container configuration.
226type containerConfigInput struct {
227 Domain string
228 Image string
229 ImageRef string
230 ExposedPorts []int
231 EnvVars []string
232 EnvHash string
233 Volumes map[string]string
234 NetworkName string
235 ImageLabels map[string]string
236 Existing *domain.Container
237}
238
239func (s *Service) buildContainerConfig(in containerConfigInput) *domain.ContainerConfig {
240 s.mu.RLock()
241 cfg := s.config
242 s.mu.RUnlock()
243
244 containerName := s.deploymentContainerName(in.Domain, in.Existing)
245 labels := map[string]string{
246 domain.LabelDomain: in.Domain,
247 domain.LabelEnvHash: in.EnvHash,
248 domain.LabelImage: in.Image,
249 domain.LabelManaged: "true",
250 domain.LabelRoute: in.Domain,
251 }
252
253 // Propagate proxy/port labels from image so readiness probes can find them
254 for _, key := range []string{domain.LabelProxyPort, domain.LabelPort, domain.LabelHealth} {
255 if v, ok := in.ImageLabels[key]; ok && v != "" {
256 labels[key] = v
257 }
258 }
259
260 // Ensure the label-specified proxy port is included in exposed ports
261 // so Docker creates a host port binding for it.
262 ports := in.ExposedPorts
263 if portStr, ok := labels[domain.LabelProxyPort]; ok {
264 if port, err := strconv.Atoi(portStr); err == nil && port > 0 && port <= 65535 {
265 if !slices.Contains(ports, port) {
266 ports = append(ports, port)
267 }
268 }
269 }
270
271 return &domain.ContainerConfig{
272 Image: in.ImageRef,
273 Name: containerName,
274 Ports: ports,
275 Env: in.EnvVars,
276 Volumes: in.Volumes,
277 NetworkMode: in.NetworkName,
278 Hostname: in.Domain,
279 Labels: labels,
280 AutoRemove: false,
281 MemoryLimit: cfg.DefaultMemoryLimit,
282 NanoCPUs: cfg.DefaultNanoCPUs,
283 PidsLimit: cfg.DefaultPidsLimit,
284 }
285}
286
287// Deploy creates and starts a container for the given route.
288// Implements zero-downtime deployment: new container starts before old one stops.
289func (s *Service) Deploy(ctx context.Context, route domain.Route) (*domain.Container, error) {
290 ctx, span := tracer.Start(ctx, "container.deploy",
291 trace.WithAttributes(
292 attribute.String("domain", route.Domain),
293 attribute.String("image", route.Image),
294 ))
295 defer span.End()
296
297 deployStart := time.Now()
298
299 // Serialize deploys for the same domain to prevent race conditions
300 // (e.g. multiple image.pushed events + explicit deploy call from CLI).
301 unlock, err := s.acquireDomainDeployLock(ctx, route.Domain)
302 if err != nil {
303 return nil, err
304 }
305 defer unlock()
306
307 // Enrich context with use case fields for all downstream logs
308 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
309 zerowrap.FieldLayer: "usecase",
310 zerowrap.FieldUseCase: "Deploy",
311 "domain": route.Domain,
312 })
313 log := zerowrap.FromCtx(ctx)
314
315 // Record deploy metrics and trace status
316 defer s.recordDeployMetrics(ctx, span, route, deployStart, &err)
317
318 existing, hasExisting := s.resolveExistingContainer(ctx, route.Domain)
319
320 resources, err := s.prepareDeployResources(ctx, route, existing)
321 if err != nil {
322 if deployErr := s.wrapPullDeployFailure(route, existing, err); deployErr != nil {
323 err = deployErr
324 return nil, err
325 }
326 return nil, err
327 }
328
329 // Skip redundant deploy: if the existing container is already running
330 // the exact same image (by Docker image ID), return it immediately.
331 // This prevents the double-deploy caused by the event-based deploy
332 // (triggered by image.pushed) racing with the explicit CLI deploy call.
333 if hasExisting {
334 existingForSkip := existing
335 if existingForSkip.ImageID == "" && existingForSkip.Image != "" && normalizeImageRef(existingForSkip.Image) == normalizeImageRef(route.Image) {
336 existingForSkip = s.containerForRedundantCheck(ctx, existingForSkip)
337 }
338 if existingForSkip.ImageID != "" {
339 if skip, container := s.skipRedundantDeploy(ctx, existingForSkip, resources.actualImageRef, resources.envHash); skip {
340 return container, nil
341 }
342 }
343 }
344
345 newContainer, err := s.createStartedContainer(ctx, route, existing, resources)
346 if err != nil {
347 return nil, err
348 }
349
350 invalidated := s.activateDeployedContainer(ctx, route.Domain, newContainer)
351
352 // Post-switch stabilization: verify new container stays running
353 if hasExisting {
354 stable, stabilizeErr := s.stabilizeNewContainer(ctx, route.Domain, newContainer, existing)
355 if stabilizeErr != nil {
356 // Both old and new containers are dead; assign to named return so
357 // the deferred recordDeployMetrics and span see the failure.
358 err = stabilizeErr
359 return nil, err
360 }
361 if !stable {
362 // Rollback performed — old container is restored
363 return existing, nil
364 }
365 }
366
367 // Finalize old container in the background — the new container is already
368 // serving traffic, so there's no reason to block the deploy response while
369 // waiting for the old container to stop (which can take 20s if the app
370 // doesn't handle SIGTERM).
371 s.cleanupWg.Add(1)
372 go func() {
373 defer s.cleanupWg.Done()
374 s.finalizePreviousContainer(context.WithoutCancel(ctx), route.Domain, existing, hasExisting, invalidated, newContainer.ID)
375 }()
376
377 // Start container log collection (non-blocking, errors don't fail deployment)
378 s.startLogCollection(ctx, newContainer.ID, route.Domain)
379
380 log.Info().
381 Str("image", route.Image).
382 Str(zerowrap.FieldEntityID, newContainer.ID).
383 Ints("ports", newContainer.Ports).
384 Str("network", resources.networkName).
385 Bool("zero_downtime", hasExisting).
386 Msg("container deployed successfully")
387
388 return newContainer, nil
389}
390
391// recordDeployMetrics records span error status and deploy metrics at the end of a Deploy call.
392// It is called via defer and receives a pointer to the named return error so it can observe
393// the final error value after all deferred functions have run.
394func (s *Service) recordDeployMetrics(ctx context.Context, span trace.Span, route domain.Route, start time.Time, errPtr *error) {
395 err := *errPtr
396 if err != nil {
397 span.RecordError(err)
398 span.SetStatus(codes.Error, err.Error())
399 }
400 if s.metrics == nil {
401 return
402 }
403 attrs := metric.WithAttributes(
404 attribute.String("domain", route.Domain),
405 attribute.String("image", route.Image),
406 )
407 s.metrics.DeployTotal.Add(ctx, 1, attrs)
408 s.metrics.DeployDuration.Record(ctx, time.Since(start).Seconds(), attrs)
409 if err != nil {
410 s.metrics.DeployErrors.Add(ctx, 1, attrs)
411 }
412}
413
414// skipRedundantDeploy checks whether the existing container is already running
415// the same image (by Docker image ID) as the one we are about to deploy.
416// When a push triggers both an event-based deploy and an explicit CLI deploy,
417// the second one arrives after the first has already completed; this avoids
418// a full redundant create-start-readiness cycle.
419func (s *Service) skipRedundantDeploy(ctx context.Context, existing *domain.Container, actualImageRef, envHash string) (bool, *domain.Container) {
420 log := zerowrap.FromCtx(ctx)
421
422 newImageID, err := s.runtime.GetImageID(ctx, actualImageRef)
423 if err != nil {
424 log.Debug().Err(err).Msg("cannot resolve image ID for redundancy check, proceeding with deploy")
425 return false, nil
426 }
427
428 if existing.ImageID != newImageID {
429 return false, nil
430 }
431
432 // Verify the container is actually still running before skipping.
433 running, err := s.runtime.IsContainerRunning(ctx, existing.ID)
434 if err != nil || !running {
435 return false, nil
436 }
437
438 if existing.Labels != nil {
439 existingEnvHash, hasEnvHash := existing.Labels[domain.LabelEnvHash]
440 if !hasEnvHash || existingEnvHash != envHash {
441 log.Info().
442 Str("container_id", existing.ID).
443 Bool("has_env_hash", hasEnvHash).
444 Msg("env changed, proceeding with deploy despite same image")
445 return false, nil
446 }
447 } else {
448 return false, nil
449 }
450
451 log.Info().
452 Str("container_id", existing.ID).
453 Str("image_id", newImageID).
454 Msg("skipping redundant deploy: container already running this image")
455
456 return true, existing
457}
458
459func (s *Service) containerForRedundantCheck(ctx context.Context, existing *domain.Container) *domain.Container {
460 if existing == nil || existing.ImageID != "" || existing.ID == "" {
461 return existing
462 }
463
464 log := zerowrap.FromCtx(ctx)
465 inspected, err := s.runtime.InspectContainer(ctx, existing.ID)
466 if err != nil {
467 log.Debug().Err(err).Str("container_id", existing.ID).Msg("cannot inspect existing container for redundancy check")
468 return existing
469 }
470 if inspected == nil || inspected.ImageID == "" {
471 return existing
472 }
473
474 existingCopy := *existing
475 existingCopy.ImageID = inspected.ImageID
476 return &existingCopy
477}
478
479type deployResources struct {
480 networkName string
481 actualImageRef string
482 exposedPorts []int
483 imageLabels map[string]string
484 envVars []string
485 envHash string
486 volumes map[string]string
487}
488
489// resolveExistingContainer returns the currently running container for a domain.
490// It first checks the in-memory map (fast path). If not found, it queries the
491// runtime for a running container with matching name and managed label. This
492// handles cases where Gordon restarted and in-memory state is stale.
493//
494// The slow path checks canonical, -new, and -next names because a previous
495// deploy may have been interrupted (e.g. eventbus timeout, Gordon restart)
496// leaving the active container under a temp name. Canonical is preferred when
497// multiple matches exist.
498func (s *Service) resolveExistingContainer(ctx context.Context, domainName string) (*domain.Container, bool) {
499 // Fast path: check in-memory state
500 s.mu.RLock()
501 container, ok := s.containers[domainName]
502 s.mu.RUnlock()
503 if ok {
504 return container, true
505 }
506
507 // Slow path: query runtime for running containers
508 log := zerowrap.FromCtx(ctx)
509 canonicalName := fmt.Sprintf("gordon-%s", domainName)
510 candidateNames := map[string]bool{
511 canonicalName: true,
512 canonicalName + "-new": true,
513 canonicalName + "-next": true,
514 }
515
516 running, err := s.runtime.ListContainers(ctx, false)
517 if err != nil {
518 log.Warn().Err(err).Msg("failed to list running containers for existing container resolution")
519 return nil, false
520 }
521
522 // Prefer canonical name; fall back to any running managed temp container.
523 var best *domain.Container
524 for _, c := range running {
525 if !candidateNames[c.Name] || c.Labels[domain.LabelManaged] != "true" {
526 continue
527 }
528 if best == nil || c.Name == canonicalName {
529 best = c
530 }
531 }
532
533 if best != nil {
534 log.Info().
535 Str("container_id", best.ID).
536 Str("container_name", best.Name).
537 Msg("resolved existing container from runtime (in-memory state was stale)")
538
539 // Update in-memory state so subsequent lookups are fast
540 s.mu.Lock()
541 if _, alreadyTracked := s.containers[domainName]; !alreadyTracked {
542 s.managedCount++
543 }
544 s.containers[domainName] = best
545 s.mu.Unlock()
546
547 return best, true
548 }
549
550 return nil, false
551}
552
553func (s *Service) prepareDeployResources(ctx context.Context, route domain.Route, existing *domain.Container) (*deployResources, error) {
554 log := zerowrap.FromCtx(ctx)
555
556 existingID := ""
557 if existing != nil {
558 existingID = existing.ID
559 }
560 if err := s.cleanupOrphanedContainers(ctx, route.Domain, existingID); err != nil {
561 log.WrapErr(err, "failed to cleanup orphaned containers")
562 }
563
564 imageRef := s.buildImageRef(route.Image)
565 actualImageRef, err := s.ensureImage(ctx, imageRef)
566 if err != nil {
567 return nil, err
568 }
569
570 networkName := s.getNetworkForApp(route.Domain)
571 if err := s.createNetworkIfNeeded(ctx, networkName); err != nil {
572 return nil, log.WrapErr(err, "failed to create network")
573 }
574 if err := s.deployAttachments(ctx, route.Domain, networkName); err != nil {
575 return nil, log.WrapErr(err, "failed to deploy attachments")
576 }
577
578 exposedPorts, err := s.runtime.GetImageExposedPorts(ctx, actualImageRef)
579 if err != nil {
580 log.WrapErr(err, "failed to get exposed ports, using defaults")
581 exposedPorts = []int{80, 8080, 3000}
582 }
583
584 imageLabels, err := s.runtime.GetImageLabels(ctx, actualImageRef)
585 if err != nil {
586 log.Warn().Err(err).Msg("failed to get image labels, skipping label propagation")
587 imageLabels = nil
588 }
589
590 envVars, err := s.loadEnvironment(ctx, route.Env, route.Domain, actualImageRef)
591 if err != nil {
592 return nil, err
593 }
594 envHash := hashEnvironment(envVars)
595
596 volumes, err := s.setupVolumes(ctx, route.Domain, actualImageRef)
597 if err != nil {
598 return nil, err
599 }
600
601 return &deployResources{
602 networkName: networkName,
603 actualImageRef: actualImageRef,
604 exposedPorts: exposedPorts,
605 imageLabels: imageLabels,
606 envVars: envVars,
607 envHash: envHash,
608 volumes: volumes,
609 }, nil
610}
611
612func (s *Service) createStartedContainer(ctx context.Context, route domain.Route, existing *domain.Container, resources *deployResources) (*domain.Container, error) {
613 ctx, span := tracer.Start(ctx, "container.create_and_start")
614 defer span.End()
615
616 log := zerowrap.FromCtx(ctx)
617
618 containerConfig := s.buildContainerConfig(containerConfigInput{
619 Domain: route.Domain,
620 Image: route.Image,
621 ImageRef: resources.actualImageRef,
622 ExposedPorts: resources.exposedPorts,
623 EnvVars: resources.envVars,
624 EnvHash: resources.envHash,
625 Volumes: resources.volumes,
626 NetworkName: resources.networkName,
627 ImageLabels: resources.imageLabels,
628 Existing: existing,
629 })
630
631 newContainer, err := s.runtime.CreateContainer(ctx, containerConfig)
632 if err != nil {
633 return nil, log.WrapErr(err, "failed to create container")
634 }
635 if err := s.runtime.StartContainer(ctx, newContainer.ID); err != nil {
636 s.runtime.RemoveContainer(ctx, newContainer.ID, true)
637 return nil, log.WrapErr(err, "failed to start container")
638 }
639 if !domain.IsSkipReadiness(ctx) {
640 if err := s.waitForReady(ctx, newContainer.ID, containerConfig); err != nil {
641 deployErr := s.newDeployFailure(newContainer, "container failed readiness check", err, s.captureRecentContainerLogs(ctx, newContainer.ID))
642 s.cleanupFailedContainer(ctx, newContainer.ID)
643 return nil, deployErr
644 }
645 }
646
647 inspected, err := s.runtime.InspectContainer(ctx, newContainer.ID)
648 if err != nil {
649 s.cleanupFailedContainer(ctx, newContainer.ID)
650 return nil, log.WrapErr(err, "failed to inspect started container")
651 }
652
653 return inspected, nil
654}
655
656func (s *Service) newDeployFailure(container *domain.Container, cause string, err error, logs []string) error {
657 deployErr := &domain.DeployFailureError{
658 Summary: "failed to deploy",
659 Cause: cause,
660 Hint: inferDeployFailureHint(err),
661 Logs: logs,
662 Err: err,
663 }
664 if container != nil {
665 deployErr.ContainerName = container.Name
666 deployErr.ContainerID = container.ID
667 }
668
669 return deployErr
670}
671
672func (s *Service) wrapPullDeployFailure(route domain.Route, existing *domain.Container, err error) error {
673 if !isPullFailure(err) {
674 return nil
675 }
676
677 return s.newDeployFailure(&domain.Container{Name: s.deploymentContainerName(route.Domain, existing)}, "failed to pull image", err, nil)
678}
679
680func (s *Service) captureRecentContainerLogs(ctx context.Context, containerID string) []string {
681 if ctx.Err() != nil {
682 return nil
683 }
684
685 logStream, err := s.runtime.GetContainerLogs(ctx, containerID, false)
686 if err != nil {
687 return nil
688 }
689 defer func() {
690 _ = logStream.Close()
691 }()
692
693 logs, err := parseDockerLogLines(logStream, deployFailureLogLimit)
694 if err != nil {
695 return nil
696 }
697
698 return logs
699}
700
701func parseDockerLogLines(r io.Reader, maxLines int) ([]string, error) {
702 reader := bufio.NewReader(r)
703 collector := newLogLineCollector(maxLines)
704
705 header, err := reader.Peek(8)
706 if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, bufio.ErrBufferFull) {
707 return nil, err
708 }
709
710 if len(header) >= 8 && looksLikeDockerLogStream(header) {
711 return parseDockerFramedLogLines(reader, collector)
712 }
713
714 return parsePlainLogLines(reader, collector)
715}
716
717func parseDockerFramedLogLines(reader *bufio.Reader, collector *logLineCollector) ([]string, error) {
718 for {
719 frameHeader := make([]byte, 8)
720 if _, err := io.ReadFull(reader, frameHeader); err != nil {
721 if isLogStreamEOF(err) {
722 collector.flush()
723 return collector.lines, nil
724 }
725 return nil, err
726 }
727
728 size := int(binary.BigEndian.Uint32(frameHeader[4:8]))
729 if size <= 0 {
730 continue
731 }
732
733 payload, err := readLogPayload(reader, size)
734 if err != nil {
735 collector.addChunk(string(payload))
736 collector.flush()
737 if isLogStreamEOF(err) {
738 return collector.lines, nil
739 }
740 return nil, err
741 }
742
743 collector.addChunk(string(payload))
744 }
745}
746
747func parsePlainLogLines(reader *bufio.Reader, collector *logLineCollector) ([]string, error) {
748
749 scanner := bufio.NewScanner(reader)
750 for scanner.Scan() {
751 collector.addLine(scanner.Text())
752 }
753 if err := scanner.Err(); err != nil {
754 return nil, err
755 }
756
757 collector.flush()
758 return collector.lines, nil
759}
760
761func readLogPayload(reader *bufio.Reader, size int) ([]byte, error) {
762 payload := make([]byte, size)
763 _, err := io.ReadFull(reader, payload)
764 return payload, err
765}
766
767func isLogStreamEOF(err error) bool {
768 return errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF)
769}
770
771func looksLikeDockerLogStream(data []byte) bool {
772 return len(data) >= 8 && data[1] == 0 && data[2] == 0 && data[3] == 0
773}
774
775type logLineCollector struct {
776 lines []string
777 pending string
778 maxLines int
779}
780
781func newLogLineCollector(maxLines int) *logLineCollector {
782 return &logLineCollector{maxLines: maxLines}
783}
784
785func (c *logLineCollector) addChunk(chunk string) {
786 for _, part := range strings.SplitAfter(chunk, "\n") {
787 c.pending += part
788 if strings.HasSuffix(part, "\n") {
789 c.flush()
790 }
791 }
792}
793
794func (c *logLineCollector) addLine(line string) {
795 line = strings.TrimRight(line, "\r")
796 if line == "" {
797 return
798 }
799 c.lines = append(c.lines, line)
800 if c.maxLines > 0 && len(c.lines) > c.maxLines {
801 c.lines = c.lines[len(c.lines)-c.maxLines:]
802 }
803}
804
805func (c *logLineCollector) flush() {
806 if c.pending == "" {
807 return
808 }
809 c.addLine(strings.TrimRight(c.pending, "\r\n"))
810 c.pending = ""
811}
812
813func inferDeployFailureHint(err error) string {
814 if err == nil {
815 return ""
816 }
817
818 msg := strings.ToLower(err.Error())
819 switch {
820 case strings.Contains(msg, "healthcheck timeout"):
821 return "check startup logs and confirm the health endpoint becomes ready within the configured timeout"
822 case strings.Contains(msg, "unhealthy"):
823 return "inspect recent app logs and verify the healthcheck command reflects actual readiness"
824 case strings.Contains(msg, "no healthcheck detected"):
825 return "add a Docker healthcheck or switch readiness mode"
826 case isPullFailure(err), strings.Contains(msg, "access denied"), strings.Contains(msg, "unauthorized"):
827 return "verify registry auth and confirm the image exists at the requested tag"
828 default:
829 return ""
830 }
831}
832
833func isPullFailure(err error) bool {
834 if err == nil {
835 return false
836 }
837
838 msg := strings.ToLower(err.Error())
839 return strings.Contains(msg, "failed to pull image") ||
840 strings.Contains(msg, "pull access denied") ||
841 strings.Contains(msg, "unauthorized")
842}
843
844func (s *Service) activateDeployedContainer(ctx context.Context, domainName string, container *domain.Container) bool {
845 s.mu.Lock()
846 _, wasTracked := s.containers[domainName]
847 s.containers[domainName] = container
848 if !wasTracked {
849 s.managedCount++
850 }
851 s.mu.Unlock()
852
853 // Track managed container count (only increment for new domains, not replacements)
854 if !wasTracked && s.metrics != nil {
855 s.metrics.ManagedContainers.Add(ctx, 1)
856 }
857
858 s.publishContainerDeployed(ctx, domainName, container.ID)
859
860 s.mu.RLock()
861 inv := s.cacheInvalidator
862 s.mu.RUnlock()
863 if inv != nil {
864 inv.InvalidateTarget(ctx, domainName)
865 return true
866 }
867
868 return false
869}
870
871// stabilizeNewContainer monitors the new container briefly after traffic switch.
872// If it crashes during this window, rolls back to old container.
873// Returns true if stabilization succeeded, false if rollback was performed.
874// Returns an error only when both new and old containers are dead.
875func (s *Service) stabilizeNewContainer(ctx context.Context, domainName string, newContainer, oldContainer *domain.Container) (bool, error) {
876 log := zerowrap.FromCtx(ctx)
877
878 if oldContainer == nil {
879 return true, nil
880 }
881
882 s.mu.RLock()
883 delay := s.config.StabilizationDelay
884 s.mu.RUnlock()
885 if delay == 0 {
886 delay = 2 * time.Second
887 }
888
889 // Brief stabilization: verify new container is still running after delay
890 select {
891 case <-time.After(delay):
892 case <-ctx.Done():
893 return true, nil
894 }
895
896 running, err := s.runtime.IsContainerRunning(ctx, newContainer.ID)
897 if err != nil || !running {
898 log.Error().
899 Str("new_container", newContainer.ID).
900 Str("old_container", oldContainer.ID).
901 Msg("new container crashed during stabilization, rolling back to old")
902
903 // Verify old container is still running before restoring it.
904 // If both old and new are dead (e.g., OOM), returning a dead
905 // container as "existing" would leave the domain in a broken state.
906 oldRunning, oldErr := s.runtime.IsContainerRunning(ctx, oldContainer.ID)
907 if oldErr != nil || !oldRunning {
908 log.Error().
909 Str("old_container", oldContainer.ID).
910 Msg("old container is also not running, cannot rollback")
911
912 // Cleanup failed new container
913 if stopErr := s.runtime.StopContainer(ctx, newContainer.ID); stopErr != nil {
914 log.WrapErrWithFields(stopErr, "failed to stop failed new container during rollback", map[string]any{zerowrap.FieldEntityID: newContainer.ID})
915 }
916 if removeErr := s.runtime.RemoveContainer(ctx, newContainer.ID, true); removeErr != nil {
917 log.WrapErrWithFields(removeErr, "failed to remove failed new container during rollback", map[string]any{zerowrap.FieldEntityID: newContainer.ID})
918 }
919
920 return false, fmt.Errorf("stabilization failed: new container crashed and old container %s is also not running", oldContainer.ID)
921 }
922
923 // Rollback: restore old container as tracked
924 s.mu.Lock()
925 s.containers[domainName] = oldContainer
926 s.mu.Unlock()
927
928 // Re-invalidate proxy cache to point back to old
929 s.mu.RLock()
930 inv := s.cacheInvalidator
931 s.mu.RUnlock()
932 if inv != nil {
933 inv.InvalidateTarget(ctx, domainName)
934 }
935
936 // Cleanup failed new container
937 if stopErr := s.runtime.StopContainer(ctx, newContainer.ID); stopErr != nil {
938 log.WrapErrWithFields(stopErr, "failed to stop failed new container during rollback", map[string]any{zerowrap.FieldEntityID: newContainer.ID})
939 }
940 if removeErr := s.runtime.RemoveContainer(ctx, newContainer.ID, true); removeErr != nil {
941 log.WrapErrWithFields(removeErr, "failed to remove failed new container during rollback", map[string]any{zerowrap.FieldEntityID: newContainer.ID})
942 }
943
944 return false, nil
945 }
946
947 return true, nil
948}
949
950func (s *Service) finalizePreviousContainer(ctx context.Context, domainName string, existing *domain.Container, hasExisting, invalidated bool, newContainerID string) {
951 if !hasExisting {
952 return
953 }
954
955 if invalidated {
956 s.waitForDrain(ctx, existing.ID)
957 }
958
959 s.cleanupOldContainer(ctx, existing, newContainerID, domainName)
960}
961
962func (s *Service) waitForDrain(ctx context.Context, oldContainerID string) {
963 log := zerowrap.FromCtx(ctx)
964
965 s.mu.RLock()
966 cfg := s.config
967 waiter := s.drainWaiter
968 s.mu.RUnlock()
969
970 drainMode := cfg.DrainMode
971 if drainMode == "" {
972 drainMode = "delay"
973 }
974
975 shouldUseInFlight := (drainMode == "inflight" || drainMode == "auto") && waiter != nil
976 if shouldUseInFlight {
977 timeout := cfg.DrainTimeout
978 if timeout == 0 {
979 timeout = 30 * time.Second
980 }
981 drained := waiter.WaitForNoInFlight(ctx, oldContainerID, timeout)
982 if !drained {
983 log.Warn().
984 Str("old_container_id", oldContainerID).
985 Dur("drain_timeout", timeout).
986 Msg("drain wait timed out; old container may still have in-flight traffic")
987 }
988 return
989 }
990
991 drainDelay := 2 * time.Second
992 if cfg.DrainDelayConfigured {
993 drainDelay = cfg.DrainDelay
994 }
995 if drainDelay <= 0 {
996 return
997 }
998 select {
999 case <-time.After(drainDelay):
1000 case <-ctx.Done():
1001 }
1002}
1003
1004// restartContainerWithRecovery restarts a container, reconciling stale state if needed.
1005// It returns the (possibly refreshed) container and updated attachment IDs.
1006func (s *Service) restartContainerWithRecovery(ctx context.Context, domainName string, container *domain.Container, attachmentIDs []string) (*domain.Container, []string, error) {
1007 log := zerowrap.FromCtx(ctx)
1008
1009 if err := s.runtime.RestartContainer(ctx, container.ID); err != nil {
1010 if !isContainerNotFoundError(err) {
1011 return nil, nil, log.WrapErr(err, "failed to restart container")
1012 }
1013
1014 // In-memory container ID can become stale after external runtime changes.
1015 // Re-sync and retry once with the latest tracked container.
1016 log.Warn().
1017 Err(err).
1018 Str(zerowrap.FieldEntityID, container.ID).
1019 Msg("tracked container missing during restart, attempting state reconciliation")
1020
1021 if syncErr := s.SyncContainers(ctx); syncErr != nil {
1022 log.Warn().Err(syncErr).Msg("failed to sync container state during restart recovery")
1023 }
1024
1025 s.mu.RLock()
1026 refreshed, refreshedExists := s.containers[domainName]
1027 attachmentIDs = append([]string{}, s.attachments[domainName]...)
1028 s.mu.RUnlock()
1029
1030 if !refreshedExists || refreshed == nil {
1031 return nil, nil, domain.ErrContainerNotFound
1032 }
1033 if err := s.runtime.RestartContainer(ctx, refreshed.ID); err != nil {
1034 return nil, nil, log.WrapErr(err, "failed to restart container after state reconciliation")
1035 }
1036 return refreshed, attachmentIDs, nil
1037 }
1038
1039 return container, attachmentIDs, nil
1040}
1041
1042// Restart restarts a running container for the given domain.
1043func (s *Service) Restart(ctx context.Context, domainName string, withAttachments bool) error {
1044 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1045 zerowrap.FieldLayer: "usecase",
1046 zerowrap.FieldUseCase: "Restart",
1047 "domain": domainName,
1048 })
1049 log := zerowrap.FromCtx(ctx)
1050
1051 s.mu.RLock()
1052 container, exists := s.containers[domainName]
1053 attachments := s.attachments[domainName]
1054 var attachmentIDs []string
1055 if len(attachments) > 0 {
1056 attachmentIDs = make([]string, len(attachments))
1057 copy(attachmentIDs, attachments)
1058 }
1059 s.mu.RUnlock()
1060
1061 if !exists || container == nil {
1062 return domain.ErrContainerNotFound
1063 }
1064
1065 // When attachments are requested, sync runtime state first to get accurate
1066 // attachment tracking, then check if configured attachments are deployed.
1067 // If fewer attachment containers are tracked than configured, fail early with
1068 // guidance rather than restarting the main container into a broken state
1069 // (e.g., missing database hostname).
1070 if withAttachments {
1071 if syncErr := s.SyncContainers(ctx); syncErr != nil {
1072 log.Warn().Err(syncErr).Msg("failed to sync container state before attachment check")
1073 }
1074
1075 // Re-read attachment IDs after sync
1076 s.mu.RLock()
1077 if freshAttachments := s.attachments[domainName]; len(freshAttachments) > 0 {
1078 attachmentIDs = make([]string, len(freshAttachments))
1079 copy(attachmentIDs, freshAttachments)
1080 } else {
1081 attachmentIDs = nil
1082 }
1083 s.mu.RUnlock()
1084
1085 configuredAttachments := s.resolveAttachmentsForDomain(domainName)
1086 if len(configuredAttachments) > 0 && len(attachmentIDs) < len(configuredAttachments) {
1087 missing := len(configuredAttachments) - len(attachmentIDs)
1088 return fmt.Errorf("%w: domain %q has %d configured attachment(s) but only %d deployed (%d missing)",
1089 domain.ErrAttachmentNotDeployed, domainName, len(configuredAttachments), len(attachmentIDs), missing)
1090 }
1091 }
1092
1093 // Restart the main container (with stale-state recovery)
1094 container, attachmentIDs, err := s.restartContainerWithRecovery(ctx, domainName, container, attachmentIDs)
1095 if err != nil {
1096 return err
1097 }
1098 log.Info().Str(zerowrap.FieldEntityID, container.ID).Msg("container restarted")
1099
1100 // Record restart metric
1101 if s.metrics != nil {
1102 s.metrics.ContainerRestarts.Add(ctx, 1, metric.WithAttributes(
1103 attribute.String("domain", domainName),
1104 attribute.String("source", "api"),
1105 ))
1106 }
1107
1108 // Restart attachments if requested
1109 if withAttachments && len(attachmentIDs) > 0 {
1110 for _, attachID := range attachmentIDs {
1111 if err := s.runtime.RestartContainer(ctx, attachID); err != nil {
1112 log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to restart attachment")
1113 continue // Don't fail the whole operation for one attachment
1114 }
1115 log.Info().Str("attachment_id", attachID).Msg("attachment restarted")
1116 }
1117 }
1118
1119 return nil
1120}
1121
1122// Stop stops a running container.
1123func (s *Service) Stop(ctx context.Context, containerID string) error {
1124 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1125 zerowrap.FieldLayer: "usecase",
1126 zerowrap.FieldUseCase: "Stop",
1127 zerowrap.FieldEntityID: containerID,
1128 })
1129 log := zerowrap.FromCtx(ctx)
1130
1131 // Stop log collection before stopping container
1132 if s.logWriter != nil {
1133 if err := s.logWriter.StopLogging(containerID); err != nil {
1134 log.Warn().Err(err).Msg("failed to stop container log collection")
1135 }
1136 }
1137
1138 if err := s.runtime.StopContainer(ctx, containerID); err != nil {
1139 return log.WrapErr(err, "failed to stop container")
1140 }
1141
1142 log.Info().Msg("container stopped")
1143 return nil
1144}
1145
1146// Remove removes a container.
1147func (s *Service) Remove(ctx context.Context, containerID string, force bool) error {
1148 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1149 zerowrap.FieldLayer: "usecase",
1150 zerowrap.FieldUseCase: "Remove",
1151 zerowrap.FieldEntityID: containerID,
1152 })
1153 log := zerowrap.FromCtx(ctx)
1154
1155 // Stop log collection before removing container
1156 if s.logWriter != nil {
1157 if err := s.logWriter.StopLogging(containerID); err != nil {
1158 log.Warn().Err(err).Msg("failed to stop container log collection")
1159 }
1160 }
1161
1162 // Find domain and attachment IDs for cleanup
1163 containerDomain, attachmentIDs, cfg := s.findContainerContext(containerID)
1164
1165 if err := s.runtime.RemoveContainer(ctx, containerID, force); err != nil {
1166 return log.WrapErr(err, "failed to remove container")
1167 }
1168
1169 // Cleanup volumes
1170 if containerDomain != "" && !cfg.VolumePreserve {
1171 if err := s.cleanupVolumesForDomain(ctx, containerDomain); err != nil {
1172 log.WrapErrWithFields(err, "failed to cleanup volumes", map[string]any{"domain": containerDomain})
1173 }
1174 }
1175
1176 // Clean up attachment containers
1177 s.removeAttachments(ctx, attachmentIDs)
1178
1179 // Remove from tracking
1180 s.mu.Lock()
1181 var removedDomain string
1182 for d, c := range s.containers {
1183 if c.ID == containerID {
1184 delete(s.containers, d)
1185 removedDomain = d
1186 s.managedCount--
1187 log.Info().Str("domain", d).Msg("container removed")
1188 break
1189 }
1190 }
1191 if removedDomain != "" {
1192 delete(s.attachments, removedDomain)
1193 }
1194 s.mu.Unlock()
1195
1196 // Decrement managed container count
1197 if removedDomain != "" && s.metrics != nil {
1198 s.metrics.ManagedContainers.Add(ctx, -1)
1199 }
1200
1201 // Note: We intentionally do NOT clean up deployMu entries to avoid a race condition:
1202 // If Remove deletes the mutex entry while a concurrent Deploy is acquiring the lock,
1203 // the Deploy would create a fresh mutex, breaking serialization and allowing concurrent
1204 // deploys to the same domain. The memory footprint is acceptable (one small struct per
1205 // domain ever deployed), and this is the safer choice for correctness.
1206
1207 // Cleanup network
1208 if removedDomain != "" && cfg.NetworkIsolation {
1209 networkName := s.getNetworkForApp(removedDomain)
1210 if err := s.cleanupNetworkIfEmpty(ctx, networkName); err != nil {
1211 log.WrapErrWithFields(err, "failed to cleanup network", map[string]any{"network": networkName})
1212 }
1213 }
1214
1215 return nil
1216}
1217
1218// findContainerContext looks up the domain, attachment IDs, and config snapshot for a container.
1219func (s *Service) findContainerContext(containerID string) (string, []string, Config) {
1220 s.mu.RLock()
1221 defer s.mu.RUnlock()
1222 cfg := s.config
1223 var containerDomain string
1224 for d, c := range s.containers {
1225 if c.ID == containerID {
1226 containerDomain = d
1227 break
1228 }
1229 }
1230 var attachmentIDs []string
1231 if containerDomain != "" {
1232 attachmentIDs = append(attachmentIDs, s.attachments[containerDomain]...)
1233 }
1234 return containerDomain, attachmentIDs, cfg
1235}
1236
1237// removeAttachments stops and removes attachment containers best-effort.
1238func (s *Service) removeAttachments(ctx context.Context, attachmentIDs []string) {
1239 log := zerowrap.FromCtx(ctx)
1240 for _, attachID := range attachmentIDs {
1241 if s.logWriter != nil {
1242 if err := s.logWriter.StopLogging(attachID); err != nil {
1243 log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to stop attachment log collection")
1244 }
1245 }
1246 if err := s.runtime.StopContainer(ctx, attachID); err != nil {
1247 log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to stop attachment container")
1248 }
1249 if err := s.runtime.RemoveContainer(ctx, attachID, true); err != nil {
1250 log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to remove attachment container")
1251 }
1252 }
1253}
1254
1255// Get retrieves a container by domain.
1256func (s *Service) Get(_ context.Context, domainName string) (*domain.Container, bool) {
1257 s.mu.RLock()
1258 defer s.mu.RUnlock()
1259 container, exists := s.containers[domainName]
1260 return container, exists
1261}
1262
1263// List returns all managed containers.
1264func (s *Service) List(_ context.Context) map[string]*domain.Container {
1265 s.mu.RLock()
1266 defer s.mu.RUnlock()
1267
1268 result := make(map[string]*domain.Container, len(s.containers))
1269 maps.Copy(result, s.containers)
1270 return result
1271}
1272
1273// ListRoutesWithDetails returns routes with network and attachment info.
1274// Note: Container map is copied under lock, then external calls are made without lock.
1275// If containers are removed between copy and runtime calls, errors are handled gracefully
1276// (network becomes empty string). This trade-off avoids holding locks during I/O.
1277func (s *Service) ListRoutesWithDetails(ctx context.Context) []domain.RouteInfo {
1278 s.mu.RLock()
1279 containers := make(map[string]*domain.Container, len(s.containers))
1280 maps.Copy(containers, s.containers)
1281 s.mu.RUnlock()
1282
1283 // Fetch all attachments once to avoid N+1 queries
1284 attachmentsByDomain := s.getAllAttachments(ctx)
1285
1286 results := make([]domain.RouteInfo, 0, len(containers))
1287 for domainName, container := range containers {
1288 network := ""
1289 image := ""
1290 containerID := ""
1291 status := ""
1292 if container != nil {
1293 containerID = container.ID
1294 status = container.Status
1295 image = container.Image
1296 if container.Labels != nil {
1297 if labelImage, ok := container.Labels[domain.LabelImage]; ok && labelImage != "" {
1298 image = labelImage
1299 }
1300 }
1301 // Strip registry domain prefix from image for cleaner display
1302 image = s.stripRegistryPrefix(image)
1303 if networkName, err := s.runtime.GetContainerNetwork(ctx, container.ID); err == nil {
1304 network = networkName
1305 }
1306 }
1307
1308 results = append(results, domain.RouteInfo{
1309 Domain: domainName,
1310 Image: image,
1311 ContainerID: containerID,
1312 ContainerStatus: status,
1313 Network: network,
1314 Attachments: attachmentsByDomain[domainName],
1315 })
1316 }
1317
1318 return results
1319}
1320
1321// stripRegistryPrefix removes the configured registry domain prefix from an image reference.
1322// For example, "reg.example.com/myapp:latest" becomes "myapp:latest" if registry domain is "reg.example.com".
1323func (s *Service) stripRegistryPrefix(image string) string {
1324 s.mu.RLock()
1325 cfg := s.config
1326 s.mu.RUnlock()
1327
1328 if cfg.RegistryDomain == "" {
1329 return image
1330 }
1331 prefix := strings.TrimSuffix(cfg.RegistryDomain, "/") + "/"
1332 if strings.HasPrefix(image, prefix) {
1333 return strings.TrimPrefix(image, prefix)
1334 }
1335 return image
1336}
1337
1338// ListAttachments returns attachments for a domain.
1339func (s *Service) ListAttachments(ctx context.Context, domainName string) []domain.Attachment {
1340 return s.getAttachmentsForDomain(ctx, domainName)
1341}
1342
1343// ListNetworks returns Gordon-managed networks.
1344func (s *Service) ListNetworks(ctx context.Context) ([]*domain.NetworkInfo, error) {
1345 s.mu.RLock()
1346 cfg := s.config
1347 s.mu.RUnlock()
1348
1349 networks, err := s.runtime.ListNetworks(ctx)
1350 if err != nil {
1351 return nil, err
1352 }
1353
1354 var filtered []*domain.NetworkInfo
1355 for _, network := range networks {
1356 if strings.HasPrefix(network.Name, cfg.NetworkPrefix+"-") && network.Labels[domain.LabelManaged] == "true" {
1357 filtered = append(filtered, network)
1358 }
1359 }
1360
1361 return filtered, nil
1362}
1363
1364// HealthCheck performs health checks on all containers.
1365func (s *Service) HealthCheck(ctx context.Context) map[string]bool {
1366 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1367 zerowrap.FieldLayer: "usecase",
1368 zerowrap.FieldUseCase: "HealthCheck",
1369 })
1370 log := zerowrap.FromCtx(ctx)
1371
1372 // Copy container IDs under lock, then release before making Docker API calls
1373 s.mu.RLock()
1374 snapshot := make(map[string]string, len(s.containers))
1375 for d, c := range s.containers {
1376 snapshot[d] = c.ID
1377 }
1378 s.mu.RUnlock()
1379
1380 health := make(map[string]bool, len(snapshot))
1381 for d, id := range snapshot {
1382 running, err := s.runtime.IsContainerRunning(ctx, id)
1383 if err != nil {
1384 log.WrapErrWithFields(err, "health check failed", map[string]any{"domain": d, zerowrap.FieldEntityID: id})
1385 health[d] = false
1386 } else {
1387 health[d] = running
1388 }
1389 }
1390 return health
1391}
1392
1393// SyncContainers synchronizes containers with runtime state.
1394func (s *Service) SyncContainers(ctx context.Context) error {
1395 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1396 zerowrap.FieldLayer: "usecase",
1397 zerowrap.FieldUseCase: "SyncContainers",
1398 })
1399 log := zerowrap.FromCtx(ctx)
1400
1401 // List containers without holding lock to avoid blocking during Docker API call
1402 allContainers, err := s.runtime.ListContainers(ctx, false)
1403 if err != nil {
1404 return log.WrapErr(err, "failed to list containers")
1405 }
1406
1407 managed := make(map[string]*domain.Container)
1408 attachments := make(map[string][]string)
1409 for _, c := range allContainers {
1410 if c.Labels == nil || c.Labels[domain.LabelManaged] != "true" {
1411 continue
1412 }
1413 if c.Labels[domain.LabelAttachment] == "true" {
1414 owner := c.Labels[domain.LabelAttachedTo]
1415 if owner != "" {
1416 attachments[owner] = append(attachments[owner], c.ID)
1417 }
1418 continue
1419 }
1420 if d, ok := c.Labels[domain.LabelDomain]; ok {
1421 managed[d] = c
1422 }
1423 }
1424
1425 s.mu.Lock()
1426 s.containers = managed
1427 s.attachments = attachments
1428 newCount := int64(len(managed))
1429 delta := newCount - s.managedCount
1430 s.managedCount = newCount
1431 s.mu.Unlock()
1432
1433 // Report initial/delta managed container count to OTel UpDownCounter.
1434 if delta != 0 && s.metrics != nil {
1435 s.metrics.ManagedContainers.Add(ctx, delta)
1436 }
1437
1438 log.Info().Int(zerowrap.FieldCount, len(managed)).Msg("container state synchronized")
1439 return nil
1440}
1441
1442// AutoStart starts containers for the provided routes that aren't running.
1443// It skips readiness checks to avoid blocking boot; the background monitor
1444// handles crash recovery.
1445func (s *Service) AutoStart(ctx context.Context, routes []domain.Route) error {
1446 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1447 zerowrap.FieldLayer: "usecase",
1448 zerowrap.FieldUseCase: "AutoStart",
1449 })
1450 log := zerowrap.FromCtx(ctx)
1451
1452 log.Info().Int("route_count", len(routes)).Msg("auto-starting containers for configured routes")
1453
1454 // Filter out routes that already have a running container.
1455 var pending []domain.Route
1456 skipped := 0
1457 for _, route := range routes {
1458 if _, exists := s.Get(ctx, route.Domain); exists {
1459 log.Debug().Str("domain", route.Domain).Msg("container already running, skipping")
1460 skipped++
1461 } else {
1462 pending = append(pending, route)
1463 }
1464 }
1465
1466 if len(pending) == 0 {
1467 log.Info().Int("skipped", skipped).Msg("auto-start completed, all containers already running")
1468 return nil
1469 }
1470
1471 // Deploy all pending routes concurrently with readiness checks skipped.
1472 deployCtx := domain.WithSkipReadiness(ctx)
1473
1474 type result struct {
1475 route domain.Route
1476 err error
1477 }
1478 results := make(chan result, len(pending))
1479 sem := make(chan struct{}, 4) // limit concurrency
1480
1481 for _, route := range pending {
1482 sem <- struct{}{}
1483 go func(r domain.Route) {
1484 defer func() { <-sem }()
1485
1486 log.Info().Str("domain", r.Domain).Str("image", r.Image).Msg("auto-starting container for route")
1487
1488 if _, err := s.Deploy(deployCtx, r); err != nil {
1489 results <- result{route: r, err: err}
1490 return
1491 }
1492 results <- result{route: r}
1493 }(route)
1494 }
1495
1496 var started, errCount int
1497 for i := 0; i < len(pending); i++ {
1498 res := <-results
1499 if res.err != nil {
1500 log.Warn().Err(res.err).Str("domain", res.route.Domain).Msg("failed to auto-start container")
1501 errCount++
1502 } else {
1503 started++
1504 }
1505 }
1506
1507 log.Info().
1508 Int("started", started).
1509 Int("skipped", skipped).
1510 Int("errors", errCount).
1511 Msg("auto-start completed")
1512
1513 if errCount > 0 {
1514 return fmt.Errorf("auto-start completed with %d errors", errCount)
1515 }
1516 return nil
1517}
1518
1519// Shutdown gracefully shuts down all managed containers.
1520func (s *Service) Shutdown(ctx context.Context) error {
1521 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1522 zerowrap.FieldLayer: "usecase",
1523 zerowrap.FieldUseCase: "Shutdown",
1524 })
1525 log := zerowrap.FromCtx(ctx)
1526 log.Info().Msg("shutting down container manager...")
1527
1528 // Wait for any in-flight background cleanup goroutines to finish
1529 s.cleanupWg.Wait()
1530
1531 // Containers are left running across Gordon restarts.
1532 // SyncContainers + AutoStart will pick them back up on next boot.
1533
1534 // Close log writer to stop all log collection
1535 if s.logWriter != nil {
1536 if err := s.logWriter.Close(); err != nil {
1537 log.Warn().Err(err).Msg("failed to close container log writer")
1538 }
1539 }
1540
1541 log.Info().Msg("container manager shutdown complete")
1542 return nil
1543}
1544
1545// WaitForCleanup blocks until all background cleanup goroutines complete.
1546// Intended for use in tests to avoid mock assertion races.
1547func (s *Service) WaitForCleanup() {
1548 s.cleanupWg.Wait()
1549}
1550
1551// StartMonitor begins background monitoring of tracked containers.
1552// The monitor restarts crashed containers and detects crash loops.
1553// Safe to call multiple times; subsequent calls are no-ops.
1554func (s *Service) StartMonitor(ctx context.Context) {
1555 s.mu.Lock()
1556 if s.monitor != nil {
1557 s.mu.Unlock()
1558 return
1559 }
1560 s.monitor = newMonitor(s)
1561 m := s.monitor
1562 m.Start(ctx)
1563 s.mu.Unlock()
1564}
1565
1566// StopMonitor stops the background container monitor.
1567// Safe to call multiple times; subsequent calls are no-ops.
1568func (s *Service) StopMonitor() {
1569 s.mu.Lock()
1570 m := s.monitor
1571 s.monitor = nil
1572 s.mu.Unlock()
1573 if m != nil {
1574 m.Stop()
1575 }
1576}
1577
1578// UpdateConfig updates the service configuration.
1579func (s *Service) UpdateConfig(config Config) {
1580 s.mu.Lock()
1581 s.config = config
1582 s.mu.Unlock()
1583}
1584
1585// UpdateAttachments updates only the attachment configuration in the service.
1586// This is called after a config reload to propagate attachment changes without restart.
1587// The incoming map is deep-copied so external callers cannot mutate service state.
1588func (s *Service) UpdateAttachments(attachments map[string][]string) {
1589 var copied map[string][]string
1590 if attachments != nil {
1591 copied = make(map[string][]string, len(attachments))
1592 for k, v := range attachments {
1593 sl := make([]string, len(v))
1594 copy(sl, v)
1595 copied[k] = sl
1596 }
1597 }
1598 s.mu.Lock()
1599 s.config.Attachments = copied
1600 s.mu.Unlock()
1601}
1602
1603// Helper methods
1604
1605// cleanupFailedContainer stops and removes a container that failed to start properly.
1606func (s *Service) cleanupFailedContainer(ctx context.Context, containerID string) {
1607 log := zerowrap.FromCtx(ctx)
1608 if err := s.runtime.StopContainer(ctx, containerID); err != nil {
1609 log.Warn().Err(err).Str(zerowrap.FieldEntityID, containerID).Msg("failed to stop container after failure")
1610 }
1611 if err := s.runtime.RemoveContainer(ctx, containerID, true); err != nil {
1612 log.Warn().Err(err).Str(zerowrap.FieldEntityID, containerID).Msg("failed to remove container after failure")
1613 }
1614}
1615
1616// cleanupOldContainer stops and removes an old container after zero-downtime switch.
1617// It also renames the new container to the canonical name.
1618func (s *Service) cleanupOldContainer(ctx context.Context, old *domain.Container, newContainerID, domainName string) {
1619 log := zerowrap.FromCtx(ctx)
1620 log.Info().Str(zerowrap.FieldEntityID, old.ID).Msg("stopping old container after zero-downtime switch")
1621
1622 if s.logWriter != nil {
1623 if err := s.logWriter.StopLogging(old.ID); err != nil {
1624 log.Warn().Err(err).Str(zerowrap.FieldEntityID, old.ID).Msg("failed to stop logging for old container")
1625 }
1626 }
1627
1628 if err := s.runtime.StopContainer(ctx, old.ID); err != nil {
1629 log.Warn().Err(err).Str(zerowrap.FieldEntityID, old.ID).Msg("failed to stop old container")
1630 }
1631
1632 if err := s.runtime.RemoveContainer(ctx, old.ID, true); err != nil {
1633 log.Warn().Err(err).Str(zerowrap.FieldEntityID, old.ID).Msg("failed to remove old container")
1634 }
1635
1636 // Rename new container to canonical name
1637 canonicalName := fmt.Sprintf("gordon-%s", domainName)
1638 if err := s.runtime.RenameContainer(ctx, newContainerID, canonicalName); err != nil {
1639 log.Warn().Err(err).Str("canonical_name", canonicalName).Msg("failed to rename container to canonical name")
1640 }
1641}
1642
1643// startLogCollection starts log collection for a container in the background.
1644// Errors are logged but don't fail the calling operation.
1645func (s *Service) startLogCollection(ctx context.Context, containerID, domainName string) {
1646 if s.logWriter == nil {
1647 return
1648 }
1649
1650 log := zerowrap.FromCtx(ctx)
1651
1652 // Use context.WithoutCancel so the log stream outlives the HTTP request.
1653 // Without this, the Docker log stream closes when the deploy response completes.
1654 bgCtx := context.WithoutCancel(ctx)
1655
1656 logStream, err := s.runtime.GetContainerLogs(bgCtx, containerID, true)
1657 if err != nil {
1658 log.Warn().Err(err).Msg("failed to get container logs for collection")
1659 return
1660 }
1661
1662 if err := s.logWriter.StartLogging(bgCtx, containerID, domainName, logStream); err != nil {
1663 log.Warn().Err(err).Msg("failed to start container log collection")
1664 logStream.Close()
1665 }
1666}
1667
1668func (s *Service) buildImageRef(image string) string {
1669 s.mu.RLock()
1670 cfg := s.config
1671 s.mu.RUnlock()
1672
1673 if !cfg.RegistryAuthEnabled || cfg.RegistryDomain == "" {
1674 return image
1675 }
1676
1677 // Normalize registry domain by trimming any trailing slash
1678 reg := strings.TrimSuffix(cfg.RegistryDomain, "/")
1679
1680 // Check if image already has the registry domain prefix
1681 prefix := reg + "/"
1682 if strings.HasPrefix(image, prefix) {
1683 return image
1684 }
1685
1686 // Check if image already has an explicit registry (host:port/ or host.domain/).
1687 // We need to detect patterns like:
1688 // - "docker.io/library/nginx:latest" (has dot and slash)
1689 // - "localhost:5001/myapp:latest" (has host:port/ pattern)
1690 // - "myregistry:5000/app:v1" (has host:port/ pattern)
1691 // - "[fd00::1]:5000/app:v1" (IPv6 with port)
1692 if hasExplicitRegistry(image) {
1693 return image
1694 }
1695
1696 return fmt.Sprintf("%s/%s", reg, image)
1697}
1698
1699// hasExplicitRegistry checks if an image reference already includes an explicit registry.
1700// Returns true for patterns like "docker.io/image", "localhost:5000/image", "localhost/image",
1701// "registry:8080/image", "[fd00::1]/image", "[fd00::1]:5000/image".
1702func hasExplicitRegistry(image string) bool {
1703 // Find the first slash which separates registry from image name
1704 slashIdx := strings.Index(image, "/")
1705 if slashIdx == -1 {
1706 return false // No slash means no registry prefix (e.g., "myapp:latest")
1707 }
1708
1709 // Extract the part before the first slash (potential registry)
1710 registryPart := image[:slashIdx]
1711
1712 // Check for bracketed IPv6 address (e.g., "[fd00::1]" or "[fd00::1]:5000")
1713 if strings.HasPrefix(registryPart, "[") {
1714 // Look for closing bracket
1715 if closeBracket := strings.Index(registryPart, "]"); closeBracket != -1 {
1716 // Valid bracketed IPv6: either ends at ] or has ]:port
1717 return true
1718 }
1719 }
1720
1721 // Check for localhost (with or without port)
1722 if registryPart == "localhost" {
1723 return true
1724 }
1725
1726 // Check for host:port pattern (e.g., "localhost:5000", "registry:8080")
1727 if colonIdx := strings.LastIndex(registryPart, ":"); colonIdx != -1 {
1728 port := registryPart[colonIdx+1:]
1729 // If everything after colon is digits, it's a port
1730 if len(port) > 0 && isNumeric(port) {
1731 return true
1732 }
1733 }
1734
1735 // Check for domain pattern (has a dot, e.g., "docker.io", "gcr.io")
1736 if strings.Contains(registryPart, ".") {
1737 return true
1738 }
1739
1740 return false
1741}
1742
1743// isNumeric checks if a string contains only digits.
1744func isNumeric(s string) bool {
1745 for _, c := range s {
1746 if c < '0' || c > '9' {
1747 return false
1748 }
1749 }
1750 return true
1751}
1752
1753func normalizePullPolicy(policy string) string {
1754 switch strings.ToLower(strings.TrimSpace(policy)) {
1755 case PullPolicyAlways:
1756 return PullPolicyAlways
1757 case PullPolicyIfTagChanged:
1758 return PullPolicyIfTagChanged
1759 case PullPolicyIfNotPresent:
1760 return PullPolicyIfNotPresent
1761 default:
1762 return PullPolicyIfNotPresent
1763 }
1764}
1765
1766func isDigestRef(imageRef string) bool {
1767 return strings.Contains(imageRef, "@sha256:")
1768}
1769
1770func (s *Service) pullRefForDeploy(ctx context.Context, imageRef string) (string, bool) {
1771 if !domain.IsInternalDeploy(ctx) {
1772 return imageRef, false
1773 }
1774 s.mu.RLock()
1775 cfg := s.config
1776 s.mu.RUnlock()
1777 return rewriteToLocalRegistry(imageRef, cfg.RegistryDomain, cfg.RegistryPort), true
1778}
1779
1780// ensureImage ensures the image is available locally, pulling if needed.
1781// Returns the image reference to use for container operations:
1782// - For digest references (@sha256:...), returns the pullRef since Docker can't tag digests
1783// - For tagged images, returns the original imageRef after tagging the pulled image
1784func (s *Service) ensureImage(ctx context.Context, imageRef string) (string, error) {
1785 ctx, span := tracer.Start(ctx, "container.ensure_image",
1786 trace.WithAttributes(attribute.String("image", imageRef)))
1787 defer span.End()
1788
1789 ctx = zerowrap.CtxWithField(ctx, "image", imageRef)
1790 log := zerowrap.FromCtx(ctx)
1791
1792 // Determine if this is an internal deploy and what reference to use for pulls.
1793 pullRef, isInternal := s.pullRefForDeploy(ctx, imageRef)
1794 if pullRef != imageRef {
1795 log.Info().
1796 Str("original_ref", imageRef).
1797 Str("pull_ref", pullRef).
1798 Msg("internal deploy: using localhost registry for pull")
1799 }
1800
1801 found, err := s.ensureLocalImage(ctx, imageRef, pullRef)
1802 if err != nil {
1803 return "", err
1804 }
1805 if found {
1806 return imageRef, nil
1807 }
1808
1809 // Pull image
1810 log.Info().Msg("pulling image from registry")
1811
1812 if err := s.pullImage(ctx, pullRef, isInternal); err != nil {
1813 return "", err
1814 }
1815
1816 // For digest references, we can't create a tag (Docker doesn't allow it).
1817 // In this case, use the pullRef directly since the image is available under that reference.
1818 if strings.Contains(imageRef, "@sha256:") {
1819 log.Info().
1820 Str("pull_ref", pullRef).
1821 Msg("digest reference: using pull reference for container operations")
1822 return pullRef, nil
1823 }
1824
1825 if err := s.tagImageIfNeeded(ctx, pullRef, imageRef); err != nil {
1826 return "", err
1827 }
1828
1829 // Clean up the temporary pull reference tag to avoid duplicate entries
1830 if pullRef != imageRef {
1831 if err := s.runtime.UntagImage(ctx, pullRef); err != nil {
1832 // Log but don't fail - the canonical tag is already applied
1833 log.Debug().Err(err).Str("pull_ref", pullRef).Msg("failed to remove temporary pull tag")
1834 }
1835 }
1836
1837 log.Info().Msg("image pulled successfully")
1838 return imageRef, nil
1839}
1840
1841func (s *Service) ensureLocalImage(ctx context.Context, imageRef, pullRef string) (bool, error) {
1842 log := zerowrap.FromCtx(ctx)
1843
1844 // For internal deploys (image push events), always pull fresh image
1845 // because the same tag may reference new content (e.g., latest tag updated)
1846 if domain.IsInternalDeploy(ctx) {
1847 log.Info().Msg("internal deploy detected, forcing image pull to ensure latest content")
1848 return false, nil
1849 }
1850
1851 s.mu.RLock()
1852 cfg := s.config
1853 s.mu.RUnlock()
1854
1855 pullPolicy := normalizePullPolicy(cfg.PullPolicy)
1856 switch pullPolicy {
1857 case PullPolicyAlways:
1858 log.Info().Str("pull_policy", pullPolicy).Msg("pull policy forces image pull")
1859 return false, nil
1860 case PullPolicyIfTagChanged:
1861 if !isDigestRef(imageRef) {
1862 log.Info().Str("pull_policy", pullPolicy).Msg("tag reference detected, pulling to check for updates")
1863 return false, nil
1864 }
1865 }
1866
1867 localImages, err := s.runtime.ListImages(ctx)
1868 if err != nil {
1869 log.WrapErr(err, "failed to list local images, will attempt pull")
1870 return false, nil
1871 }
1872
1873 normalizedRef := normalizeImageRef(imageRef)
1874 normalizedPullRef := normalizeImageRef(pullRef)
1875 for _, img := range localImages {
1876 normalizedImage := normalizeImageRef(img)
1877 if normalizedImage == normalizedRef {
1878 log.Info().Msg("image found locally, skipping pull")
1879 return true, nil
1880 }
1881 if normalizedImage == normalizedPullRef {
1882 if err := s.tagImageIfNeeded(ctx, pullRef, imageRef); err != nil {
1883 return false, err
1884 }
1885 // Clean up the temporary pull reference tag
1886 if pullRef != imageRef {
1887 if err := s.runtime.UntagImage(ctx, pullRef); err != nil {
1888 log.Debug().Err(err).Str("pull_ref", pullRef).Msg("failed to remove temporary pull tag")
1889 }
1890 }
1891 log.Info().Msg("image found locally, skipping pull")
1892 return true, nil
1893 }
1894 }
1895
1896 return false, nil
1897}
1898
1899func (s *Service) pullImage(ctx context.Context, pullRef string, isInternal bool) error {
1900 s.mu.RLock()
1901 cfg := s.config
1902 s.mu.RUnlock()
1903
1904 log := zerowrap.FromCtx(ctx)
1905
1906 switch {
1907 case isInternal && cfg.RegistryAuthEnabled:
1908 if cfg.InternalRegistryUsername == "" || cfg.InternalRegistryPassword == "" {
1909 return log.WrapErr(fmt.Errorf("internal registry auth not configured"), "failed to pull image for internal deploy")
1910 }
1911 if err := s.pullImageWithRetry(ctx, func(ctx context.Context) error {
1912 return s.runtime.PullImageWithAuth(ctx, pullRef, cfg.InternalRegistryUsername, cfg.InternalRegistryPassword)
1913 }, isInternal); err != nil {
1914 return log.WrapErr(err, "failed to pull image with internal auth")
1915 }
1916 case isInternal:
1917 if err := s.pullImageWithRetry(ctx, func(ctx context.Context) error {
1918 return s.runtime.PullImage(ctx, pullRef)
1919 }, isInternal); err != nil {
1920 return log.WrapErr(err, "failed to pull image")
1921 }
1922 case cfg.RegistryAuthEnabled:
1923 if cfg.ServiceTokenUsername == "" || cfg.ServiceToken == "" {
1924 return log.WrapErr(fmt.Errorf("registry service token not configured"), "failed to pull image for registry auth")
1925 }
1926 if err := s.runtime.PullImageWithAuth(ctx, pullRef, cfg.ServiceTokenUsername, cfg.ServiceToken); err != nil {
1927 return log.WrapErr(err, "failed to pull image with auth")
1928 }
1929 default:
1930 if err := s.runtime.PullImage(ctx, pullRef); err != nil {
1931 return log.WrapErr(err, "failed to pull image")
1932 }
1933 }
1934
1935 return nil
1936}
1937
1938func (s *Service) pullImageWithRetry(ctx context.Context, pullFn func(context.Context) error, isInternal bool) error {
1939 log := zerowrap.FromCtx(ctx)
1940
1941 for attempt := 1; attempt <= internalPullMaxAttempts; attempt++ {
1942 err := pullFn(ctx)
1943 if err == nil {
1944 return nil
1945 }
1946 if !isInternal || !isConnectionRefusedError(err) || attempt == internalPullMaxAttempts {
1947 return fmt.Errorf("internal image pull failed after %d attempts: %w", attempt, err)
1948 }
1949
1950 backoff := time.Duration(attempt) * time.Second
1951 log.Warn().
1952 Err(err).
1953 Int("attempt", attempt).
1954 Dur("backoff", backoff).
1955 Msg("internal image pull failed with connection refused, retrying")
1956
1957 select {
1958 case <-time.After(backoff):
1959 case <-ctx.Done():
1960 return ctx.Err()
1961 }
1962 }
1963 return nil
1964}
1965
1966func (s *Service) tagImageIfNeeded(ctx context.Context, sourceRef, targetRef string) error {
1967 if sourceRef == targetRef {
1968 return nil
1969 }
1970
1971 // Cannot create a tag with a digest reference - Docker/Podman doesn't allow it.
1972 // When using digest references (image@sha256:...), skip tagging as the image
1973 // is already available by its digest.
1974 if strings.Contains(targetRef, "@sha256:") {
1975 return nil
1976 }
1977
1978 log := zerowrap.FromCtx(ctx)
1979 if err := s.runtime.TagImage(ctx, sourceRef, targetRef); err != nil {
1980 return log.WrapErr(err, "failed to tag image from pull reference")
1981 }
1982 return nil
1983}
1984
1985func (s *Service) loadEnvironment(ctx context.Context, preResolved []string, domainName, imageRef string) ([]string, error) {
1986 log := zerowrap.FromCtx(ctx)
1987
1988 var userEnvVars []string
1989 if len(preResolved) > 0 {
1990 userEnvVars = preResolved
1991 } else {
1992 var err error
1993 userEnvVars, err = s.envLoader.LoadEnv(ctx, domainName)
1994 if err != nil {
1995 return nil, log.WrapErr(err, "failed to load environment variables")
1996 }
1997 }
1998
1999 dockerfileEnvVars, err := s.runtime.InspectImageEnv(ctx, imageRef)
2000 if err != nil {
2001 log.WrapErr(err, "failed to inspect image environment")
2002 dockerfileEnvVars = []string{}
2003 }
2004
2005 return mergeEnvironmentVariables(dockerfileEnvVars, userEnvVars), nil
2006}
2007
2008func (s *Service) setupVolumes(ctx context.Context, domainName, imageRef string) (map[string]string, error) {
2009 s.mu.RLock()
2010 cfg := s.config
2011 s.mu.RUnlock()
2012
2013 log := zerowrap.FromCtx(ctx)
2014 volumes := make(map[string]string)
2015
2016 if !cfg.VolumeAutoCreate {
2017 return volumes, nil
2018 }
2019
2020 volumePaths, err := s.runtime.InspectImageVolumes(ctx, imageRef)
2021 if err != nil {
2022 log.WrapErr(err, "failed to inspect image volumes")
2023 return volumes, nil
2024 }
2025
2026 for _, path := range volumePaths {
2027 name := generateVolumeName(cfg.VolumePrefix, domainName, path)
2028
2029 exists, err := s.runtime.VolumeExists(ctx, name)
2030 if err != nil {
2031 log.WrapErrWithFields(err, "failed to check volume", map[string]any{"volume": name})
2032 continue
2033 }
2034
2035 if !exists {
2036 if err := s.runtime.CreateVolume(ctx, name); err != nil {
2037 log.WrapErrWithFields(err, "failed to create volume", map[string]any{"volume": name})
2038 continue
2039 }
2040 log.Info().Str("volume", name).Str(zerowrap.FieldPath, path).Msg("created volume")
2041 }
2042
2043 volumes[path] = name
2044 }
2045
2046 return volumes, nil
2047}
2048
2049func (s *Service) getNetworkForApp(domainName string) string {
2050 var networkIsolation bool
2051 var networkGroups map[string][]string
2052
2053 if s.configProvider != nil {
2054 snap := s.configProvider.GetAttachmentConfig()
2055 networkGroups = snap.NetworkGroups
2056 s.mu.RLock()
2057 networkIsolation = s.config.NetworkIsolation
2058 s.mu.RUnlock()
2059 } else {
2060 s.mu.RLock()
2061 networkIsolation = s.config.NetworkIsolation
2062 networkGroups = s.config.NetworkGroups
2063 s.mu.RUnlock()
2064 }
2065
2066 if !networkIsolation {
2067 return "bridge"
2068 }
2069
2070 for groupName, domains := range networkGroups {
2071 if slices.Contains(domains, domainName) {
2072 return s.generateNetworkName(groupName)
2073 }
2074 }
2075
2076 return s.generateNetworkName(domainName)
2077}
2078
2079func (s *Service) generateNetworkName(identifier string) string {
2080 s.mu.RLock()
2081 cfg := s.config
2082 s.mu.RUnlock()
2083
2084 return fmt.Sprintf("%s-%s", cfg.NetworkPrefix, strings.ReplaceAll(identifier, ".", "-"))
2085}
2086
2087func (s *Service) createNetworkIfNeeded(ctx context.Context, networkName string) error {
2088 if networkName == "bridge" || networkName == "default" {
2089 return nil
2090 }
2091
2092 ctx = zerowrap.CtxWithField(ctx, "network", networkName)
2093 log := zerowrap.FromCtx(ctx)
2094
2095 exists, err := s.runtime.NetworkExists(ctx, networkName)
2096 if err != nil {
2097 return log.WrapErr(err, "failed to check network existence")
2098 }
2099
2100 if !exists {
2101 if err := s.runtime.CreateNetwork(ctx, networkName, map[string]string{"driver": "bridge"}); err != nil {
2102 return log.WrapErr(err, "failed to create network")
2103 }
2104 log.Info().Msg("created network for app isolation")
2105 }
2106
2107 return nil
2108}
2109
2110func (s *Service) cleanupOrphanedContainers(ctx context.Context, domainName string, skipContainerID string) error {
2111 log := zerowrap.FromCtx(ctx)
2112 expectedName := fmt.Sprintf("gordon-%s", domainName)
2113 expectedNewName := expectedName + "-new"
2114 expectedNextName := expectedName + "-next"
2115
2116 allContainers, err := s.runtime.ListContainers(ctx, true)
2117 if err != nil {
2118 return err
2119 }
2120
2121 for _, c := range allContainers {
2122 if (c.Name == expectedName || c.Name == expectedNewName || c.Name == expectedNextName) && c.ID != skipContainerID {
2123 // Skip any running container regardless of name — a running
2124 // temp container may be actively serving traffic while the
2125 // new container stabilizes, or may be from a concurrent deploy.
2126 if c.Status == "running" {
2127 log.Debug().
2128 Str(zerowrap.FieldEntityID, c.ID).
2129 Str("container_name", c.Name).
2130 Str(zerowrap.FieldStatus, c.Status).
2131 Msg("skipping running container during orphan cleanup")
2132 continue
2133 }
2134
2135 log.Info().Str(zerowrap.FieldEntityID, c.ID).Str("container_name", c.Name).Str(zerowrap.FieldStatus, c.Status).Msg("found orphaned container, removing")
2136
2137 if err := s.runtime.StopContainer(ctx, c.ID); err != nil {
2138 log.WrapErrWithFields(err, "failed to stop orphaned container", map[string]any{zerowrap.FieldEntityID: c.ID})
2139 }
2140
2141 if err := s.runtime.RemoveContainer(ctx, c.ID, true); err != nil {
2142 log.WrapErrWithFields(err, "failed to remove orphaned container", map[string]any{zerowrap.FieldEntityID: c.ID})
2143 }
2144 }
2145 }
2146
2147 return nil
2148}
2149
2150func (s *Service) cleanupVolumesForDomain(_ context.Context, _ string) error {
2151 return nil
2152}
2153
2154func (s *Service) cleanupNetworkIfEmpty(ctx context.Context, networkName string) error {
2155 if networkName == "bridge" || networkName == "default" {
2156 return nil
2157 }
2158
2159 ctx = zerowrap.CtxWithField(ctx, "network", networkName)
2160 log := zerowrap.FromCtx(ctx)
2161
2162 networks, err := s.runtime.ListNetworks(ctx)
2163 if err != nil {
2164 return log.WrapErr(err, "failed to list networks")
2165 }
2166
2167 for _, n := range networks {
2168 if n.Name == networkName && len(n.Containers) == 0 {
2169 if err := s.runtime.RemoveNetwork(ctx, networkName); err != nil {
2170 return err
2171 }
2172 log.Info().Msg("cleaned up empty network")
2173 break
2174 }
2175 }
2176
2177 return nil
2178}
2179
2180func (s *Service) deployAttachments(ctx context.Context, domainName, networkName string) error {
2181 // Collect attachments from both domain-specific and network group configs
2182 attachments := s.resolveAttachmentsForDomain(domainName)
2183 if len(attachments) == 0 {
2184 return nil
2185 }
2186
2187 log := zerowrap.FromCtx(ctx)
2188 var deployed []string // track successfully deployed attachment IDs for rollback
2189 for _, svc := range attachments {
2190 if err := s.deployAttachedService(ctx, domainName, svc, networkName); err != nil {
2191 log.WrapErrWithFields(err, "failed to deploy attachment", map[string]any{zerowrap.FieldService: svc, "domain": domainName})
2192
2193 // Rollback: clean up already-deployed attachments
2194 for _, id := range deployed {
2195 if stopErr := s.runtime.StopContainer(ctx, id); stopErr != nil {
2196 log.Warn().Err(stopErr).Str("attachment_id", id).Msg("rollback: failed to stop attachment")
2197 }
2198 if rmErr := s.runtime.RemoveContainer(ctx, id, true); rmErr != nil {
2199 log.Warn().Err(rmErr).Str("attachment_id", id).Msg("rollback: failed to remove attachment")
2200 }
2201 }
2202 // Deregister rolled-back attachments
2203 if len(deployed) > 0 {
2204 s.mu.Lock()
2205 remaining := s.attachments[domainName]
2206 filtered := remaining[:0]
2207 rollbackSet := make(map[string]bool, len(deployed))
2208 for _, id := range deployed {
2209 rollbackSet[id] = true
2210 }
2211 for _, id := range remaining {
2212 if !rollbackSet[id] {
2213 filtered = append(filtered, id)
2214 }
2215 }
2216 s.attachments[domainName] = filtered
2217 s.mu.Unlock()
2218 }
2219
2220 return fmt.Errorf("failed to deploy attachment %q (rolled back %d already-deployed)", svc, len(deployed))
2221 }
2222 // Collect IDs of attachments tracked under this domain after successful deploy
2223 s.mu.RLock()
2224 ids := s.attachments[domainName]
2225 if len(ids) > 0 {
2226 latest := ids[len(ids)-1]
2227 deployed = append(deployed, latest)
2228 }
2229 s.mu.RUnlock()
2230 }
2231 return nil
2232}
2233
2234// resolveAttachmentsForDomain returns attachments for a domain by checking:
2235// 1. Direct domain attachments (attachments[domain])
2236// 2. Network group attachments (attachments[group] where domain is in network_groups[group])
2237func (s *Service) resolveAttachmentsForDomain(domainName string) []string {
2238 var attachments map[string][]string
2239 var networkGroups map[string][]string
2240
2241 if s.configProvider != nil {
2242 snap := s.configProvider.GetAttachmentConfig()
2243 attachments = snap.Attachments
2244 networkGroups = snap.NetworkGroups
2245 } else {
2246 s.mu.RLock()
2247 attachments = s.config.Attachments
2248 networkGroups = s.config.NetworkGroups
2249 s.mu.RUnlock()
2250 }
2251
2252 seen := make(map[string]bool)
2253 var result []string
2254
2255 // First, add domain-specific attachments
2256 if domainAttachments, ok := attachments[domainName]; ok {
2257 for _, img := range domainAttachments {
2258 if !seen[img] {
2259 seen[img] = true
2260 result = append(result, img)
2261 }
2262 }
2263 }
2264
2265 // Then, find which network group this domain belongs to and add group attachments
2266 for groupName, domains := range networkGroups {
2267 if slices.Contains(domains, domainName) {
2268 if groupAttachments, ok := attachments[groupName]; ok {
2269 for _, img := range groupAttachments {
2270 if !seen[img] {
2271 seen[img] = true
2272 result = append(result, img)
2273 }
2274 }
2275 }
2276 break // Domain can only be in one network group
2277 }
2278 }
2279
2280 return result
2281}
2282
2283func (s *Service) getAttachmentsForDomain(ctx context.Context, domainName string) []domain.Attachment {
2284 containers, err := s.runtime.ListContainers(ctx, true)
2285 if err != nil {
2286 return nil
2287 }
2288
2289 return s.filterAttachments(ctx, containers, domainName)
2290}
2291
2292// getAllAttachments fetches all containers once and returns attachments grouped by domain.
2293// This avoids N+1 queries when listing multiple routes.
2294func (s *Service) getAllAttachments(ctx context.Context) map[string][]domain.Attachment {
2295 containers, err := s.runtime.ListContainers(ctx, true)
2296 if err != nil {
2297 return nil
2298 }
2299
2300 result := make(map[string][]domain.Attachment)
2301 for _, container := range containers {
2302 if container.Labels == nil {
2303 continue
2304 }
2305 if container.Labels[domain.LabelAttachment] != "true" {
2306 continue
2307 }
2308 ownerDomain := container.Labels[domain.LabelAttachedTo]
2309 if ownerDomain == "" {
2310 continue
2311 }
2312 image := container.Image
2313 serviceName := container.Name
2314 if labelImage, ok := container.Labels[domain.LabelImage]; ok && labelImage != "" {
2315 image = labelImage
2316 serviceName = extractServiceName(labelImage)
2317 }
2318 // Get the container's network for display in the routes table
2319 network := ""
2320 if networkName, err := s.runtime.GetContainerNetwork(ctx, container.ID); err == nil {
2321 network = networkName
2322 }
2323 attachment := domain.Attachment{
2324 Name: serviceName,
2325 Image: s.stripRegistryPrefix(image),
2326 ContainerID: container.ID,
2327 Status: container.Status,
2328 Network: network,
2329 Ports: append([]int(nil), container.Ports...),
2330 }
2331 result[ownerDomain] = append(result[ownerDomain], attachment)
2332 }
2333
2334 return result
2335}
2336
2337// filterAttachments extracts attachments for a specific domain from a container list.
2338func (s *Service) filterAttachments(ctx context.Context, containers []*domain.Container, domainName string) []domain.Attachment {
2339 attachments := make([]domain.Attachment, 0)
2340 for _, container := range containers {
2341 if container.Labels == nil {
2342 continue
2343 }
2344 if container.Labels[domain.LabelAttachment] != "true" {
2345 continue
2346 }
2347 if container.Labels[domain.LabelAttachedTo] != domainName {
2348 continue
2349 }
2350 image := container.Image
2351 serviceName := container.Name
2352 if labelImage, ok := container.Labels[domain.LabelImage]; ok && labelImage != "" {
2353 image = labelImage
2354 serviceName = extractServiceName(labelImage)
2355 }
2356 // Get the container's network for display in the routes table
2357 network := ""
2358 if networkName, err := s.runtime.GetContainerNetwork(ctx, container.ID); err == nil {
2359 network = networkName
2360 }
2361 attachment := domain.Attachment{
2362 Name: serviceName,
2363 Image: s.stripRegistryPrefix(image),
2364 ContainerID: container.ID,
2365 Status: container.Status,
2366 Network: network,
2367 Ports: append([]int(nil), container.Ports...),
2368 }
2369 attachments = append(attachments, attachment)
2370 }
2371
2372 return attachments
2373}
2374
2375func (s *Service) deployAttachedService(ctx context.Context, ownerDomain, serviceImage, networkName string) error {
2376 log := zerowrap.FromCtx(ctx)
2377
2378 // Parse service name from image (e.g., "my-postgres:latest" → "postgres")
2379 serviceName := extractServiceName(serviceImage)
2380 containerName := fmt.Sprintf("gordon-%s-%s", sanitizeName(ownerDomain), serviceName)
2381
2382 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
2383 "attachment": serviceName,
2384 "container_name": containerName,
2385 "owner_domain": ownerDomain,
2386 })
2387
2388 // Check if already running (idempotent)
2389 // Try new name first, then fall back to legacy name for backwards compatibility
2390 existingContainer, err := s.resolveExistingAttachment(ctx, ownerDomain, serviceName)
2391 if err != nil {
2392 return err
2393 }
2394
2395 if existingContainer != nil {
2396 shouldSkip, err := s.handleRunningAttachment(ctx, existingContainer, containerName, serviceImage)
2397 if err != nil {
2398 return err
2399 }
2400 if shouldSkip {
2401 return nil
2402 }
2403 if existingContainer.Status == string(domain.ContainerStatusRunning) {
2404 existingContainer = nil
2405 }
2406 }
2407
2408 if err := s.removeStoppedAttachment(ctx, existingContainer, containerName); err != nil {
2409 return err
2410 }
2411
2412 log.Info().Str(zerowrap.FieldService, serviceImage).Msg("deploying attached service")
2413
2414 // Ensure image (canonical ref is returned; internal pulls may use the local registry).
2415 imageRef := s.buildImageRef(serviceImage)
2416 actualImageRef, err := s.ensureImage(ctx, imageRef)
2417 if err != nil {
2418 return err
2419 }
2420
2421 // Get image metadata
2422 exposedPorts, err := s.runtime.GetImageExposedPorts(ctx, actualImageRef)
2423 if err != nil {
2424 log.WrapErr(err, "failed to get exposed ports for attachment, using defaults")
2425 exposedPorts = []int{}
2426 }
2427
2428 // Setup volumes (attachments need persistent data)
2429 volumes, err := s.setupVolumes(ctx, containerName, actualImageRef)
2430 if err != nil {
2431 log.WrapErr(err, "failed to setup volumes for attachment")
2432 volumes = make(map[string]string)
2433 }
2434
2435 // Load environment (attachment-specific env file)
2436 envVars, err := s.loadEnvironment(ctx, nil, containerName, actualImageRef)
2437 if err != nil {
2438 log.WrapErr(err, "failed to load environment for attachment")
2439 envVars = []string{}
2440 }
2441 envHash := hashEnvironment(envVars)
2442
2443 s.mu.RLock()
2444 cfg := s.config
2445 s.mu.RUnlock()
2446
2447 // Create container with attachment labels (use actual ref, track original in labels)
2448 config := &domain.ContainerConfig{
2449 Image: actualImageRef,
2450 Name: containerName,
2451 Hostname: serviceName, // Internal DNS: postgres, redis, etc.
2452 Aliases: []string{serviceName},
2453 Ports: exposedPorts,
2454 Env: envVars,
2455 Volumes: volumes,
2456 NetworkMode: networkName, // Same network as main app
2457 Labels: map[string]string{
2458 domain.LabelManaged: "true",
2459 domain.LabelAttachment: "true",
2460 domain.LabelAttachedTo: ownerDomain,
2461 domain.LabelEnvHash: envHash,
2462 domain.LabelImage: serviceImage,
2463 },
2464 MemoryLimit: cfg.DefaultMemoryLimit,
2465 NanoCPUs: cfg.DefaultNanoCPUs,
2466 PidsLimit: cfg.DefaultPidsLimit,
2467 }
2468
2469 container, err := s.runtime.CreateContainer(ctx, config)
2470 if err != nil {
2471 return log.WrapErr(err, "failed to create attachment container")
2472 }
2473
2474 // Start container
2475 if err := s.runtime.StartContainer(ctx, container.ID); err != nil {
2476 s.runtime.RemoveContainer(ctx, container.ID, true)
2477 return log.WrapErr(err, "failed to start attachment container")
2478 }
2479
2480 // Wait for attachment to be ready before proceeding
2481 if err := s.waitForAttachmentReady(ctx, container.ID, config); err != nil {
2482 log.WrapErr(err, "attachment readiness check failed, cleaning up")
2483 if stopErr := s.runtime.StopContainer(ctx, container.ID); stopErr != nil {
2484 log.Warn().Err(stopErr).Str(zerowrap.FieldEntityID, container.ID).Msg("failed to stop unready attachment")
2485 }
2486 s.runtime.RemoveContainer(ctx, container.ID, true)
2487 return fmt.Errorf("attachment %q not ready: %w", serviceName, err)
2488 }
2489
2490 // Track attachment
2491 s.mu.Lock()
2492 s.attachments[ownerDomain] = append(s.attachments[ownerDomain], container.ID)
2493 s.mu.Unlock()
2494
2495 // Start log collection for attachment
2496 s.startLogCollection(ctx, container.ID, containerName)
2497
2498 log.Info().Str(zerowrap.FieldEntityID, container.ID).Msg("attachment deployed successfully")
2499 return nil
2500}
2501
2502func (s *Service) attachmentEnvDrifted(ctx context.Context, existing *domain.Container, containerName, serviceImage string) (bool, error) {
2503 currentEnv, err := s.loadEnvironment(ctx, nil, containerName, s.buildImageRef(serviceImage))
2504 if err != nil {
2505 return false, err
2506 }
2507
2508 currentHash := hashEnvironment(currentEnv)
2509 existingHash, ok := existing.Labels[domain.LabelEnvHash]
2510 if ok && existingHash == currentHash {
2511 return false, nil
2512 }
2513
2514 return true, nil
2515}
2516
2517func (s *Service) handleRunningAttachment(ctx context.Context, existing *domain.Container, containerName, serviceImage string) (bool, error) {
2518 if existing.Status != string(domain.ContainerStatusRunning) {
2519 return false, nil
2520 }
2521
2522 log := zerowrap.FromCtx(ctx)
2523
2524 // Check image drift
2525 existingImage := existing.Labels[domain.LabelImage]
2526 imageDrifted := existingImage != "" && existingImage != serviceImage
2527
2528 // Check env drift
2529 envDrifted, err := s.attachmentEnvDrifted(ctx, existing, containerName, serviceImage)
2530 if err != nil {
2531 return false, log.WrapErr(err, "failed to check attachment env drift")
2532 }
2533
2534 if !envDrifted && !imageDrifted {
2535 log.Debug().Str("container_name", containerName).Msg("attachment already running with current env and image, skipping")
2536 return true, nil
2537 }
2538
2539 if imageDrifted {
2540 log.Info().Str("container_name", containerName).Str("existing_image", existingImage).Str("desired_image", serviceImage).Msg("attachment image changed, recreating")
2541 }
2542 if envDrifted {
2543 log.Info().Str("container_name", containerName).Msg("attachment env changed, recreating")
2544 }
2545
2546 if err := s.runtime.StopContainer(ctx, existing.ID); err != nil {
2547 return false, log.WrapErr(err, "failed to stop attachment for drift update")
2548 }
2549 if err := s.runtime.RemoveContainer(ctx, existing.ID, true); err != nil {
2550 return false, log.WrapErr(err, "failed to remove attachment for drift update")
2551 }
2552
2553 return false, nil
2554}
2555
2556func (s *Service) removeStoppedAttachment(ctx context.Context, existing *domain.Container, containerName string) error {
2557 if existing == nil {
2558 return nil
2559 }
2560
2561 log := zerowrap.FromCtx(ctx)
2562 log.Info().Str("container_name", containerName).Msg("removing stopped attachment container")
2563 if err := s.runtime.RemoveContainer(ctx, existing.ID, true); err != nil {
2564 return log.WrapErr(err, "failed to remove existing attachment container")
2565 }
2566
2567 return nil
2568}
2569
2570func (s *Service) resolveExistingAttachment(ctx context.Context, ownerDomain, serviceName string) (*domain.Container, error) {
2571 log := zerowrap.FromCtx(ctx)
2572 containerName := fmt.Sprintf("gordon-%s-%s", sanitizeName(ownerDomain), serviceName)
2573 existingContainer := s.findContainerByName(ctx, containerName)
2574 if existingContainer != nil {
2575 return existingContainer, nil
2576 }
2577
2578 containerNameLegacy := fmt.Sprintf("gordon-%s-%s", sanitizeNameLegacy(ownerDomain), serviceName)
2579 existingContainer = s.findContainerByName(ctx, containerNameLegacy)
2580 if existingContainer == nil {
2581 return nil, nil
2582 }
2583
2584 log.Info().Str("container_name", containerNameLegacy).Msg("found attachment with legacy naming, will be replaced")
2585 if err := s.runtime.StopContainer(ctx, existingContainer.ID); err != nil {
2586 return nil, log.WrapErr(err, "failed to stop legacy attachment container")
2587 }
2588 if err := s.runtime.RemoveContainer(ctx, existingContainer.ID, true); err != nil {
2589 return nil, log.WrapErr(err, "failed to remove legacy attachment container")
2590 }
2591
2592 return nil, nil
2593}
2594
2595// Utility functions
2596
2597func normalizeImageRef(image string) string {
2598 // Extract repository from image reference, handling port numbers correctly.
2599 // Examples:
2600 // nginx:latest -> docker.io/library/nginx
2601 // user/repo:tag -> docker.io/user/repo
2602 // localhost:5000/image:tag -> localhost:5000/image
2603 // registry.example.com/image:tag -> registry.example.com/image
2604 //
2605 // Note: This function assumes valid Docker image references. Edge cases like
2606 // "registry.com:5000" (registry with port but no image name) are not valid
2607 // image references per Docker's naming conventions, which require at least
2608 // one path component after the registry (e.g., "registry.com:5000/image").
2609
2610 // Find the tag separator: the last colon that isn't part of a port number.
2611 // A colon is part of a port if there's no slash after it until the next colon.
2612 repo := image
2613 lastColon := strings.LastIndex(image, ":")
2614 if lastColon != -1 {
2615 afterColon := image[lastColon+1:]
2616 // If there's no slash after the colon, it's the tag separator
2617 if !strings.Contains(afterColon, "/") {
2618 repo = image[:lastColon]
2619 }
2620 }
2621
2622 if !strings.Contains(repo, "/") {
2623 return "docker.io/library/" + repo
2624 }
2625
2626 if strings.Count(repo, "/") == 1 && !strings.Contains(strings.Split(repo, "/")[0], ".") && !strings.Contains(strings.Split(repo, "/")[0], ":") {
2627 return "docker.io/" + repo
2628 }
2629
2630 return repo
2631}
2632
2633func generateVolumeName(prefix, domainName, volumePath string) string {
2634 return fmt.Sprintf("%s-%s-%s",
2635 prefix,
2636 strings.ReplaceAll(domainName, ".", "-"),
2637 strings.ReplaceAll(strings.Trim(volumePath, "/"), "/", "-"))
2638}
2639
2640func mergeEnvironmentVariables(dockerfileEnv, userEnv []string) []string {
2641 envMap := make(map[string]string)
2642
2643 for _, env := range dockerfileEnv {
2644 if k, v, ok := strings.Cut(env, "="); ok {
2645 envMap[k] = v
2646 }
2647 }
2648
2649 for _, env := range userEnv {
2650 if k, v, ok := strings.Cut(env, "="); ok {
2651 envMap[k] = v
2652 }
2653 }
2654
2655 keys := make([]string, 0, len(envMap))
2656 for k := range envMap {
2657 keys = append(keys, k)
2658 }
2659 slices.Sort(keys)
2660
2661 result := make([]string, 0, len(envMap))
2662 for _, k := range keys {
2663 result = append(result, k+"="+envMap[k])
2664 }
2665
2666 return result
2667}
2668
2669// extractServiceName gets service name from image reference.
2670// "my-postgres:latest" → "postgres", "redis:7" → "redis"
2671func extractServiceName(image string) string {
2672 // Remove tag
2673 parts := strings.Split(image, ":")
2674 name := parts[0]
2675
2676 // Remove registry prefix if present
2677 if strings.Contains(name, "/") {
2678 nameParts := strings.Split(name, "/")
2679 name = nameParts[len(nameParts)-1]
2680 }
2681
2682 // Remove common prefixes like "my-"
2683 name = strings.TrimPrefix(name, "my-")
2684
2685 return name
2686}
2687
2688// sanitizeName is a convenience wrapper around domain.SanitizeDomainForContainer.
2689func sanitizeName(d string) string {
2690 return domain.SanitizeDomainForContainer(d)
2691}
2692
2693// sanitizeNameLegacy is a convenience wrapper around domain.SanitizeDomainForContainerLegacy.
2694func sanitizeNameLegacy(d string) string {
2695 return domain.SanitizeDomainForContainerLegacy(d)
2696}
2697
2698func isContainerNotFoundError(err error) bool {
2699 if err == nil {
2700 return false
2701 }
2702 if errors.Is(err, domain.ErrContainerNotFound) {
2703 return true
2704 }
2705 msg := strings.ToLower(err.Error())
2706 return strings.Contains(msg, "no such container") ||
2707 strings.Contains(msg, "no container with name or id") ||
2708 strings.Contains(msg, "container not found")
2709}
2710
2711func isConnectionRefusedError(err error) bool {
2712 if err == nil {
2713 return false
2714 }
2715 return strings.Contains(strings.ToLower(err.Error()), "connection refused")
2716}
2717
2718// findContainerByName finds a container by its name.
2719func (s *Service) findContainerByName(ctx context.Context, name string) *domain.Container {
2720 containers, err := s.runtime.ListContainers(ctx, true)
2721 if err != nil {
2722 return nil
2723 }
2724
2725 for _, c := range containers {
2726 if c.Name == name {
2727 return c
2728 }
2729 }
2730 return nil
2731}
2732
2733func (s *Service) waitForReady(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig) error {
2734 if err := s.pollContainerRunning(ctx, containerID); err != nil {
2735 return err
2736 }
2737
2738 s.mu.RLock()
2739 cfg := s.config
2740 s.mu.RUnlock()
2741
2742 readinessMode := cfg.ReadinessMode
2743 if readinessMode == "" {
2744 readinessMode = "delay"
2745 }
2746
2747 switch readinessMode {
2748 case "delay":
2749 return s.waitForReadyByDelay(ctx, containerID)
2750 case "docker-health":
2751 _, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(ctx, containerID)
2752 if err != nil {
2753 return err
2754 }
2755 if !hasHealthcheck {
2756 return errors.New("no healthcheck detected")
2757 }
2758 return s.waitForHealthy(ctx, containerID, cfg.HealthTimeout)
2759 default: // auto
2760 return s.readinessCascade(ctx, containerID, containerConfig, cfg)
2761 }
2762}
2763
2764// readinessCascade auto-detects the strongest available readiness signal:
2765// 1. Docker healthcheck (if present) → wait for healthy status
2766// 2. HTTP probe (if gordon.health label set) → GET until 2xx/3xx
2767// 3. Default HTTP probe (GET / on exposed port) → wait for 2xx/3xx
2768// 4. TCP probe (if port info available) → connect until accepted
2769// 5. Delay fallback (last resort) → waitForReadyByDelay
2770func (s *Service) readinessCascade(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) error {
2771 log := zerowrap.FromCtx(ctx)
2772
2773 // 1. Docker healthcheck
2774 _, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(ctx, containerID)
2775 if err != nil {
2776 return err
2777 }
2778 if hasHealthcheck {
2779 log.Info().Msg("readiness cascade: using Docker healthcheck")
2780 return s.waitForHealthy(ctx, containerID, cfg.HealthTimeout)
2781 }
2782
2783 // 2. HTTP probe via gordon.health label
2784 if probed, probeErr := s.tryHTTPProbe(ctx, containerID, containerConfig, cfg); probed {
2785 return probeErr
2786 }
2787
2788 // 3. Default HTTP probe on root path (if port info available)
2789 if probed, probeErr := s.tryDefaultHTTPProbe(ctx, containerID, containerConfig, cfg); probed {
2790 return probeErr
2791 }
2792
2793 // 4. TCP probe (if port info available)
2794 if probed, probeErr := s.tryTCPProbe(ctx, containerID, containerConfig, cfg); probed {
2795 return probeErr
2796 }
2797
2798 // 5. Delay fallback
2799 log.Info().Msg("readiness cascade: using delay fallback")
2800 return s.waitForReadyByDelay(ctx, containerID)
2801}
2802
2803// waitForAttachmentReady runs a reduced readiness cascade for attachments:
2804// Docker healthcheck → TCP probe → delay fallback.
2805// HTTP probes are skipped — attachments are typically databases/caches.
2806func (s *Service) waitForAttachmentReady(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig) error {
2807 if err := s.pollContainerRunning(ctx, containerID); err != nil {
2808 return err
2809 }
2810
2811 log := zerowrap.FromCtx(ctx)
2812
2813 s.mu.RLock()
2814 cfg := s.config
2815 s.mu.RUnlock()
2816
2817 // 1. Docker healthcheck (if present)
2818 _, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(ctx, containerID)
2819 if err != nil {
2820 return err
2821 }
2822 if hasHealthcheck {
2823 log.Info().Msg("attachment readiness: using Docker healthcheck")
2824 timeout := cfg.HealthTimeout
2825 if timeout == 0 {
2826 timeout = 90 * time.Second
2827 }
2828 return s.waitForHealthy(ctx, containerID, timeout)
2829 }
2830
2831 // 2. TCP probe (if port info available)
2832 if containerConfig != nil && len(containerConfig.Ports) > 0 {
2833 ip, port, probeErr := s.resolveProbeEndpoint(ctx, containerID, containerConfig)
2834 if probeErr == nil && ip != "" && port > 0 {
2835 addr := fmt.Sprintf("%s:%d", ip, port)
2836 timeout := cfg.AttachmentReadinessTimeout
2837 if timeout == 0 {
2838 timeout = 30 * time.Second
2839 }
2840 log.Info().Str("addr", addr).Dur("timeout", timeout).Msg("attachment readiness: using TCP probe")
2841 return tcpProbe(ctx, addr, timeout)
2842 }
2843 log.Debug().Err(probeErr).Msg("attachment readiness: TCP probe skipped, could not resolve endpoint")
2844 }
2845
2846 // 3. Delay fallback
2847 log.Info().Msg("attachment readiness: using delay fallback")
2848 return s.waitForReadyByDelay(ctx, containerID)
2849}
2850
2851// tryHTTPProbe attempts an HTTP probe if the gordon.health label is set.
2852// Returns (true, err) if the probe was attempted, (false, nil) if skipped.
2853func (s *Service) tryHTTPProbe(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) (bool, error) {
2854 log := zerowrap.FromCtx(ctx)
2855 if containerConfig == nil {
2856 return false, nil
2857 }
2858 healthPath, ok := containerConfig.Labels[domain.LabelHealth]
2859 if !ok || healthPath == "" {
2860 return false, nil
2861 }
2862 ip, port, probeErr := s.resolveProbeEndpoint(ctx, containerID, containerConfig)
2863 if probeErr != nil || ip == "" || port <= 0 {
2864 log.Debug().Err(probeErr).Msg("readiness cascade: HTTP probe skipped, could not resolve container endpoint")
2865 return false, nil
2866 }
2867 url := fmt.Sprintf("http://%s:%d%s", ip, port, healthPath)
2868 timeout := cfg.HTTPProbeTimeout
2869 if timeout == 0 {
2870 timeout = 60 * time.Second
2871 }
2872 log.Info().Str("url", url).Dur("timeout", timeout).Msg("readiness cascade: using HTTP probe")
2873 return true, httpProbe(ctx, url, timeout)
2874}
2875
2876// tryDefaultHTTPProbe attempts an HTTP GET on "/" using the container's
2877// exposed port. This is the default readiness check for web containers —
2878// it verifies the app is actually serving HTTP, not just accepting TCP.
2879// Returns (true, err) if the probe was attempted, (false, nil) if skipped.
2880func (s *Service) tryDefaultHTTPProbe(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) (bool, error) {
2881 log := zerowrap.FromCtx(ctx)
2882 if containerConfig == nil || len(containerConfig.Ports) == 0 {
2883 return false, nil
2884 }
2885 ip, port, probeErr := s.resolveProbeEndpoint(ctx, containerID, containerConfig)
2886 if probeErr != nil || ip == "" || port <= 0 {
2887 log.Debug().Err(probeErr).Msg("readiness cascade: default HTTP probe skipped, could not resolve container endpoint")
2888 return false, nil
2889 }
2890 url := fmt.Sprintf("http://%s:%d/", ip, port)
2891 timeout := cfg.HTTPProbeTimeout
2892 if timeout == 0 {
2893 timeout = 60 * time.Second
2894 }
2895 log.Info().Str("url", url).Dur("timeout", timeout).Msg("readiness cascade: using default HTTP alive probe")
2896 return true, httpAliveProbe(ctx, url, timeout)
2897}
2898
2899// tryTCPProbe attempts a TCP probe if port info is available.
2900// Returns (true, err) if the probe was attempted, (false, nil) if skipped.
2901func (s *Service) tryTCPProbe(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) (bool, error) {
2902 log := zerowrap.FromCtx(ctx)
2903 if containerConfig == nil || len(containerConfig.Ports) == 0 {
2904 return false, nil
2905 }
2906 ip, port, probeErr := s.resolveProbeEndpoint(ctx, containerID, containerConfig)
2907 if probeErr != nil || ip == "" || port <= 0 {
2908 log.Debug().Err(probeErr).Msg("readiness cascade: TCP probe skipped, could not resolve container endpoint")
2909 return false, nil
2910 }
2911 addr := fmt.Sprintf("%s:%d", ip, port)
2912 timeout := cfg.TCPProbeTimeout
2913 if timeout == 0 {
2914 timeout = 30 * time.Second
2915 }
2916 log.Info().Str("addr", addr).Dur("timeout", timeout).Msg("readiness cascade: using TCP probe")
2917 return true, tcpProbe(ctx, addr, timeout)
2918}
2919
2920// resolveContainerEndpoint returns a host-reachable address for probing.
2921// In rootless podman/Docker setups, container internal IPs are not routable
2922// from the host. We use the host port binding (127.0.0.1:<mapped_port>)
2923// which is always reachable. Falls back to the container's internal IP
2924// only if no host port mapping exists (e.g. host-network mode).
2925func (s *Service) resolveProbeEndpoint(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig) (string, int, error) {
2926 log := zerowrap.FromCtx(ctx)
2927
2928 internalPort := 0
2929 if containerConfig != nil {
2930 if labels := containerConfig.Labels; labels != nil {
2931 for _, key := range []string{domain.LabelProxyPort, domain.LabelPort} {
2932 if portStr, ok := labels[key]; ok && portStr != "" {
2933 port, err := strconv.Atoi(portStr)
2934 if err == nil && port > 0 && port <= 65535 {
2935 internalPort = port
2936 break
2937 }
2938 log.Warn().Str("label", key).Str("port_value", portStr).Msg("invalid probe port label value")
2939 }
2940 }
2941 }
2942 }
2943
2944 if internalPort == 0 {
2945 _, resolvedPort, err := s.runtime.GetContainerNetworkInfo(ctx, containerID)
2946 if err != nil {
2947 return "", 0, err
2948 }
2949 internalPort = resolvedPort
2950 }
2951
2952 hostPort, hostErr := s.runtime.GetContainerPort(ctx, containerID, internalPort)
2953 if hostErr == nil && hostPort > 0 {
2954 log.Debug().
2955 Int("internal_port", internalPort).
2956 Int("host_port", hostPort).
2957 Msg("resolved container endpoint via host port binding")
2958 return "127.0.0.1", hostPort, nil
2959 }
2960
2961 ip, _, fallbackErr := s.runtime.GetContainerNetworkInfo(ctx, containerID)
2962 if fallbackErr != nil {
2963 return "", 0, fallbackErr
2964 }
2965 log.Debug().
2966 Str("ip", ip).
2967 Int("port", internalPort).
2968 Msg("resolved container endpoint via internal IP (no host port binding)")
2969 return ip, internalPort, nil
2970}
2971
2972// waitForReadyByDelay waits using the legacy running+delay strategy.
2973func (s *Service) waitForReadyByDelay(ctx context.Context, containerID string) error {
2974 log := zerowrap.FromCtx(ctx)
2975
2976 s.mu.RLock()
2977 cfg := s.config
2978 s.mu.RUnlock()
2979
2980 delay := cfg.ReadinessDelay
2981 if delay == 0 {
2982 delay = 5 * time.Second
2983 }
2984
2985 log.Debug().Dur("delay", delay).Msg("waiting for container readiness")
2986 select {
2987 case <-time.After(delay):
2988 case <-ctx.Done():
2989 return ctx.Err()
2990 }
2991
2992 // Verify still running after delay
2993 running, err := s.runtime.IsContainerRunning(ctx, containerID)
2994 if err != nil {
2995 return err
2996 }
2997 if running {
2998 return nil
2999 }
3000
3001 return s.waitForRecovery(ctx, containerID)
3002}
3003
3004func (s *Service) waitForHealthy(ctx context.Context, containerID string, timeout time.Duration) error {
3005 if timeout == 0 {
3006 timeout = 90 * time.Second
3007 }
3008 waitCtx, cancel := context.WithTimeout(ctx, timeout)
3009 defer cancel()
3010
3011 var lastStatus string
3012 for {
3013 status, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(waitCtx, containerID)
3014 if err != nil {
3015 return err
3016 }
3017 if !hasHealthcheck {
3018 return errors.New("no healthcheck detected")
3019 }
3020 // Treat empty health status as transitional startup. Some runtimes can
3021 // temporarily report an empty status before first probe results.
3022 if status == "" {
3023 status = "starting"
3024 }
3025 if status == "healthy" {
3026 return nil
3027 }
3028 if status == "unhealthy" {
3029 return errors.New("container reported unhealthy")
3030 }
3031 lastStatus = status
3032
3033 select {
3034 case <-time.After(time.Second):
3035 case <-waitCtx.Done():
3036 return fmt.Errorf("container healthcheck timeout after %s (last status: %s)", timeout, lastStatus)
3037 }
3038 }
3039}
3040
3041// pollContainerRunning polls until the container is running (max 30 seconds).
3042func (s *Service) pollContainerRunning(ctx context.Context, containerID string) error {
3043 for i := 0; i < 30; i++ {
3044 running, err := s.runtime.IsContainerRunning(ctx, containerID)
3045 if err != nil {
3046 return err
3047 }
3048 if running {
3049 return nil
3050 }
3051 // Check if the container has already exited (crash) rather than still starting up.
3052 if info, inspectErr := s.runtime.InspectContainer(ctx, containerID); inspectErr == nil && info != nil && info.Status == string(domain.ContainerStatusExited) {
3053 return fmt.Errorf("container exited immediately with code %d: %w", info.ExitCode, domain.ErrContainerExited)
3054 }
3055 if i == 29 {
3056 return fmt.Errorf("container did not start within 30 seconds")
3057 }
3058 select {
3059 case <-time.After(time.Second):
3060 case <-ctx.Done():
3061 return ctx.Err()
3062 }
3063 }
3064 return fmt.Errorf("container did not start within 30 seconds")
3065}
3066
3067// waitForRecovery waits for a container to recover within the recovery window.
3068func (s *Service) waitForRecovery(ctx context.Context, containerID string) error {
3069 log := zerowrap.FromCtx(ctx)
3070 log.Warn().
3071 Dur("recovery_window", readinessRecoveryWindow).
3072 Msg("container not running after readiness delay, waiting for recovery")
3073
3074 deadline := time.Now().Add(readinessRecoveryWindow)
3075 for time.Now().Before(deadline) {
3076 select {
3077 case <-time.After(time.Second):
3078 case <-ctx.Done():
3079 return ctx.Err()
3080 }
3081
3082 running, err := s.runtime.IsContainerRunning(ctx, containerID)
3083 if err != nil {
3084 return err
3085 }
3086 if running {
3087 log.Info().Msg("container recovered during readiness recovery window")
3088 return nil
3089 }
3090 }
3091
3092 return fmt.Errorf("container not running after readiness delay and recovery window")
3093}
3094
3095// publishContainerDeployed publishes a container.deployed event.
3096func (s *Service) publishContainerDeployed(ctx context.Context, domainName, containerID string) {
3097 payload := &domain.ContainerEventPayload{
3098 ContainerID: containerID,
3099 Domain: domainName,
3100 Action: "deployed",
3101 }
3102
3103 if err := s.eventBus.Publish(domain.EventContainerDeployed, payload); err != nil {
3104 log := zerowrap.FromCtx(ctx)
3105 log.Warn().Err(err).Msg("failed to publish container deployed event")
3106 }
3107}
3108
3109// rewriteToRegistryDomain rewrites an image reference to use the configured registry domain.
3110// e.g., "myapp:latest" -> "registry.example.com/myapp:latest"
3111func rewriteToRegistryDomain(imageRef, registryDomain string) string {
3112 if registryDomain == "" {
3113 return imageRef
3114 }
3115
3116 prefix := registryDomain + "/"
3117 if strings.HasPrefix(imageRef, prefix) {
3118 return imageRef
3119 }
3120
3121 return prefix + imageRef
3122}
3123
3124// rewriteToLocalRegistry rewrites an image reference to use the local registry address.
3125// e.g., "registry.example.com/myapp:latest" -> "localhost:5000/myapp:latest"
3126func rewriteToLocalRegistry(imageRef, registryDomain string, registryPort int) string {
3127 if imageRef == "" {
3128 return imageRef
3129 }
3130
3131 localRegistry := fmt.Sprintf("localhost:%d", registryPort)
3132 localPrefix := localRegistry + "/"
3133 imageRef = strings.TrimPrefix(imageRef, localPrefix)
3134
3135 registryDomain = strings.TrimSuffix(registryDomain, "/")
3136 if registryDomain != "" {
3137 prefix := registryDomain + "/"
3138 imageRef = strings.TrimPrefix(imageRef, prefix)
3139 }
3140
3141 return localPrefix + imageRef
3142}