1// Package container implements the container management use case.
2package container
3
4import (
5 "context"
6 "errors"
7 "fmt"
8 "maps"
9 "slices"
10 "strings"
11 "sync"
12 "time"
13
14 "github.com/bnema/zerowrap"
15 "go.opentelemetry.io/otel"
16 "go.opentelemetry.io/otel/attribute"
17 "go.opentelemetry.io/otel/codes"
18 "go.opentelemetry.io/otel/metric"
19 "go.opentelemetry.io/otel/trace"
20
21 "github.com/bnema/gordon/internal/adapters/out/telemetry"
22 "github.com/bnema/gordon/internal/boundaries/out"
23 "github.com/bnema/gordon/internal/domain"
24)
25
26// Config holds configuration needed by the container service.
27type Config struct {
28 RegistryAuthEnabled bool
29 RegistryDomain string
30 RegistryPort int
31 ServiceTokenUsername string
32 ServiceToken string
33 InternalRegistryUsername string
34 InternalRegistryPassword string
35 PullPolicy string
36 VolumeAutoCreate bool
37 VolumePrefix string
38 VolumePreserve bool
39 NetworkIsolation bool
40 NetworkPrefix string
41 DNSSuffix string
42 NetworkGroups map[string][]string
43 Attachments map[string][]string
44 ReadinessDelay time.Duration // Delay after container starts before considering it ready
45 ReadinessMode string // Readiness strategy: auto, docker-health, delay
46 HealthTimeout time.Duration // Max wait for health-based readiness
47 DrainDelay time.Duration // Grace period after cache invalidation before stopping old container
48 DrainDelayConfigured bool // True when deploy.drain_delay was explicitly configured
49 DrainMode string // Drain strategy: auto, inflight, delay
50 DrainTimeout time.Duration // Max wait for in-flight requests to drain
51}
52
53var tracer = otel.Tracer("gordon.container")
54
55const (
56 PullPolicyAlways = "always"
57 PullPolicyIfNotPresent = "if-not-present"
58 PullPolicyIfTagChanged = "if-tag-changed"
59
60 // readinessRecoveryWindow is an additional grace window used when a container
61 // is briefly not running at the end of readiness delay. This avoids false
62 // negatives during short startup flaps.
63 readinessRecoveryWindow = 30 * time.Second
64 internalPullMaxAttempts = 3
65)
66
67// Service implements the ContainerService interface.
68type Service struct {
69 runtime out.ContainerRuntime
70 envLoader out.EnvLoader
71 eventBus out.EventPublisher
72 logWriter out.ContainerLogWriter
73 cacheInvalidator out.ProxyCacheInvalidator
74 drainWaiter out.ProxyDrainWaiter
75 config Config
76 metrics *telemetry.Metrics
77 containers map[string]*domain.Container
78 attachments map[string][]string // ownerDomain → []containerIDs
79 managedCount int64 // tracks UpDownCounter value for delta computation
80 mu sync.RWMutex
81 deployMu sync.Map // per-domain deploy locks (domain → *domainDeployLock)
82 monitor *Monitor
83}
84
85// domainDeployLock is a context-aware mutex using a buffered channel.
86// It allows Lock() to be interrupted by context cancellation.
87type domainDeployLock struct {
88 ch chan struct{}
89}
90
91// newDomainDeployLock creates a new lock that is initially unlocked.
92func newDomainDeployLock() *domainDeployLock {
93 l := &domainDeployLock{ch: make(chan struct{}, 1)}
94 l.ch <- struct{}{}
95 return l
96}
97
98// Lock acquires the lock or returns an error if the context is cancelled.
99func (l *domainDeployLock) Lock(ctx context.Context) error {
100 // Fast path: honor already-canceled contexts before blocking.
101 if err := ctx.Err(); err != nil {
102 return err
103 }
104 select {
105 case <-ctx.Done():
106 return ctx.Err()
107 case <-l.ch:
108 // Re-check context after acquisition: if canceled while we were
109 // in the select, release the token so the lock isn't consumed.
110 if err := ctx.Err(); err != nil {
111 l.ch <- struct{}{}
112 return err
113 }
114 return nil
115 }
116}
117
118// Unlock releases the lock.
119func (l *domainDeployLock) Unlock() {
120 select {
121 case l.ch <- struct{}{}:
122 default:
123 // This should never happen if Lock/Unlock are used correctly.
124 // Panic to detect double-unlocks or other misuse early.
125 panic("domainDeployLock: double unlock detected")
126 }
127}
128
129// NewService creates a new container service.
130func NewService(
131 runtime out.ContainerRuntime,
132 envLoader out.EnvLoader,
133 eventBus out.EventPublisher,
134 logWriter out.ContainerLogWriter,
135 config Config,
136) *Service {
137 return &Service{
138 runtime: runtime,
139 envLoader: envLoader,
140 eventBus: eventBus,
141 logWriter: logWriter,
142 config: config,
143 containers: make(map[string]*domain.Container),
144 attachments: make(map[string][]string),
145 }
146}
147
148// SetMetrics sets the telemetry metrics for the container service.
149func (s *Service) SetMetrics(m *telemetry.Metrics) {
150 s.metrics = m
151}
152
153// SetProxyCacheInvalidator sets the proxy cache invalidator for synchronous
154// cache invalidation during zero-downtime deployments. This must be called
155// after construction because the proxy service is created after the container
156// service during application initialization.
157func (s *Service) SetProxyCacheInvalidator(inv out.ProxyCacheInvalidator) {
158 s.mu.Lock()
159 s.cacheInvalidator = inv
160 s.mu.Unlock()
161}
162
163// SetProxyDrainWaiter sets the proxy in-flight drain waiter used during
164// zero-downtime replacement before stopping the old container.
165func (s *Service) SetProxyDrainWaiter(waiter out.ProxyDrainWaiter) {
166 s.mu.Lock()
167 s.drainWaiter = waiter
168 s.mu.Unlock()
169}
170
171// domainDeployMu returns a per-domain lock for serializing deploy operations.
172func (s *Service) domainDeployMu(domain string) *domainDeployLock {
173 v, _ := s.deployMu.LoadOrStore(domain, newDomainDeployLock())
174 // Type assertion is safe: LoadOrStore always stores *domainDeployLock values,
175 // and any concurrent LoadOrStore calls will also store *domainDeployLock.
176 return v.(*domainDeployLock)
177}
178
179// acquireDomainDeployLock acquires the per-domain deploy lock with context cancellation support.
180// Returns an unlock function that must be called to release the lock.
181func (s *Service) acquireDomainDeployLock(ctx context.Context, domain string) (func(), error) {
182 lock := s.domainDeployMu(domain)
183 if err := lock.Lock(ctx); err != nil {
184 return nil, err
185 }
186 return func() { lock.Unlock() }, nil
187}
188
189// buildContainerConfig constructs the container configuration for deployment.
190func (s *Service) deploymentContainerName(containerDomain string, existing *domain.Container) string {
191 canonicalName := fmt.Sprintf("gordon-%s", containerDomain)
192 if existing == nil {
193 return canonicalName
194 }
195
196 // Alternate temporary names if the tracked container was left with a temp suffix.
197 // This prevents name collisions after interrupted zero-downtime deploys.
198 newName := canonicalName + "-new"
199 nextName := canonicalName + "-next"
200 switch existing.Name {
201 case newName:
202 return nextName
203 case nextName:
204 return newName
205 default:
206 return newName
207 }
208}
209
210func (s *Service) buildContainerConfig(containerDomain, image, actualImageRef string, exposedPorts []int, envVars []string, volumes map[string]string, networkName string, existing *domain.Container) *domain.ContainerConfig {
211 containerName := s.deploymentContainerName(containerDomain, existing)
212
213 return &domain.ContainerConfig{
214 Image: actualImageRef,
215 Name: containerName,
216 Ports: exposedPorts,
217 Env: envVars,
218 Volumes: volumes,
219 NetworkMode: networkName,
220 Hostname: containerDomain,
221 Labels: map[string]string{
222 "gordon.domain": containerDomain,
223 "gordon.image": image,
224 "gordon.managed": "true",
225 "gordon.route": containerDomain,
226 },
227 AutoRemove: false,
228 }
229}
230
231// Deploy creates and starts a container for the given route.
232// Implements zero-downtime deployment: new container starts before old one stops.
233func (s *Service) Deploy(ctx context.Context, route domain.Route) (*domain.Container, error) {
234 ctx, span := tracer.Start(ctx, "container.deploy",
235 trace.WithAttributes(
236 attribute.String("domain", route.Domain),
237 attribute.String("image", route.Image),
238 ))
239 defer span.End()
240
241 deployStart := time.Now()
242
243 // Serialize deploys for the same domain to prevent race conditions
244 // (e.g. multiple image.pushed events + explicit deploy call from CLI).
245 unlock, err := s.acquireDomainDeployLock(ctx, route.Domain)
246 if err != nil {
247 return nil, err
248 }
249 defer unlock()
250
251 // Enrich context with use case fields for all downstream logs
252 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
253 zerowrap.FieldLayer: "usecase",
254 zerowrap.FieldUseCase: "Deploy",
255 "domain": route.Domain,
256 })
257 log := zerowrap.FromCtx(ctx)
258
259 // Record deploy metrics and trace status
260 defer func() {
261 if err != nil {
262 span.RecordError(err)
263 span.SetStatus(codes.Error, err.Error())
264 }
265 if s.metrics == nil {
266 return
267 }
268 attrs := metric.WithAttributes(
269 attribute.String("domain", route.Domain),
270 attribute.String("image", route.Image),
271 )
272 s.metrics.DeployTotal.Add(ctx, 1, attrs)
273 s.metrics.DeployDuration.Record(ctx, time.Since(deployStart).Seconds(), attrs)
274 if err != nil {
275 s.metrics.DeployErrors.Add(ctx, 1, attrs)
276 }
277 }()
278
279 existing, hasExisting := s.getTrackedContainer(route.Domain)
280
281 resources, err := s.prepareDeployResources(ctx, route, existing)
282 if err != nil {
283 return nil, err
284 }
285
286 // Skip redundant deploy: if the existing container is already running
287 // the exact same image (by Docker image ID), return it immediately.
288 // This prevents the double-deploy caused by the event-based deploy
289 // (triggered by image.pushed) racing with the explicit CLI deploy call.
290 if hasExisting && existing.ImageID != "" {
291 if skip, container := s.skipRedundantDeploy(ctx, existing, resources.actualImageRef); skip {
292 return container, nil
293 }
294 }
295
296 newContainer, err := s.createStartedContainer(ctx, route, existing, resources)
297 if err != nil {
298 return nil, err
299 }
300
301 invalidated := s.activateDeployedContainer(ctx, route.Domain, newContainer)
302 s.finalizePreviousContainer(ctx, route.Domain, existing, hasExisting, invalidated, newContainer.ID)
303
304 // Start container log collection (non-blocking, errors don't fail deployment)
305 s.startLogCollection(ctx, newContainer.ID, route.Domain)
306
307 log.Info().
308 Str("image", route.Image).
309 Str(zerowrap.FieldEntityID, newContainer.ID).
310 Ints("ports", newContainer.Ports).
311 Str("network", resources.networkName).
312 Bool("zero_downtime", hasExisting).
313 Msg("container deployed successfully")
314
315 return newContainer, nil
316}
317
318// skipRedundantDeploy checks whether the existing container is already running
319// the same image (by Docker image ID) as the one we are about to deploy.
320// When a push triggers both an event-based deploy and an explicit CLI deploy,
321// the second one arrives after the first has already completed; this avoids
322// a full redundant create-start-readiness cycle.
323func (s *Service) skipRedundantDeploy(ctx context.Context, existing *domain.Container, actualImageRef string) (bool, *domain.Container) {
324 log := zerowrap.FromCtx(ctx)
325
326 newImageID, err := s.runtime.GetImageID(ctx, actualImageRef)
327 if err != nil {
328 log.Debug().Err(err).Msg("cannot resolve image ID for redundancy check, proceeding with deploy")
329 return false, nil
330 }
331
332 if existing.ImageID != newImageID {
333 return false, nil
334 }
335
336 // Verify the container is actually still running before skipping.
337 running, err := s.runtime.IsContainerRunning(ctx, existing.ID)
338 if err != nil || !running {
339 return false, nil
340 }
341
342 log.Info().
343 Str("container_id", existing.ID).
344 Str("image_id", newImageID).
345 Msg("skipping redundant deploy: container already running this image")
346
347 return true, existing
348}
349
350type deployResources struct {
351 networkName string
352 actualImageRef string
353 exposedPorts []int
354 envVars []string
355 volumes map[string]string
356}
357
358func (s *Service) getTrackedContainer(domainName string) (*domain.Container, bool) {
359 s.mu.RLock()
360 container, ok := s.containers[domainName]
361 s.mu.RUnlock()
362 return container, ok
363}
364
365func (s *Service) prepareDeployResources(ctx context.Context, route domain.Route, existing *domain.Container) (*deployResources, error) {
366 log := zerowrap.FromCtx(ctx)
367
368 existingID := ""
369 if existing != nil {
370 existingID = existing.ID
371 }
372 if err := s.cleanupOrphanedContainers(ctx, route.Domain, existingID); err != nil {
373 log.WrapErr(err, "failed to cleanup orphaned containers")
374 }
375
376 imageRef := s.buildImageRef(route.Image)
377 actualImageRef, err := s.ensureImage(ctx, imageRef)
378 if err != nil {
379 return nil, err
380 }
381
382 networkName := s.getNetworkForApp(route.Domain)
383 if err := s.createNetworkIfNeeded(ctx, networkName); err != nil {
384 return nil, log.WrapErr(err, "failed to create network")
385 }
386 if err := s.deployAttachments(ctx, route.Domain, networkName); err != nil {
387 return nil, log.WrapErr(err, "failed to deploy attachments")
388 }
389
390 exposedPorts, err := s.runtime.GetImageExposedPorts(ctx, actualImageRef)
391 if err != nil {
392 log.WrapErr(err, "failed to get exposed ports, using defaults")
393 exposedPorts = []int{80, 8080, 3000}
394 }
395
396 envVars, err := s.loadEnvironment(ctx, route.Domain, actualImageRef)
397 if err != nil {
398 return nil, err
399 }
400
401 volumes, err := s.setupVolumes(ctx, route.Domain, actualImageRef)
402 if err != nil {
403 return nil, err
404 }
405
406 return &deployResources{
407 networkName: networkName,
408 actualImageRef: actualImageRef,
409 exposedPorts: exposedPorts,
410 envVars: envVars,
411 volumes: volumes,
412 }, nil
413}
414
415func (s *Service) createStartedContainer(ctx context.Context, route domain.Route, existing *domain.Container, resources *deployResources) (*domain.Container, error) {
416 ctx, span := tracer.Start(ctx, "container.create_and_start")
417 defer span.End()
418
419 log := zerowrap.FromCtx(ctx)
420
421 containerConfig := s.buildContainerConfig(
422 route.Domain,
423 route.Image,
424 resources.actualImageRef,
425 resources.exposedPorts,
426 resources.envVars,
427 resources.volumes,
428 resources.networkName,
429 existing,
430 )
431
432 newContainer, err := s.runtime.CreateContainer(ctx, containerConfig)
433 if err != nil {
434 return nil, log.WrapErr(err, "failed to create container")
435 }
436 if err := s.runtime.StartContainer(ctx, newContainer.ID); err != nil {
437 s.runtime.RemoveContainer(ctx, newContainer.ID, true)
438 return nil, log.WrapErr(err, "failed to start container")
439 }
440 if !domain.IsSkipReadiness(ctx) {
441 if err := s.waitForReady(ctx, newContainer.ID); err != nil {
442 s.cleanupFailedContainer(ctx, newContainer.ID)
443 return nil, log.WrapErr(err, "container failed readiness check")
444 }
445 }
446
447 inspected, err := s.runtime.InspectContainer(ctx, newContainer.ID)
448 if err != nil {
449 s.cleanupFailedContainer(ctx, newContainer.ID)
450 return nil, log.WrapErr(err, "failed to inspect started container")
451 }
452
453 return inspected, nil
454}
455
456func (s *Service) activateDeployedContainer(ctx context.Context, domainName string, container *domain.Container) bool {
457 s.mu.Lock()
458 _, wasTracked := s.containers[domainName]
459 s.containers[domainName] = container
460 if !wasTracked {
461 s.managedCount++
462 }
463 s.mu.Unlock()
464
465 // Track managed container count (only increment for new domains, not replacements)
466 if !wasTracked && s.metrics != nil {
467 s.metrics.ManagedContainers.Add(ctx, 1)
468 }
469
470 s.publishContainerDeployed(ctx, domainName, container.ID)
471
472 s.mu.RLock()
473 inv := s.cacheInvalidator
474 s.mu.RUnlock()
475 if inv != nil {
476 inv.InvalidateTarget(ctx, domainName)
477 return true
478 }
479
480 return false
481}
482
483func (s *Service) finalizePreviousContainer(ctx context.Context, domainName string, existing *domain.Container, hasExisting, invalidated bool, newContainerID string) {
484 if !hasExisting {
485 return
486 }
487 log := zerowrap.FromCtx(ctx)
488
489 if invalidated {
490 s.waitForDrain(ctx, existing.ID)
491 }
492
493 if err := ctx.Err(); err != nil {
494 log.Debug().Err(err).Str("domain", domainName).Str("new_container_id", newContainerID).Msg("skipping old container cleanup due to canceled context")
495 return
496 }
497
498 s.cleanupOldContainer(ctx, existing, newContainerID, domainName)
499}
500
501func (s *Service) waitForDrain(ctx context.Context, oldContainerID string) {
502 log := zerowrap.FromCtx(ctx)
503
504 s.mu.RLock()
505 cfg := s.config
506 waiter := s.drainWaiter
507 s.mu.RUnlock()
508
509 drainMode := cfg.DrainMode
510 if drainMode == "" {
511 drainMode = "delay"
512 }
513
514 shouldUseInFlight := (drainMode == "inflight" || drainMode == "auto") && waiter != nil
515 if shouldUseInFlight {
516 timeout := cfg.DrainTimeout
517 if timeout == 0 {
518 timeout = 30 * time.Second
519 }
520 drained := waiter.WaitForNoInFlight(ctx, oldContainerID, timeout)
521 if !drained {
522 log.Warn().
523 Str("old_container_id", oldContainerID).
524 Dur("drain_timeout", timeout).
525 Msg("drain wait timed out; old container may still have in-flight traffic")
526 }
527 return
528 }
529
530 drainDelay := 2 * time.Second
531 if cfg.DrainDelayConfigured {
532 drainDelay = cfg.DrainDelay
533 }
534 if drainDelay <= 0 {
535 return
536 }
537 select {
538 case <-time.After(drainDelay):
539 case <-ctx.Done():
540 }
541}
542
543// Restart restarts a running container for the given domain.
544func (s *Service) Restart(ctx context.Context, domainName string, withAttachments bool) error {
545 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
546 zerowrap.FieldLayer: "usecase",
547 zerowrap.FieldUseCase: "Restart",
548 "domain": domainName,
549 })
550 log := zerowrap.FromCtx(ctx)
551
552 s.mu.RLock()
553 container, exists := s.containers[domainName]
554 attachments := s.attachments[domainName]
555 var attachmentIDs []string
556 if len(attachments) > 0 {
557 attachmentIDs = make([]string, len(attachments))
558 copy(attachmentIDs, attachments)
559 }
560 s.mu.RUnlock()
561
562 if !exists || container == nil {
563 return domain.ErrContainerNotFound
564 }
565
566 // Restart the main container
567 if err := s.runtime.RestartContainer(ctx, container.ID); err != nil {
568 if !isContainerNotFoundError(err) {
569 return log.WrapErr(err, "failed to restart container")
570 }
571
572 // In-memory container ID can become stale after external runtime changes.
573 // Re-sync and retry once with the latest tracked container.
574 log.Warn().
575 Err(err).
576 Str(zerowrap.FieldEntityID, container.ID).
577 Msg("tracked container missing during restart, attempting state reconciliation")
578
579 if syncErr := s.SyncContainers(ctx); syncErr != nil {
580 log.Warn().Err(syncErr).Msg("failed to sync container state during restart recovery")
581 }
582
583 s.mu.RLock()
584 refreshed, refreshedExists := s.containers[domainName]
585 attachmentIDs = append([]string{}, s.attachments[domainName]...)
586 s.mu.RUnlock()
587
588 if !refreshedExists || refreshed == nil {
589 return domain.ErrContainerNotFound
590 }
591 if err := s.runtime.RestartContainer(ctx, refreshed.ID); err != nil {
592 return log.WrapErr(err, "failed to restart container after state reconciliation")
593 }
594 container = refreshed
595 }
596 log.Info().Str(zerowrap.FieldEntityID, container.ID).Msg("container restarted")
597
598 // Record restart metric
599 if s.metrics != nil {
600 s.metrics.ContainerRestarts.Add(ctx, 1, metric.WithAttributes(
601 attribute.String("domain", domainName),
602 attribute.String("source", "api"),
603 ))
604 }
605
606 // Restart attachments if requested
607 if withAttachments && len(attachmentIDs) > 0 {
608 for _, attachID := range attachmentIDs {
609 if err := s.runtime.RestartContainer(ctx, attachID); err != nil {
610 log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to restart attachment")
611 continue // Don't fail the whole operation for one attachment
612 }
613 log.Info().Str("attachment_id", attachID).Msg("attachment restarted")
614 }
615 }
616
617 return nil
618}
619
620// Stop stops a running container.
621func (s *Service) Stop(ctx context.Context, containerID string) error {
622 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
623 zerowrap.FieldLayer: "usecase",
624 zerowrap.FieldUseCase: "Stop",
625 zerowrap.FieldEntityID: containerID,
626 })
627 log := zerowrap.FromCtx(ctx)
628
629 // Stop log collection before stopping container
630 if s.logWriter != nil {
631 if err := s.logWriter.StopLogging(containerID); err != nil {
632 log.Warn().Err(err).Msg("failed to stop container log collection")
633 }
634 }
635
636 if err := s.runtime.StopContainer(ctx, containerID); err != nil {
637 return log.WrapErr(err, "failed to stop container")
638 }
639
640 log.Info().Msg("container stopped")
641 return nil
642}
643
644// Remove removes a container.
645func (s *Service) Remove(ctx context.Context, containerID string, force bool) error {
646 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
647 zerowrap.FieldLayer: "usecase",
648 zerowrap.FieldUseCase: "Remove",
649 zerowrap.FieldEntityID: containerID,
650 })
651 log := zerowrap.FromCtx(ctx)
652
653 // Stop log collection before removing container
654 if s.logWriter != nil {
655 if err := s.logWriter.StopLogging(containerID); err != nil {
656 log.Warn().Err(err).Msg("failed to stop container log collection")
657 }
658 }
659
660 // Find domain and attachment IDs for cleanup
661 containerDomain, attachmentIDs, cfg := s.findContainerContext(containerID)
662
663 if err := s.runtime.RemoveContainer(ctx, containerID, force); err != nil {
664 return log.WrapErr(err, "failed to remove container")
665 }
666
667 // Cleanup volumes
668 if containerDomain != "" && !cfg.VolumePreserve {
669 if err := s.cleanupVolumesForDomain(ctx, containerDomain); err != nil {
670 log.WrapErrWithFields(err, "failed to cleanup volumes", map[string]any{"domain": containerDomain})
671 }
672 }
673
674 // Clean up attachment containers
675 s.removeAttachments(ctx, attachmentIDs)
676
677 // Remove from tracking
678 s.mu.Lock()
679 var removedDomain string
680 for d, c := range s.containers {
681 if c.ID == containerID {
682 delete(s.containers, d)
683 removedDomain = d
684 s.managedCount--
685 log.Info().Str("domain", d).Msg("container removed")
686 break
687 }
688 }
689 if removedDomain != "" {
690 delete(s.attachments, removedDomain)
691 }
692 s.mu.Unlock()
693
694 // Decrement managed container count
695 if removedDomain != "" && s.metrics != nil {
696 s.metrics.ManagedContainers.Add(ctx, -1)
697 }
698
699 // Note: We intentionally do NOT clean up deployMu entries to avoid a race condition:
700 // If Remove deletes the mutex entry while a concurrent Deploy is acquiring the lock,
701 // the Deploy would create a fresh mutex, breaking serialization and allowing concurrent
702 // deploys to the same domain. The memory footprint is acceptable (one small struct per
703 // domain ever deployed), and this is the safer choice for correctness.
704
705 // Cleanup network
706 if removedDomain != "" && cfg.NetworkIsolation {
707 networkName := s.getNetworkForApp(removedDomain)
708 if err := s.cleanupNetworkIfEmpty(ctx, networkName); err != nil {
709 log.WrapErrWithFields(err, "failed to cleanup network", map[string]any{"network": networkName})
710 }
711 }
712
713 return nil
714}
715
716// findContainerContext looks up the domain, attachment IDs, and config snapshot for a container.
717func (s *Service) findContainerContext(containerID string) (string, []string, Config) {
718 s.mu.RLock()
719 defer s.mu.RUnlock()
720 cfg := s.config
721 var containerDomain string
722 for d, c := range s.containers {
723 if c.ID == containerID {
724 containerDomain = d
725 break
726 }
727 }
728 var attachmentIDs []string
729 if containerDomain != "" {
730 attachmentIDs = append(attachmentIDs, s.attachments[containerDomain]...)
731 }
732 return containerDomain, attachmentIDs, cfg
733}
734
735// removeAttachments stops and removes attachment containers best-effort.
736func (s *Service) removeAttachments(ctx context.Context, attachmentIDs []string) {
737 log := zerowrap.FromCtx(ctx)
738 for _, attachID := range attachmentIDs {
739 if s.logWriter != nil {
740 if err := s.logWriter.StopLogging(attachID); err != nil {
741 log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to stop attachment log collection")
742 }
743 }
744 if err := s.runtime.StopContainer(ctx, attachID); err != nil {
745 log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to stop attachment container")
746 }
747 if err := s.runtime.RemoveContainer(ctx, attachID, true); err != nil {
748 log.Warn().Err(err).Str("attachment_id", attachID).Msg("failed to remove attachment container")
749 }
750 }
751}
752
753// Get retrieves a container by domain.
754func (s *Service) Get(_ context.Context, domainName string) (*domain.Container, bool) {
755 s.mu.RLock()
756 defer s.mu.RUnlock()
757 container, exists := s.containers[domainName]
758 return container, exists
759}
760
761// List returns all managed containers.
762func (s *Service) List(_ context.Context) map[string]*domain.Container {
763 s.mu.RLock()
764 defer s.mu.RUnlock()
765
766 result := make(map[string]*domain.Container, len(s.containers))
767 maps.Copy(result, s.containers)
768 return result
769}
770
771// ListRoutesWithDetails returns routes with network and attachment info.
772// Note: Container map is copied under lock, then external calls are made without lock.
773// If containers are removed between copy and runtime calls, errors are handled gracefully
774// (network becomes empty string). This trade-off avoids holding locks during I/O.
775func (s *Service) ListRoutesWithDetails(ctx context.Context) []domain.RouteInfo {
776 s.mu.RLock()
777 containers := make(map[string]*domain.Container, len(s.containers))
778 maps.Copy(containers, s.containers)
779 s.mu.RUnlock()
780
781 // Fetch all attachments once to avoid N+1 queries
782 attachmentsByDomain := s.getAllAttachments(ctx)
783
784 results := make([]domain.RouteInfo, 0, len(containers))
785 for domainName, container := range containers {
786 network := ""
787 image := ""
788 containerID := ""
789 status := ""
790 if container != nil {
791 containerID = container.ID
792 status = container.Status
793 image = container.Image
794 if container.Labels != nil {
795 if labelImage, ok := container.Labels[domain.LabelImage]; ok && labelImage != "" {
796 image = labelImage
797 }
798 }
799 // Strip registry domain prefix from image for cleaner display
800 image = s.stripRegistryPrefix(image)
801 if networkName, err := s.runtime.GetContainerNetwork(ctx, container.ID); err == nil {
802 network = networkName
803 }
804 }
805
806 results = append(results, domain.RouteInfo{
807 Domain: domainName,
808 Image: image,
809 ContainerID: containerID,
810 ContainerStatus: status,
811 Network: network,
812 Attachments: attachmentsByDomain[domainName],
813 })
814 }
815
816 return results
817}
818
819// stripRegistryPrefix removes the configured registry domain prefix from an image reference.
820// For example, "reg.example.com/myapp:latest" becomes "myapp:latest" if registry domain is "reg.example.com".
821func (s *Service) stripRegistryPrefix(image string) string {
822 s.mu.RLock()
823 cfg := s.config
824 s.mu.RUnlock()
825
826 if cfg.RegistryDomain == "" {
827 return image
828 }
829 prefix := strings.TrimSuffix(cfg.RegistryDomain, "/") + "/"
830 if strings.HasPrefix(image, prefix) {
831 return strings.TrimPrefix(image, prefix)
832 }
833 return image
834}
835
836// ListAttachments returns attachments for a domain.
837func (s *Service) ListAttachments(ctx context.Context, domainName string) []domain.Attachment {
838 return s.getAttachmentsForDomain(ctx, domainName)
839}
840
841// ListNetworks returns Gordon-managed networks.
842func (s *Service) ListNetworks(ctx context.Context) ([]*domain.NetworkInfo, error) {
843 s.mu.RLock()
844 cfg := s.config
845 s.mu.RUnlock()
846
847 networks, err := s.runtime.ListNetworks(ctx)
848 if err != nil {
849 return nil, err
850 }
851
852 var filtered []*domain.NetworkInfo
853 for _, network := range networks {
854 if strings.HasPrefix(network.Name, cfg.NetworkPrefix+"-") {
855 filtered = append(filtered, network)
856 }
857 }
858
859 return filtered, nil
860}
861
862// HealthCheck performs health checks on all containers.
863func (s *Service) HealthCheck(ctx context.Context) map[string]bool {
864 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
865 zerowrap.FieldLayer: "usecase",
866 zerowrap.FieldUseCase: "HealthCheck",
867 })
868 log := zerowrap.FromCtx(ctx)
869
870 // Copy container IDs under lock, then release before making Docker API calls
871 s.mu.RLock()
872 snapshot := make(map[string]string, len(s.containers))
873 for d, c := range s.containers {
874 snapshot[d] = c.ID
875 }
876 s.mu.RUnlock()
877
878 health := make(map[string]bool, len(snapshot))
879 for d, id := range snapshot {
880 running, err := s.runtime.IsContainerRunning(ctx, id)
881 if err != nil {
882 log.WrapErrWithFields(err, "health check failed", map[string]any{"domain": d, zerowrap.FieldEntityID: id})
883 health[d] = false
884 } else {
885 health[d] = running
886 }
887 }
888 return health
889}
890
891// SyncContainers synchronizes containers with runtime state.
892func (s *Service) SyncContainers(ctx context.Context) error {
893 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
894 zerowrap.FieldLayer: "usecase",
895 zerowrap.FieldUseCase: "SyncContainers",
896 })
897 log := zerowrap.FromCtx(ctx)
898
899 // List containers without holding lock to avoid blocking during Docker API call
900 allContainers, err := s.runtime.ListContainers(ctx, false)
901 if err != nil {
902 return log.WrapErr(err, "failed to list containers")
903 }
904
905 managed := make(map[string]*domain.Container)
906 for _, c := range allContainers {
907 if c.Labels != nil {
908 if d, ok := c.Labels["gordon.domain"]; ok && c.Labels["gordon.managed"] == "true" {
909 managed[d] = c
910 }
911 }
912 }
913
914 s.mu.Lock()
915 s.containers = managed
916 newCount := int64(len(managed))
917 delta := newCount - s.managedCount
918 s.managedCount = newCount
919 s.mu.Unlock()
920
921 // Report initial/delta managed container count to OTel UpDownCounter.
922 if delta != 0 && s.metrics != nil {
923 s.metrics.ManagedContainers.Add(ctx, delta)
924 }
925
926 log.Info().Int(zerowrap.FieldCount, len(managed)).Msg("container state synchronized")
927 return nil
928}
929
930// AutoStart starts containers for the provided routes that aren't running.
931// It skips readiness checks to avoid blocking boot; the background monitor
932// handles crash recovery.
933func (s *Service) AutoStart(ctx context.Context, routes []domain.Route) error {
934 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
935 zerowrap.FieldLayer: "usecase",
936 zerowrap.FieldUseCase: "AutoStart",
937 })
938 log := zerowrap.FromCtx(ctx)
939
940 log.Info().Int("route_count", len(routes)).Msg("auto-starting containers for configured routes")
941
942 // Filter out routes that already have a running container.
943 var pending []domain.Route
944 skipped := 0
945 for _, route := range routes {
946 if _, exists := s.Get(ctx, route.Domain); exists {
947 log.Debug().Str("domain", route.Domain).Msg("container already running, skipping")
948 skipped++
949 } else {
950 pending = append(pending, route)
951 }
952 }
953
954 if len(pending) == 0 {
955 log.Info().Int("skipped", skipped).Msg("auto-start completed, all containers already running")
956 return nil
957 }
958
959 // Deploy all pending routes concurrently with readiness checks skipped.
960 deployCtx := domain.WithSkipReadiness(ctx)
961
962 type result struct {
963 route domain.Route
964 err error
965 }
966 results := make(chan result, len(pending))
967 sem := make(chan struct{}, 4) // limit concurrency
968
969 for _, route := range pending {
970 sem <- struct{}{}
971 go func(r domain.Route) {
972 defer func() { <-sem }()
973
974 log.Info().Str("domain", r.Domain).Str("image", r.Image).Msg("auto-starting container for route")
975
976 if _, err := s.Deploy(deployCtx, r); err != nil {
977 results <- result{route: r, err: err}
978 return
979 }
980 results <- result{route: r}
981 }(route)
982 }
983
984 var started, errCount int
985 for i := 0; i < len(pending); i++ {
986 res := <-results
987 if res.err != nil {
988 log.Warn().Err(res.err).Str("domain", res.route.Domain).Msg("failed to auto-start container")
989 errCount++
990 } else {
991 started++
992 }
993 }
994
995 log.Info().
996 Int("started", started).
997 Int("skipped", skipped).
998 Int("errors", errCount).
999 Msg("auto-start completed")
1000
1001 if errCount > 0 {
1002 return fmt.Errorf("auto-start completed with %d errors", errCount)
1003 }
1004 return nil
1005}
1006
1007// Shutdown gracefully shuts down all managed containers.
1008func (s *Service) Shutdown(ctx context.Context) error {
1009 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1010 zerowrap.FieldLayer: "usecase",
1011 zerowrap.FieldUseCase: "Shutdown",
1012 })
1013 log := zerowrap.FromCtx(ctx)
1014 log.Info().Msg("shutting down container manager...")
1015
1016 // Containers are left running across Gordon restarts.
1017 // SyncContainers + AutoStart will pick them back up on next boot.
1018
1019 // Close log writer to stop all log collection
1020 if s.logWriter != nil {
1021 if err := s.logWriter.Close(); err != nil {
1022 log.Warn().Err(err).Msg("failed to close container log writer")
1023 }
1024 }
1025
1026 log.Info().Msg("container manager shutdown complete")
1027 return nil
1028}
1029
1030// StartMonitor begins background monitoring of tracked containers.
1031// The monitor restarts crashed containers and detects crash loops.
1032// Safe to call multiple times; subsequent calls are no-ops.
1033func (s *Service) StartMonitor(ctx context.Context) {
1034 s.mu.Lock()
1035 if s.monitor != nil {
1036 s.mu.Unlock()
1037 return
1038 }
1039 s.monitor = newMonitor(s)
1040 m := s.monitor
1041 m.Start(ctx)
1042 s.mu.Unlock()
1043}
1044
1045// StopMonitor stops the background container monitor.
1046// Safe to call multiple times; subsequent calls are no-ops.
1047func (s *Service) StopMonitor() {
1048 s.mu.Lock()
1049 m := s.monitor
1050 s.monitor = nil
1051 s.mu.Unlock()
1052 if m != nil {
1053 m.Stop()
1054 }
1055}
1056
1057// UpdateConfig updates the service configuration.
1058func (s *Service) UpdateConfig(config Config) {
1059 s.mu.Lock()
1060 s.config = config
1061 s.mu.Unlock()
1062}
1063
1064// Helper methods
1065
1066// cleanupFailedContainer stops and removes a container that failed to start properly.
1067func (s *Service) cleanupFailedContainer(ctx context.Context, containerID string) {
1068 log := zerowrap.FromCtx(ctx)
1069 if err := s.runtime.StopContainer(ctx, containerID); err != nil {
1070 log.Warn().Err(err).Str(zerowrap.FieldEntityID, containerID).Msg("failed to stop container after failure")
1071 }
1072 if err := s.runtime.RemoveContainer(ctx, containerID, true); err != nil {
1073 log.Warn().Err(err).Str(zerowrap.FieldEntityID, containerID).Msg("failed to remove container after failure")
1074 }
1075}
1076
1077// cleanupOldContainer stops and removes an old container after zero-downtime switch.
1078// It also renames the new container to the canonical name.
1079func (s *Service) cleanupOldContainer(ctx context.Context, old *domain.Container, newContainerID, domainName string) {
1080 log := zerowrap.FromCtx(ctx)
1081 log.Info().Str(zerowrap.FieldEntityID, old.ID).Msg("stopping old container after zero-downtime switch")
1082
1083 if s.logWriter != nil {
1084 if err := s.logWriter.StopLogging(old.ID); err != nil {
1085 log.Warn().Err(err).Str(zerowrap.FieldEntityID, old.ID).Msg("failed to stop logging for old container")
1086 }
1087 }
1088
1089 if err := s.runtime.StopContainer(ctx, old.ID); err != nil {
1090 log.Warn().Err(err).Str(zerowrap.FieldEntityID, old.ID).Msg("failed to stop old container")
1091 }
1092
1093 if err := s.runtime.RemoveContainer(ctx, old.ID, true); err != nil {
1094 log.Warn().Err(err).Str(zerowrap.FieldEntityID, old.ID).Msg("failed to remove old container")
1095 }
1096
1097 // Rename new container to canonical name
1098 canonicalName := fmt.Sprintf("gordon-%s", domainName)
1099 if err := s.runtime.RenameContainer(ctx, newContainerID, canonicalName); err != nil {
1100 log.Warn().Err(err).Str("canonical_name", canonicalName).Msg("failed to rename container to canonical name")
1101 }
1102}
1103
1104// startLogCollection starts log collection for a container in the background.
1105// Errors are logged but don't fail the calling operation.
1106func (s *Service) startLogCollection(ctx context.Context, containerID, domainName string) {
1107 if s.logWriter == nil {
1108 return
1109 }
1110
1111 log := zerowrap.FromCtx(ctx)
1112
1113 logStream, err := s.runtime.GetContainerLogs(ctx, containerID, true)
1114 if err != nil {
1115 log.Warn().Err(err).Msg("failed to get container logs for collection")
1116 return
1117 }
1118
1119 if err := s.logWriter.StartLogging(ctx, containerID, domainName, logStream); err != nil {
1120 log.Warn().Err(err).Msg("failed to start container log collection")
1121 logStream.Close()
1122 }
1123}
1124
1125func (s *Service) buildImageRef(image string) string {
1126 s.mu.RLock()
1127 cfg := s.config
1128 s.mu.RUnlock()
1129
1130 if !cfg.RegistryAuthEnabled || cfg.RegistryDomain == "" {
1131 return image
1132 }
1133
1134 // Normalize registry domain by trimming any trailing slash
1135 reg := strings.TrimSuffix(cfg.RegistryDomain, "/")
1136
1137 // Check if image already has the registry domain prefix
1138 prefix := reg + "/"
1139 if strings.HasPrefix(image, prefix) {
1140 return image
1141 }
1142
1143 // Check if image already has an explicit registry (host:port/ or host.domain/).
1144 // We need to detect patterns like:
1145 // - "docker.io/library/nginx:latest" (has dot and slash)
1146 // - "localhost:5001/myapp:latest" (has host:port/ pattern)
1147 // - "myregistry:5000/app:v1" (has host:port/ pattern)
1148 // - "[fd00::1]:5000/app:v1" (IPv6 with port)
1149 if hasExplicitRegistry(image) {
1150 return image
1151 }
1152
1153 return fmt.Sprintf("%s/%s", reg, image)
1154}
1155
1156// hasExplicitRegistry checks if an image reference already includes an explicit registry.
1157// Returns true for patterns like "docker.io/image", "localhost:5000/image", "localhost/image",
1158// "registry:8080/image", "[fd00::1]/image", "[fd00::1]:5000/image".
1159func hasExplicitRegistry(image string) bool {
1160 // Find the first slash which separates registry from image name
1161 slashIdx := strings.Index(image, "/")
1162 if slashIdx == -1 {
1163 return false // No slash means no registry prefix (e.g., "myapp:latest")
1164 }
1165
1166 // Extract the part before the first slash (potential registry)
1167 registryPart := image[:slashIdx]
1168
1169 // Check for bracketed IPv6 address (e.g., "[fd00::1]" or "[fd00::1]:5000")
1170 if strings.HasPrefix(registryPart, "[") {
1171 // Look for closing bracket
1172 if closeBracket := strings.Index(registryPart, "]"); closeBracket != -1 {
1173 // Valid bracketed IPv6: either ends at ] or has ]:port
1174 return true
1175 }
1176 }
1177
1178 // Check for localhost (with or without port)
1179 if registryPart == "localhost" {
1180 return true
1181 }
1182
1183 // Check for host:port pattern (e.g., "localhost:5000", "registry:8080")
1184 if colonIdx := strings.LastIndex(registryPart, ":"); colonIdx != -1 {
1185 port := registryPart[colonIdx+1:]
1186 // If everything after colon is digits, it's a port
1187 if len(port) > 0 && isNumeric(port) {
1188 return true
1189 }
1190 }
1191
1192 // Check for domain pattern (has a dot, e.g., "docker.io", "gcr.io")
1193 if strings.Contains(registryPart, ".") {
1194 return true
1195 }
1196
1197 return false
1198}
1199
1200// isNumeric checks if a string contains only digits.
1201func isNumeric(s string) bool {
1202 for _, c := range s {
1203 if c < '0' || c > '9' {
1204 return false
1205 }
1206 }
1207 return true
1208}
1209
1210func normalizePullPolicy(policy string) string {
1211 switch strings.ToLower(strings.TrimSpace(policy)) {
1212 case PullPolicyAlways:
1213 return PullPolicyAlways
1214 case PullPolicyIfTagChanged:
1215 return PullPolicyIfTagChanged
1216 case PullPolicyIfNotPresent:
1217 return PullPolicyIfNotPresent
1218 default:
1219 return PullPolicyIfNotPresent
1220 }
1221}
1222
1223func isDigestRef(imageRef string) bool {
1224 return strings.Contains(imageRef, "@sha256:")
1225}
1226
1227func (s *Service) pullRefForDeploy(ctx context.Context, imageRef string) (string, bool) {
1228 if !domain.IsInternalDeploy(ctx) {
1229 return imageRef, false
1230 }
1231 s.mu.RLock()
1232 cfg := s.config
1233 s.mu.RUnlock()
1234 return rewriteToLocalRegistry(imageRef, cfg.RegistryDomain, cfg.RegistryPort), true
1235}
1236
1237// ensureImage ensures the image is available locally, pulling if needed.
1238// Returns the image reference to use for container operations:
1239// - For digest references (@sha256:...), returns the pullRef since Docker can't tag digests
1240// - For tagged images, returns the original imageRef after tagging the pulled image
1241func (s *Service) ensureImage(ctx context.Context, imageRef string) (string, error) {
1242 ctx, span := tracer.Start(ctx, "container.ensure_image",
1243 trace.WithAttributes(attribute.String("image", imageRef)))
1244 defer span.End()
1245
1246 ctx = zerowrap.CtxWithField(ctx, "image", imageRef)
1247 log := zerowrap.FromCtx(ctx)
1248
1249 // Determine if this is an internal deploy and what reference to use for pulls.
1250 pullRef, isInternal := s.pullRefForDeploy(ctx, imageRef)
1251 if pullRef != imageRef {
1252 log.Info().
1253 Str("original_ref", imageRef).
1254 Str("pull_ref", pullRef).
1255 Msg("internal deploy: using localhost registry for pull")
1256 }
1257
1258 found, err := s.ensureLocalImage(ctx, imageRef, pullRef)
1259 if err != nil {
1260 return "", err
1261 }
1262 if found {
1263 return imageRef, nil
1264 }
1265
1266 // Pull image
1267 log.Info().Msg("pulling image from registry")
1268
1269 if err := s.pullImage(ctx, pullRef, isInternal); err != nil {
1270 return "", err
1271 }
1272
1273 // For digest references, we can't create a tag (Docker doesn't allow it).
1274 // In this case, use the pullRef directly since the image is available under that reference.
1275 if strings.Contains(imageRef, "@sha256:") {
1276 log.Info().
1277 Str("pull_ref", pullRef).
1278 Msg("digest reference: using pull reference for container operations")
1279 return pullRef, nil
1280 }
1281
1282 if err := s.tagImageIfNeeded(ctx, pullRef, imageRef); err != nil {
1283 return "", err
1284 }
1285
1286 // Clean up the temporary pull reference tag to avoid duplicate entries
1287 if pullRef != imageRef {
1288 if err := s.runtime.UntagImage(ctx, pullRef); err != nil {
1289 // Log but don't fail - the canonical tag is already applied
1290 log.Debug().Err(err).Str("pull_ref", pullRef).Msg("failed to remove temporary pull tag")
1291 }
1292 }
1293
1294 log.Info().Msg("image pulled successfully")
1295 return imageRef, nil
1296}
1297
1298func (s *Service) ensureLocalImage(ctx context.Context, imageRef, pullRef string) (bool, error) {
1299 log := zerowrap.FromCtx(ctx)
1300
1301 // For internal deploys (image push events), always pull fresh image
1302 // because the same tag may reference new content (e.g., latest tag updated)
1303 if domain.IsInternalDeploy(ctx) {
1304 log.Info().Msg("internal deploy detected, forcing image pull to ensure latest content")
1305 return false, nil
1306 }
1307
1308 s.mu.RLock()
1309 cfg := s.config
1310 s.mu.RUnlock()
1311
1312 pullPolicy := normalizePullPolicy(cfg.PullPolicy)
1313 switch pullPolicy {
1314 case PullPolicyAlways:
1315 log.Info().Str("pull_policy", pullPolicy).Msg("pull policy forces image pull")
1316 return false, nil
1317 case PullPolicyIfTagChanged:
1318 if !isDigestRef(imageRef) {
1319 log.Info().Str("pull_policy", pullPolicy).Msg("tag reference detected, pulling to check for updates")
1320 return false, nil
1321 }
1322 }
1323
1324 localImages, err := s.runtime.ListImages(ctx)
1325 if err != nil {
1326 log.WrapErr(err, "failed to list local images, will attempt pull")
1327 return false, nil
1328 }
1329
1330 normalizedRef := normalizeImageRef(imageRef)
1331 normalizedPullRef := normalizeImageRef(pullRef)
1332 for _, img := range localImages {
1333 normalizedImage := normalizeImageRef(img)
1334 if normalizedImage == normalizedRef {
1335 log.Info().Msg("image found locally, skipping pull")
1336 return true, nil
1337 }
1338 if normalizedImage == normalizedPullRef {
1339 if err := s.tagImageIfNeeded(ctx, pullRef, imageRef); err != nil {
1340 return false, err
1341 }
1342 // Clean up the temporary pull reference tag
1343 if pullRef != imageRef {
1344 if err := s.runtime.UntagImage(ctx, pullRef); err != nil {
1345 log.Debug().Err(err).Str("pull_ref", pullRef).Msg("failed to remove temporary pull tag")
1346 }
1347 }
1348 log.Info().Msg("image found locally, skipping pull")
1349 return true, nil
1350 }
1351 }
1352
1353 return false, nil
1354}
1355
1356func (s *Service) pullImage(ctx context.Context, pullRef string, isInternal bool) error {
1357 s.mu.RLock()
1358 cfg := s.config
1359 s.mu.RUnlock()
1360
1361 log := zerowrap.FromCtx(ctx)
1362
1363 switch {
1364 case isInternal && cfg.RegistryAuthEnabled:
1365 if cfg.InternalRegistryUsername == "" || cfg.InternalRegistryPassword == "" {
1366 return log.WrapErr(fmt.Errorf("internal registry auth not configured"), "failed to pull image for internal deploy")
1367 }
1368 if err := s.pullImageWithRetry(ctx, func(ctx context.Context) error {
1369 return s.runtime.PullImageWithAuth(ctx, pullRef, cfg.InternalRegistryUsername, cfg.InternalRegistryPassword)
1370 }, isInternal); err != nil {
1371 return log.WrapErr(err, "failed to pull image with internal auth")
1372 }
1373 case isInternal:
1374 if err := s.pullImageWithRetry(ctx, func(ctx context.Context) error {
1375 return s.runtime.PullImage(ctx, pullRef)
1376 }, isInternal); err != nil {
1377 return log.WrapErr(err, "failed to pull image")
1378 }
1379 case cfg.RegistryAuthEnabled:
1380 if cfg.ServiceTokenUsername == "" || cfg.ServiceToken == "" {
1381 return log.WrapErr(fmt.Errorf("registry service token not configured"), "failed to pull image for registry auth")
1382 }
1383 if err := s.runtime.PullImageWithAuth(ctx, pullRef, cfg.ServiceTokenUsername, cfg.ServiceToken); err != nil {
1384 return log.WrapErr(err, "failed to pull image with auth")
1385 }
1386 default:
1387 if err := s.runtime.PullImage(ctx, pullRef); err != nil {
1388 return log.WrapErr(err, "failed to pull image")
1389 }
1390 }
1391
1392 return nil
1393}
1394
1395func (s *Service) pullImageWithRetry(ctx context.Context, pullFn func(context.Context) error, isInternal bool) error {
1396 log := zerowrap.FromCtx(ctx)
1397
1398 for attempt := 1; attempt <= internalPullMaxAttempts; attempt++ {
1399 err := pullFn(ctx)
1400 if err == nil {
1401 return nil
1402 }
1403 if !isInternal || !isConnectionRefusedError(err) || attempt == internalPullMaxAttempts {
1404 return fmt.Errorf("internal image pull failed after %d attempts: %w", attempt, err)
1405 }
1406
1407 backoff := time.Duration(attempt) * time.Second
1408 log.Warn().
1409 Err(err).
1410 Int("attempt", attempt).
1411 Dur("backoff", backoff).
1412 Msg("internal image pull failed with connection refused, retrying")
1413
1414 select {
1415 case <-time.After(backoff):
1416 case <-ctx.Done():
1417 return ctx.Err()
1418 }
1419 }
1420 return nil
1421}
1422
1423func (s *Service) tagImageIfNeeded(ctx context.Context, sourceRef, targetRef string) error {
1424 if sourceRef == targetRef {
1425 return nil
1426 }
1427
1428 // Cannot create a tag with a digest reference - Docker/Podman doesn't allow it.
1429 // When using digest references (image@sha256:...), skip tagging as the image
1430 // is already available by its digest.
1431 if strings.Contains(targetRef, "@sha256:") {
1432 return nil
1433 }
1434
1435 log := zerowrap.FromCtx(ctx)
1436 if err := s.runtime.TagImage(ctx, sourceRef, targetRef); err != nil {
1437 return log.WrapErr(err, "failed to tag image from pull reference")
1438 }
1439 return nil
1440}
1441
1442func (s *Service) loadEnvironment(ctx context.Context, domainName, imageRef string) ([]string, error) {
1443 log := zerowrap.FromCtx(ctx)
1444
1445 userEnvVars, err := s.envLoader.LoadEnv(ctx, domainName)
1446 if err != nil {
1447 return nil, log.WrapErr(err, "failed to load environment variables")
1448 }
1449
1450 dockerfileEnvVars, err := s.runtime.InspectImageEnv(ctx, imageRef)
1451 if err != nil {
1452 log.WrapErr(err, "failed to inspect image environment")
1453 dockerfileEnvVars = []string{}
1454 }
1455
1456 return mergeEnvironmentVariables(dockerfileEnvVars, userEnvVars), nil
1457}
1458
1459func (s *Service) setupVolumes(ctx context.Context, domainName, imageRef string) (map[string]string, error) {
1460 s.mu.RLock()
1461 cfg := s.config
1462 s.mu.RUnlock()
1463
1464 log := zerowrap.FromCtx(ctx)
1465 volumes := make(map[string]string)
1466
1467 if !cfg.VolumeAutoCreate {
1468 return volumes, nil
1469 }
1470
1471 volumePaths, err := s.runtime.InspectImageVolumes(ctx, imageRef)
1472 if err != nil {
1473 log.WrapErr(err, "failed to inspect image volumes")
1474 return volumes, nil
1475 }
1476
1477 for _, path := range volumePaths {
1478 name := generateVolumeName(cfg.VolumePrefix, domainName, path)
1479
1480 exists, err := s.runtime.VolumeExists(ctx, name)
1481 if err != nil {
1482 log.WrapErrWithFields(err, "failed to check volume", map[string]any{"volume": name})
1483 continue
1484 }
1485
1486 if !exists {
1487 if err := s.runtime.CreateVolume(ctx, name); err != nil {
1488 log.WrapErrWithFields(err, "failed to create volume", map[string]any{"volume": name})
1489 continue
1490 }
1491 log.Info().Str("volume", name).Str(zerowrap.FieldPath, path).Msg("created volume")
1492 }
1493
1494 volumes[path] = name
1495 }
1496
1497 return volumes, nil
1498}
1499
1500func (s *Service) getNetworkForApp(domainName string) string {
1501 s.mu.RLock()
1502 cfg := s.config
1503 s.mu.RUnlock()
1504
1505 if !cfg.NetworkIsolation {
1506 return "bridge"
1507 }
1508
1509 for groupName, domains := range cfg.NetworkGroups {
1510 if slices.Contains(domains, domainName) {
1511 return s.generateNetworkName(groupName)
1512 }
1513 }
1514
1515 return s.generateNetworkName(domainName)
1516}
1517
1518func (s *Service) generateNetworkName(identifier string) string {
1519 s.mu.RLock()
1520 cfg := s.config
1521 s.mu.RUnlock()
1522
1523 return fmt.Sprintf("%s-%s", cfg.NetworkPrefix, strings.ReplaceAll(identifier, ".", "-"))
1524}
1525
1526func (s *Service) createNetworkIfNeeded(ctx context.Context, networkName string) error {
1527 if networkName == "bridge" || networkName == "default" {
1528 return nil
1529 }
1530
1531 ctx = zerowrap.CtxWithField(ctx, "network", networkName)
1532 log := zerowrap.FromCtx(ctx)
1533
1534 exists, err := s.runtime.NetworkExists(ctx, networkName)
1535 if err != nil {
1536 return log.WrapErr(err, "failed to check network existence")
1537 }
1538
1539 if !exists {
1540 if err := s.runtime.CreateNetwork(ctx, networkName, map[string]string{"driver": "bridge"}); err != nil {
1541 return log.WrapErr(err, "failed to create network")
1542 }
1543 log.Info().Msg("created network for app isolation")
1544 }
1545
1546 return nil
1547}
1548
1549func (s *Service) cleanupOrphanedContainers(ctx context.Context, domainName string, skipContainerID string) error {
1550 log := zerowrap.FromCtx(ctx)
1551 expectedName := fmt.Sprintf("gordon-%s", domainName)
1552 expectedNewName := expectedName + "-new"
1553 expectedNextName := expectedName + "-next"
1554
1555 allContainers, err := s.runtime.ListContainers(ctx, true)
1556 if err != nil {
1557 return err
1558 }
1559
1560 for _, c := range allContainers {
1561 if (c.Name == expectedName || c.Name == expectedNewName || c.Name == expectedNextName) && c.ID != skipContainerID {
1562 log.Info().Str(zerowrap.FieldEntityID, c.ID).Str(zerowrap.FieldStatus, c.Status).Msg("found orphaned container, removing")
1563
1564 if err := s.runtime.StopContainer(ctx, c.ID); err != nil {
1565 log.WrapErrWithFields(err, "failed to stop orphaned container", map[string]any{zerowrap.FieldEntityID: c.ID})
1566 }
1567
1568 if err := s.runtime.RemoveContainer(ctx, c.ID, true); err != nil {
1569 log.WrapErrWithFields(err, "failed to remove orphaned container", map[string]any{zerowrap.FieldEntityID: c.ID})
1570 }
1571 }
1572 }
1573
1574 return nil
1575}
1576
1577func (s *Service) cleanupVolumesForDomain(_ context.Context, _ string) error {
1578 return nil
1579}
1580
1581func (s *Service) cleanupNetworkIfEmpty(ctx context.Context, networkName string) error {
1582 if networkName == "bridge" || networkName == "default" {
1583 return nil
1584 }
1585
1586 ctx = zerowrap.CtxWithField(ctx, "network", networkName)
1587 log := zerowrap.FromCtx(ctx)
1588
1589 networks, err := s.runtime.ListNetworks(ctx)
1590 if err != nil {
1591 return log.WrapErr(err, "failed to list networks")
1592 }
1593
1594 for _, n := range networks {
1595 if n.Name == networkName && len(n.Containers) == 0 {
1596 if err := s.runtime.RemoveNetwork(ctx, networkName); err != nil {
1597 return err
1598 }
1599 log.Info().Msg("cleaned up empty network")
1600 break
1601 }
1602 }
1603
1604 return nil
1605}
1606
1607func (s *Service) deployAttachments(ctx context.Context, domainName, networkName string) error {
1608 // Collect attachments from both domain-specific and network group configs
1609 attachments := s.resolveAttachmentsForDomain(domainName)
1610 if len(attachments) == 0 {
1611 return nil
1612 }
1613
1614 log := zerowrap.FromCtx(ctx)
1615 var deployed []string // track successfully deployed attachment IDs for rollback
1616 for _, svc := range attachments {
1617 if err := s.deployAttachedService(ctx, domainName, svc, networkName); err != nil {
1618 log.WrapErrWithFields(err, "failed to deploy attachment", map[string]any{zerowrap.FieldService: svc, "domain": domainName})
1619
1620 // Rollback: clean up already-deployed attachments
1621 for _, id := range deployed {
1622 if stopErr := s.runtime.StopContainer(ctx, id); stopErr != nil {
1623 log.Warn().Err(stopErr).Str("attachment_id", id).Msg("rollback: failed to stop attachment")
1624 }
1625 if rmErr := s.runtime.RemoveContainer(ctx, id, true); rmErr != nil {
1626 log.Warn().Err(rmErr).Str("attachment_id", id).Msg("rollback: failed to remove attachment")
1627 }
1628 }
1629 // Deregister rolled-back attachments
1630 if len(deployed) > 0 {
1631 s.mu.Lock()
1632 remaining := s.attachments[domainName]
1633 filtered := remaining[:0]
1634 rollbackSet := make(map[string]bool, len(deployed))
1635 for _, id := range deployed {
1636 rollbackSet[id] = true
1637 }
1638 for _, id := range remaining {
1639 if !rollbackSet[id] {
1640 filtered = append(filtered, id)
1641 }
1642 }
1643 s.attachments[domainName] = filtered
1644 s.mu.Unlock()
1645 }
1646
1647 return fmt.Errorf("failed to deploy attachment %q (rolled back %d already-deployed)", svc, len(deployed))
1648 }
1649 // Collect IDs of attachments tracked under this domain after successful deploy
1650 s.mu.RLock()
1651 ids := s.attachments[domainName]
1652 if len(ids) > 0 {
1653 latest := ids[len(ids)-1]
1654 deployed = append(deployed, latest)
1655 }
1656 s.mu.RUnlock()
1657 }
1658 return nil
1659}
1660
1661// resolveAttachmentsForDomain returns attachments for a domain by checking:
1662// 1. Direct domain attachments (attachments[domain])
1663// 2. Network group attachments (attachments[group] where domain is in network_groups[group])
1664func (s *Service) resolveAttachmentsForDomain(domainName string) []string {
1665 s.mu.RLock()
1666 cfg := s.config
1667 s.mu.RUnlock()
1668
1669 seen := make(map[string]bool)
1670 var result []string
1671
1672 // First, add domain-specific attachments
1673 if domainAttachments, ok := cfg.Attachments[domainName]; ok {
1674 for _, img := range domainAttachments {
1675 if !seen[img] {
1676 seen[img] = true
1677 result = append(result, img)
1678 }
1679 }
1680 }
1681
1682 // Then, find which network group this domain belongs to and add group attachments
1683 for groupName, domains := range cfg.NetworkGroups {
1684 if slices.Contains(domains, domainName) {
1685 if groupAttachments, ok := cfg.Attachments[groupName]; ok {
1686 for _, img := range groupAttachments {
1687 if !seen[img] {
1688 seen[img] = true
1689 result = append(result, img)
1690 }
1691 }
1692 }
1693 break // Domain can only be in one network group
1694 }
1695 }
1696
1697 return result
1698}
1699
1700func (s *Service) getAttachmentsForDomain(ctx context.Context, domainName string) []domain.Attachment {
1701 containers, err := s.runtime.ListContainers(ctx, true)
1702 if err != nil {
1703 return nil
1704 }
1705
1706 return s.filterAttachments(ctx, containers, domainName)
1707}
1708
1709// getAllAttachments fetches all containers once and returns attachments grouped by domain.
1710// This avoids N+1 queries when listing multiple routes.
1711func (s *Service) getAllAttachments(ctx context.Context) map[string][]domain.Attachment {
1712 containers, err := s.runtime.ListContainers(ctx, true)
1713 if err != nil {
1714 return nil
1715 }
1716
1717 result := make(map[string][]domain.Attachment)
1718 for _, container := range containers {
1719 if container.Labels == nil {
1720 continue
1721 }
1722 if container.Labels[domain.LabelAttachment] != "true" {
1723 continue
1724 }
1725 ownerDomain := container.Labels[domain.LabelAttachedTo]
1726 if ownerDomain == "" {
1727 continue
1728 }
1729 image := container.Image
1730 serviceName := container.Name
1731 if labelImage, ok := container.Labels[domain.LabelImage]; ok && labelImage != "" {
1732 image = labelImage
1733 serviceName = extractServiceName(labelImage)
1734 }
1735 // Get the container's network for display in the routes table
1736 network := ""
1737 if networkName, err := s.runtime.GetContainerNetwork(ctx, container.ID); err == nil {
1738 network = networkName
1739 }
1740 attachment := domain.Attachment{
1741 Name: serviceName,
1742 Image: s.stripRegistryPrefix(image),
1743 ContainerID: container.ID,
1744 Status: container.Status,
1745 Network: network,
1746 Ports: append([]int(nil), container.Ports...),
1747 }
1748 result[ownerDomain] = append(result[ownerDomain], attachment)
1749 }
1750
1751 return result
1752}
1753
1754// filterAttachments extracts attachments for a specific domain from a container list.
1755func (s *Service) filterAttachments(ctx context.Context, containers []*domain.Container, domainName string) []domain.Attachment {
1756 attachments := make([]domain.Attachment, 0)
1757 for _, container := range containers {
1758 if container.Labels == nil {
1759 continue
1760 }
1761 if container.Labels[domain.LabelAttachment] != "true" {
1762 continue
1763 }
1764 if container.Labels[domain.LabelAttachedTo] != domainName {
1765 continue
1766 }
1767 image := container.Image
1768 serviceName := container.Name
1769 if labelImage, ok := container.Labels[domain.LabelImage]; ok && labelImage != "" {
1770 image = labelImage
1771 serviceName = extractServiceName(labelImage)
1772 }
1773 // Get the container's network for display in the routes table
1774 network := ""
1775 if networkName, err := s.runtime.GetContainerNetwork(ctx, container.ID); err == nil {
1776 network = networkName
1777 }
1778 attachment := domain.Attachment{
1779 Name: serviceName,
1780 Image: s.stripRegistryPrefix(image),
1781 ContainerID: container.ID,
1782 Status: container.Status,
1783 Network: network,
1784 Ports: append([]int(nil), container.Ports...),
1785 }
1786 attachments = append(attachments, attachment)
1787 }
1788
1789 return attachments
1790}
1791
1792func (s *Service) deployAttachedService(ctx context.Context, ownerDomain, serviceImage, networkName string) error {
1793 log := zerowrap.FromCtx(ctx)
1794
1795 // Parse service name from image (e.g., "my-postgres:latest" → "postgres")
1796 serviceName := extractServiceName(serviceImage)
1797 containerName := fmt.Sprintf("gordon-%s-%s", sanitizeName(ownerDomain), serviceName)
1798
1799 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
1800 "attachment": serviceName,
1801 "container_name": containerName,
1802 "owner_domain": ownerDomain,
1803 })
1804
1805 // Check if already running (idempotent)
1806 // Try new name first, then fall back to legacy name for backwards compatibility
1807 existingContainer := s.findContainerByName(ctx, containerName)
1808 if existingContainer == nil {
1809 // Try legacy naming scheme for backwards compatibility during upgrades
1810 containerNameLegacy := fmt.Sprintf("gordon-%s-%s", sanitizeNameLegacy(ownerDomain), serviceName)
1811 existingContainer = s.findContainerByName(ctx, containerNameLegacy)
1812 if existingContainer != nil {
1813 log.Info().Str("container_name", containerNameLegacy).Msg("found attachment with legacy naming, will be replaced")
1814 // Remove the old container and recreate with new name
1815 if err := s.runtime.StopContainer(ctx, existingContainer.ID); err != nil {
1816 return log.WrapErr(err, "failed to stop legacy attachment container")
1817 }
1818 if err := s.runtime.RemoveContainer(ctx, existingContainer.ID, true); err != nil {
1819 return log.WrapErr(err, "failed to remove legacy attachment container")
1820 }
1821 existingContainer = nil // Clear so we create new one below
1822 }
1823 }
1824
1825 if existingContainer != nil && existingContainer.Status == string(domain.ContainerStatusRunning) {
1826 log.Debug().Str("container_name", containerName).Msg("attachment already running, skipping")
1827 return nil
1828 }
1829
1830 // Remove existing stopped container if present
1831 if existingContainer != nil {
1832 log.Info().Str("container_name", containerName).Msg("removing stopped attachment container")
1833 if err := s.runtime.RemoveContainer(ctx, existingContainer.ID, true); err != nil {
1834 return log.WrapErr(err, "failed to remove existing attachment container")
1835 }
1836 }
1837
1838 log.Info().Str(zerowrap.FieldService, serviceImage).Msg("deploying attached service")
1839
1840 // Ensure image (canonical ref is returned; internal pulls may use the local registry).
1841 imageRef := s.buildImageRef(serviceImage)
1842 actualImageRef, err := s.ensureImage(ctx, imageRef)
1843 if err != nil {
1844 return err
1845 }
1846
1847 // Get image metadata
1848 exposedPorts, err := s.runtime.GetImageExposedPorts(ctx, actualImageRef)
1849 if err != nil {
1850 log.WrapErr(err, "failed to get exposed ports for attachment, using defaults")
1851 exposedPorts = []int{}
1852 }
1853
1854 // Setup volumes (attachments need persistent data)
1855 volumes, err := s.setupVolumes(ctx, containerName, actualImageRef)
1856 if err != nil {
1857 log.WrapErr(err, "failed to setup volumes for attachment")
1858 volumes = make(map[string]string)
1859 }
1860
1861 // Load environment (attachment-specific env file)
1862 envVars, err := s.loadEnvironment(ctx, containerName, actualImageRef)
1863 if err != nil {
1864 log.WrapErr(err, "failed to load environment for attachment")
1865 envVars = []string{}
1866 }
1867
1868 // Create container with attachment labels (use actual ref, track original in labels)
1869 config := &domain.ContainerConfig{
1870 Image: actualImageRef,
1871 Name: containerName,
1872 Hostname: serviceName, // Internal DNS: postgres, redis, etc.
1873 Ports: exposedPorts,
1874 Env: envVars,
1875 Volumes: volumes,
1876 NetworkMode: networkName, // Same network as main app
1877 Labels: map[string]string{
1878 "gordon.managed": "true",
1879 "gordon.attachment": "true",
1880 "gordon.attached-to": ownerDomain,
1881 "gordon.image": serviceImage,
1882 },
1883 }
1884
1885 container, err := s.runtime.CreateContainer(ctx, config)
1886 if err != nil {
1887 return log.WrapErr(err, "failed to create attachment container")
1888 }
1889
1890 // Start container
1891 if err := s.runtime.StartContainer(ctx, container.ID); err != nil {
1892 s.runtime.RemoveContainer(ctx, container.ID, true)
1893 return log.WrapErr(err, "failed to start attachment container")
1894 }
1895
1896 // Track attachment
1897 s.mu.Lock()
1898 s.attachments[ownerDomain] = append(s.attachments[ownerDomain], container.ID)
1899 s.mu.Unlock()
1900
1901 // Start log collection for attachment
1902 s.startLogCollection(ctx, container.ID, containerName)
1903
1904 log.Info().Str(zerowrap.FieldEntityID, container.ID).Msg("attachment deployed successfully")
1905 return nil
1906}
1907
1908// Utility functions
1909
1910func normalizeImageRef(image string) string {
1911 // Extract repository from image reference, handling port numbers correctly.
1912 // Examples:
1913 // nginx:latest -> docker.io/library/nginx
1914 // user/repo:tag -> docker.io/user/repo
1915 // localhost:5000/image:tag -> localhost:5000/image
1916 // registry.example.com/image:tag -> registry.example.com/image
1917 //
1918 // Note: This function assumes valid Docker image references. Edge cases like
1919 // "registry.com:5000" (registry with port but no image name) are not valid
1920 // image references per Docker's naming conventions, which require at least
1921 // one path component after the registry (e.g., "registry.com:5000/image").
1922
1923 // Find the tag separator: the last colon that isn't part of a port number.
1924 // A colon is part of a port if there's no slash after it until the next colon.
1925 repo := image
1926 lastColon := strings.LastIndex(image, ":")
1927 if lastColon != -1 {
1928 afterColon := image[lastColon+1:]
1929 // If there's no slash after the colon, it's the tag separator
1930 if !strings.Contains(afterColon, "/") {
1931 repo = image[:lastColon]
1932 }
1933 }
1934
1935 if !strings.Contains(repo, "/") {
1936 return "docker.io/library/" + repo
1937 }
1938
1939 if strings.Count(repo, "/") == 1 && !strings.Contains(strings.Split(repo, "/")[0], ".") && !strings.Contains(strings.Split(repo, "/")[0], ":") {
1940 return "docker.io/" + repo
1941 }
1942
1943 return repo
1944}
1945
1946func generateVolumeName(prefix, domainName, volumePath string) string {
1947 return fmt.Sprintf("%s-%s-%s",
1948 prefix,
1949 strings.ReplaceAll(domainName, ".", "-"),
1950 strings.ReplaceAll(strings.Trim(volumePath, "/"), "/", "-"))
1951}
1952
1953func mergeEnvironmentVariables(dockerfileEnv, userEnv []string) []string {
1954 envMap := make(map[string]string)
1955
1956 for _, env := range dockerfileEnv {
1957 if k, v, ok := strings.Cut(env, "="); ok {
1958 envMap[k] = v
1959 }
1960 }
1961
1962 for _, env := range userEnv {
1963 if k, v, ok := strings.Cut(env, "="); ok {
1964 envMap[k] = v
1965 }
1966 }
1967
1968 result := make([]string, 0, len(envMap))
1969 for k, v := range envMap {
1970 result = append(result, k+"="+v)
1971 }
1972
1973 return result
1974}
1975
1976// extractServiceName gets service name from image reference.
1977// "my-postgres:latest" → "postgres", "redis:7" → "redis"
1978func extractServiceName(image string) string {
1979 // Remove tag
1980 parts := strings.Split(image, ":")
1981 name := parts[0]
1982
1983 // Remove registry prefix if present
1984 if strings.Contains(name, "/") {
1985 nameParts := strings.Split(name, "/")
1986 name = nameParts[len(nameParts)-1]
1987 }
1988
1989 // Remove common prefixes like "my-"
1990 name = strings.TrimPrefix(name, "my-")
1991
1992 return name
1993}
1994
1995// sanitizeName is a convenience wrapper around domain.SanitizeDomainForContainer.
1996func sanitizeName(d string) string {
1997 return domain.SanitizeDomainForContainer(d)
1998}
1999
2000// sanitizeNameLegacy is a convenience wrapper around domain.SanitizeDomainForContainerLegacy.
2001func sanitizeNameLegacy(d string) string {
2002 return domain.SanitizeDomainForContainerLegacy(d)
2003}
2004
2005func isContainerNotFoundError(err error) bool {
2006 if err == nil {
2007 return false
2008 }
2009 if errors.Is(err, domain.ErrContainerNotFound) {
2010 return true
2011 }
2012 msg := strings.ToLower(err.Error())
2013 return strings.Contains(msg, "no such container") ||
2014 strings.Contains(msg, "no container with name or id") ||
2015 strings.Contains(msg, "container not found")
2016}
2017
2018func isConnectionRefusedError(err error) bool {
2019 if err == nil {
2020 return false
2021 }
2022 return strings.Contains(strings.ToLower(err.Error()), "connection refused")
2023}
2024
2025// findContainerByName finds a container by its name.
2026func (s *Service) findContainerByName(ctx context.Context, name string) *domain.Container {
2027 containers, err := s.runtime.ListContainers(ctx, true)
2028 if err != nil {
2029 return nil
2030 }
2031
2032 for _, c := range containers {
2033 if c.Name == name {
2034 return c
2035 }
2036 }
2037 return nil
2038}
2039
2040func (s *Service) waitForReady(ctx context.Context, containerID string) error {
2041 if err := s.pollContainerRunning(ctx, containerID); err != nil {
2042 return err
2043 }
2044
2045 s.mu.RLock()
2046 cfg := s.config
2047 s.mu.RUnlock()
2048
2049 readinessMode := cfg.ReadinessMode
2050 if readinessMode == "" {
2051 readinessMode = "delay"
2052 }
2053
2054 switch readinessMode {
2055 case "delay":
2056 return s.waitForReadyByDelay(ctx, containerID)
2057 case "docker-health":
2058 _, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(ctx, containerID)
2059 if err != nil {
2060 return err
2061 }
2062 if !hasHealthcheck {
2063 return errors.New("no healthcheck detected")
2064 }
2065 return s.waitForHealthy(ctx, containerID, cfg.HealthTimeout)
2066 default: // auto
2067 _, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(ctx, containerID)
2068 if err != nil {
2069 return err
2070 }
2071 if hasHealthcheck {
2072 return s.waitForHealthy(ctx, containerID, cfg.HealthTimeout)
2073 }
2074 return s.waitForReadyByDelay(ctx, containerID)
2075 }
2076}
2077
2078// waitForReadyByDelay waits using the legacy running+delay strategy.
2079func (s *Service) waitForReadyByDelay(ctx context.Context, containerID string) error {
2080 log := zerowrap.FromCtx(ctx)
2081
2082 s.mu.RLock()
2083 cfg := s.config
2084 s.mu.RUnlock()
2085
2086 delay := cfg.ReadinessDelay
2087 if delay == 0 {
2088 delay = 5 * time.Second
2089 }
2090
2091 log.Debug().Dur("delay", delay).Msg("waiting for container readiness")
2092 select {
2093 case <-time.After(delay):
2094 case <-ctx.Done():
2095 return ctx.Err()
2096 }
2097
2098 // Verify still running after delay
2099 running, err := s.runtime.IsContainerRunning(ctx, containerID)
2100 if err != nil {
2101 return err
2102 }
2103 if running {
2104 return nil
2105 }
2106
2107 return s.waitForRecovery(ctx, containerID)
2108}
2109
2110func (s *Service) waitForHealthy(ctx context.Context, containerID string, timeout time.Duration) error {
2111 if timeout == 0 {
2112 timeout = 90 * time.Second
2113 }
2114 waitCtx, cancel := context.WithTimeout(ctx, timeout)
2115 defer cancel()
2116
2117 var lastStatus string
2118 for {
2119 status, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(waitCtx, containerID)
2120 if err != nil {
2121 return err
2122 }
2123 if !hasHealthcheck {
2124 return errors.New("no healthcheck detected")
2125 }
2126 // Treat empty health status as transitional startup. Some runtimes can
2127 // temporarily report an empty status before first probe results.
2128 if status == "" {
2129 status = "starting"
2130 }
2131 if status == "healthy" {
2132 return nil
2133 }
2134 lastStatus = status
2135
2136 select {
2137 case <-time.After(time.Second):
2138 case <-waitCtx.Done():
2139 return fmt.Errorf("container healthcheck timeout after %s (last status: %s)", timeout, lastStatus)
2140 }
2141 }
2142}
2143
2144// pollContainerRunning polls until the container is running (max 30 seconds).
2145func (s *Service) pollContainerRunning(ctx context.Context, containerID string) error {
2146 for i := 0; i < 30; i++ {
2147 running, err := s.runtime.IsContainerRunning(ctx, containerID)
2148 if err != nil {
2149 return err
2150 }
2151 if running {
2152 return nil
2153 }
2154 if i == 29 {
2155 return fmt.Errorf("container did not start within 30 seconds")
2156 }
2157 select {
2158 case <-time.After(time.Second):
2159 case <-ctx.Done():
2160 return ctx.Err()
2161 }
2162 }
2163 return fmt.Errorf("container did not start within 30 seconds")
2164}
2165
2166// waitForRecovery waits for a container to recover within the recovery window.
2167func (s *Service) waitForRecovery(ctx context.Context, containerID string) error {
2168 log := zerowrap.FromCtx(ctx)
2169 log.Warn().
2170 Dur("recovery_window", readinessRecoveryWindow).
2171 Msg("container not running after readiness delay, waiting for recovery")
2172
2173 deadline := time.Now().Add(readinessRecoveryWindow)
2174 for time.Now().Before(deadline) {
2175 select {
2176 case <-time.After(time.Second):
2177 case <-ctx.Done():
2178 return ctx.Err()
2179 }
2180
2181 running, err := s.runtime.IsContainerRunning(ctx, containerID)
2182 if err != nil {
2183 return err
2184 }
2185 if running {
2186 log.Info().Msg("container recovered during readiness recovery window")
2187 return nil
2188 }
2189 }
2190
2191 return fmt.Errorf("container not running after readiness delay and recovery window")
2192}
2193
2194// publishContainerDeployed publishes a container.deployed event.
2195func (s *Service) publishContainerDeployed(ctx context.Context, domainName, containerID string) {
2196 payload := &domain.ContainerEventPayload{
2197 ContainerID: containerID,
2198 Domain: domainName,
2199 Action: "deployed",
2200 }
2201
2202 if err := s.eventBus.Publish(domain.EventContainerDeployed, payload); err != nil {
2203 log := zerowrap.FromCtx(ctx)
2204 log.Warn().Err(err).Msg("failed to publish container deployed event")
2205 }
2206}
2207
2208// rewriteToRegistryDomain rewrites an image reference to use the configured registry domain.
2209// e.g., "myapp:latest" -> "registry.example.com/myapp:latest"
2210func rewriteToRegistryDomain(imageRef, registryDomain string) string {
2211 if registryDomain == "" {
2212 return imageRef
2213 }
2214
2215 prefix := registryDomain + "/"
2216 if strings.HasPrefix(imageRef, prefix) {
2217 return imageRef
2218 }
2219
2220 return prefix + imageRef
2221}
2222
2223// rewriteToLocalRegistry rewrites an image reference to use the local registry address.
2224// e.g., "registry.example.com/myapp:latest" -> "localhost:5000/myapp:latest"
2225func rewriteToLocalRegistry(imageRef, registryDomain string, registryPort int) string {
2226 if imageRef == "" {
2227 return imageRef
2228 }
2229
2230 localRegistry := fmt.Sprintf("localhost:%d", registryPort)
2231 localPrefix := localRegistry + "/"
2232 imageRef = strings.TrimPrefix(imageRef, localPrefix)
2233
2234 registryDomain = strings.TrimSuffix(registryDomain, "/")
2235 if registryDomain != "" {
2236 prefix := registryDomain + "/"
2237 imageRef = strings.TrimPrefix(imageRef, prefix)
2238 }
2239
2240 return localPrefix + imageRef
2241}