fix(deploy): cleanup failed containers on inspect errors and warn on drain timeout

commit: b0d058e7f5f5eb0681d9d65899d86a6c11981607
author: bnema · 8mo ago · 2026-02-07 21:23 UTC
parent: 7e326be
signature: GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email

2 files · +11 -2

M docs/config/index.md +1 -1


@@ -61,7 +61,7 @@ health_timeout = "90s" # Max wait for health-based readiness
61	61		readiness_delay = "5s" # Wait after running before ready
62	62		drain_mode = "auto" # auto, inflight, delay
63	63		drain_timeout = "30s" # Max wait for in-flight drain
64		-	drain_delay = "2s" # Wait after proxy invalidation before old stop
	64	+	drain_delay = "2s" # Wait after proxy invalidation before stopping the old container
65	65
66	66		# Logging
67	67		[logging]


@@ -61,7 +61,7 @@ health_timeout = "90s" # Max wait for health-based readiness
61		readiness_delay = "5s" # Wait after running before ready	61		readiness_delay = "5s" # Wait after running before ready
62		drain_mode = "auto" # auto, inflight, delay	62		drain_mode = "auto" # auto, inflight, delay
63		drain_timeout = "30s" # Max wait for in-flight drain	63		drain_timeout = "30s" # Max wait for in-flight drain
64	-	drain_delay = "2s" # Wait after proxy invalidation before old stop	64	+	drain_delay = "2s" # Wait after proxy invalidation before stopping the old container
65			65
66		# Logging	66		# Logging
67		[logging]	67		[logging]

M internal/usecase/container/service.go +10 -1


@@ -353,6 +353,7 @@ func (s *Service) createStartedContainer(ctx context.Context, route domain.Route
353	353
354	354		inspected, err := s.runtime.InspectContainer(ctx, newContainer.ID)
355	355		if err != nil {
	356	+	s.cleanupFailedContainer(ctx, newContainer.ID)
356	357		return nil, log.WrapErr(err, "failed to inspect started container")
357	358		}
358	359

@@ -396,6 +397,8 @@ func (s *Service) finalizePreviousContainer(ctx context.Context, domainName stri
396	397		}
397	398
398	399		func (s *Service) waitForDrain(ctx context.Context, oldContainerID string) {
	400	+	log := zerowrap.FromCtx(ctx)
	401	+
399	402		s.mu.RLock()
400	403		cfg := s.config
401	404		waiter := s.drainWaiter

@@ -412,7 +415,13 @@ func (s *Service) waitForDrain(ctx context.Context, oldContainerID string) {
412	415		if timeout == 0 {
413	416		timeout = 30 * time.Second
414	417		}
415		-	waiter.WaitForNoInFlight(ctx, oldContainerID, timeout)
	418	+	drained := waiter.WaitForNoInFlight(ctx, oldContainerID, timeout)
	419	+	if !drained {
	420	+	log.Warn().
	421	+	Str("old_container_id", oldContainerID).
	422	+	Dur("drain_timeout", timeout).
	423	+	Msg("drain wait timed out; old container may still have in-flight traffic")
	424	+	}
416	425		return
417	426		}
418	427

 	inspected, err := s.runtime.InspectContainer(ctx, newContainer.ID)
 	if err != nil {
+		s.cleanupFailedContainer(ctx, newContainer.ID)
 		return nil, log.WrapErr(err, "failed to inspect started container")
 	}
 }
 func (s *Service) waitForDrain(ctx context.Context, oldContainerID string) {
+	log := zerowrap.FromCtx(ctx)
 	s.mu.RLock()
 	cfg := s.config
 	waiter := s.drainWaiter
 		if timeout == 0 {
 			timeout = 30 * time.Second
 		}
-		waiter.WaitForNoInFlight(ctx, oldContainerID, timeout)
+		drained := waiter.WaitForNoInFlight(ctx, oldContainerID, timeout)
+		if !drained {
+			log.Warn().
+				Str("old_container_id", oldContainerID).
+				Dur("drain_timeout", timeout).
+				Msg("drain wait timed out; old container may still have in-flight traffic")
+		}
 		return
 	}