fix: drain in-flight registry requests before shutdown

gracefulShutdown now stops TLS and proxy frontends first, then calls
DrainRegistryInFlight (25s timeout) to wait for active manifest pushes
to complete before stopping the registry backend. Fixes 503 errors on
gordon push during Gordon restarts.

commit: 2c769c5f9317868193819f43253d732557e60fe7
author: brice · 7mo ago · 2026-03-01 11:10 UTC
committer: brice · 7mo ago · 2026-03-01 11:14 UTC
parent: b93860a
signature: unsigned

2 files · +24 -7

M internal/app/run.go +19 -7


@@ -1750,7 +1750,7 @@ func runServers(ctx context.Context, v *viper.Viper, cfg Config, registryHandler
1750	1750		syncAndAutoStart(ctx, svc, log)
1751	1751
1752	1752		waitForShutdown(ctx, errChan, reloadChan, deployChan, eventBus, log)
1753		-	gracefulShutdown(registrySrv, proxySrv, tlsSrv, containerSvc, log)
	1753	+	gracefulShutdown(registrySrv, proxySrv, tlsSrv, containerSvc, svc.proxySvc, log)
1754	1754		return nil
1755	1755		}
1756	1756

@@ -1950,17 +1950,14 @@ func waitForShutdown(ctx context.Context, errChan <-chan error, reloadChan, depl
1950	1950
1951	1951		// gracefulShutdown stops HTTP servers with a 30s timeout, then shuts down
1952	1952		// the container service and cleans up runtime files.
1953		-	func gracefulShutdown(registrySrv, proxySrv, tlsSrv http.Server, containerSvc container.Service, log zerowrap.Logger) {
	1953	+	func gracefulShutdown(registrySrv, proxySrv, tlsSrv http.Server, containerSvc container.Service, proxySvc *proxy.Service, log zerowrap.Logger) {
1954	1954		log.Info().Msg("shutting down Gordon...")
1955	1955
1956	1956		shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 30*time.Second)
1957	1957		defer shutdownCancel()
1958	1958
1959		-	// Shutdown order matters: stop ingress frontends (TLS, proxy) before the
1960		-	// registry backend. If the backend stops first, the proxy—still alive—will
1961		-	// forward registry traffic to a dead backend and return 503 on in-flight
1962		-	// manifest pushes. Drain order: tlsSrv → proxySrv → registrySrv.
1963		-	for _, srv := range []*http.Server{tlsSrv, proxySrv, registrySrv} {
	1959	+	// Phase 1: Stop ingress frontends (TLS, then proxy) — no new traffic accepted
	1960	+	for _, srv := range []*http.Server{tlsSrv, proxySrv} {
1964	1961		if srv == nil {
1965	1962		continue
1966	1963		}

@@ -1969,6 +1966,21 @@ func gracefulShutdown(registrySrv, proxySrv, tlsSrv http.Server, containerSvc
1969	1966		}
1970	1967		}
1971	1968
	1969	+	// Phase 2: Drain in-flight registry push sessions before stopping the backend
	1970	+	if proxySvc != nil {
	1971	+	log.Info().Msg("draining in-flight registry requests...")
	1972	+	if drained := proxySvc.DrainRegistryInFlight(25 * time.Second); !drained {
	1973	+	log.Warn().Int64("in_flight", proxySvc.RegistryInFlight()).Msg("registry drain timed out; some in-flight pushes may be interrupted")
	1974	+	}
	1975	+	}
	1976	+
	1977	+	// Phase 3: Stop the registry backend
	1978	+	if registrySrv != nil {
	1979	+	if err := registrySrv.Shutdown(shutdownCtx); err != nil {
	1980	+	log.Warn().Err(err).Str("addr", registrySrv.Addr).Msg("server shutdown error")
	1981	+	}
	1982	+	}
	1983	+
1972	1984		containerSvc.StopMonitor()
1973	1985
1974	1986		if err := containerSvc.Shutdown(shutdownCtx); err != nil {

 	syncAndAutoStart(ctx, svc, log)
 	waitForShutdown(ctx, errChan, reloadChan, deployChan, eventBus, log)
-	gracefulShutdown(registrySrv, proxySrv, tlsSrv, containerSvc, log)
+	gracefulShutdown(registrySrv, proxySrv, tlsSrv, containerSvc, svc.proxySvc, log)
 	return nil
 }
 // gracefulShutdown stops HTTP servers with a 30s timeout, then shuts down
 // the container service and cleans up runtime files.
-func gracefulShutdown(registrySrv, proxySrv, tlsSrv *http.Server, containerSvc *container.Service, log zerowrap.Logger) {
+func gracefulShutdown(registrySrv, proxySrv, tlsSrv *http.Server, containerSvc *container.Service, proxySvc *proxy.Service, log zerowrap.Logger) {
 	log.Info().Msg("shutting down Gordon...")
 	shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer shutdownCancel()
-	// Shutdown order matters: stop ingress frontends (TLS, proxy) before the
+	// Phase 1: Stop ingress frontends (TLS, then proxy) — no new traffic accepted
-	// registry backend. If the backend stops first, the proxy—still alive—will
+	for _, srv := range []*http.Server{tlsSrv, proxySrv} {
-	// forward registry traffic to a dead backend and return 503 on in-flight
-	// manifest pushes. Drain order: tlsSrv → proxySrv → registrySrv.
-	for _, srv := range []*http.Server{tlsSrv, proxySrv, registrySrv} {
 		if srv == nil {
 			continue
 		}
 		}
 	}
+	// Phase 2: Drain in-flight registry push sessions before stopping the backend
+	if proxySvc != nil {
+		log.Info().Msg("draining in-flight registry requests...")
+		if drained := proxySvc.DrainRegistryInFlight(25 * time.Second); !drained {
+			log.Warn().Int64("in_flight", proxySvc.RegistryInFlight()).Msg("registry drain timed out; some in-flight pushes may be interrupted")
+		}
+	}
+	// Phase 3: Stop the registry backend
+	if registrySrv != nil {
+		if err := registrySrv.Shutdown(shutdownCtx); err != nil {
+			log.Warn().Err(err).Str("addr", registrySrv.Addr).Msg("server shutdown error")
+		}
+	}
 	containerSvc.StopMonitor()
 	if err := containerSvc.Shutdown(shutdownCtx); err != nil {

M internal/usecase/proxy/service.go +5 -0


@@ -575,6 +575,11 @@ func (s *Service) trackInFlight(containerID string) func() {
575	575		}
576	576		}
577	577
	578	+	// RegistryInFlight returns the current count of active registry proxy requests.
	579	+	func (s *Service) RegistryInFlight() int64 {
	580	+	return s.registryInFlight.Load()
	581	+	}
	582	+
578	583		// DrainRegistryInFlight blocks until all in-flight registry proxy requests
579	584		// complete or the timeout elapses. Returns true if drained cleanly, false if
580	585		// timed out with requests still in flight. Call this before shutting down the


@@ -575,6 +575,11 @@ func (s *Service) trackInFlight(containerID string) func() {
575	}	575		}
576	}	576		}
577		577
		578	+	// RegistryInFlight returns the current count of active registry proxy requests.
		579	+	func (s *Service) RegistryInFlight() int64 {
		580	+	return s.registryInFlight.Load()
		581	+	}
		582	+
578	// DrainRegistryInFlight blocks until all in-flight registry proxy requests	583		// DrainRegistryInFlight blocks until all in-flight registry proxy requests
579	// complete or the timeout elapses. Returns true if drained cleanly, false if	584		// complete or the timeout elapses. Returns true if drained cleanly, false if
580	// timed out with requests still in flight. Call this before shutting down the	585		// timed out with requests still in flight. Call this before shutting down the