fix: use HTTP alive probe as default readiness check instead of TCP

TCP-only readiness allowed premature traffic cutover — the new container
accepted TCP connections before the app was actually serving HTTP, causing
502/500s during zero-downtime deploys.

The readiness cascade now defaults to an HTTP GET on / (any HTTP response
= ready) before falling back to TCP. Also fixes log streaming context
cancellation by detaching from the deploy request context.

commit: a8e226f065704d1e49d53346716f4f54a04239b7
author: brice · 7mo ago · 2026-03-12 06:52 UTC
parent: 36dfce1
signature: unsigned

2 files · +67 -5

M internal/usecase/container/readiness.go +33 -0


@@ -135,3 +135,36 @@ func httpProbe(ctx context.Context, url string, timeout time.Duration) error {
135	135		}
136	136		return err
137	137		}
	138	+
	139	+	// httpAliveProbe performs HTTP GET requests to url, retrying every 1s until
	140	+	// any HTTP response is received or timeout. Unlike httpProbe, ANY status code
	141	+	// counts as success -- the goal is to verify the HTTP server is accepting and
	142	+	// processing requests, not that a specific endpoint is healthy.
	143	+	// Used as the default readiness check for web containers.
	144	+	func httpAliveProbe(ctx context.Context, url string, timeout time.Duration) error {
	145	+	client := &http.Client{
	146	+	CheckRedirect: func(req http.Request, via []http.Request) error {
	147	+	return http.ErrUseLastResponse
	148	+	},
	149	+	}
	150	+	deadline := time.Now().Add(timeout)
	151	+	err := probeLoop(ctx, deadline, time.Second, 2*time.Second, func(attemptCtx context.Context) (bool, error) {
	152	+	req, reqErr := http.NewRequestWithContext(attemptCtx, http.MethodGet, url, nil)
	153	+	if reqErr != nil {
	154	+	return false, reqErr
	155	+	}
	156	+
	157	+	resp, err := client.Do(req)
	158	+	if err != nil {
	159	+	// Connection refused, timeout, etc. -- not ready yet.
	160	+	return false, nil
	161	+	}
	162	+	resp.Body.Close()
	163	+	// Any HTTP response means the server is alive and processing.
	164	+	return true, nil
	165	+	})
	166	+	if errors.Is(err, errProbeTimeout) {
	167	+	return fmt.Errorf("HTTP alive probe timeout after %s: no HTTP response from %s", timeout, url)
	168	+	}
	169	+	return err
	170	+	}


@@ -135,3 +135,36 @@ func httpProbe(ctx context.Context, url string, timeout time.Duration) error {
135	}	135		}
136	return err	136		return err
137	}	137		}
		138	+
		139	+	// httpAliveProbe performs HTTP GET requests to url, retrying every 1s until
		140	+	// any HTTP response is received or timeout. Unlike httpProbe, ANY status code
		141	+	// counts as success -- the goal is to verify the HTTP server is accepting and
		142	+	// processing requests, not that a specific endpoint is healthy.
		143	+	// Used as the default readiness check for web containers.
		144	+	func httpAliveProbe(ctx context.Context, url string, timeout time.Duration) error {
		145	+	client := &http.Client{
		146	+	CheckRedirect: func(req http.Request, via []http.Request) error {
		147	+	return http.ErrUseLastResponse
		148	+	},
		149	+	}
		150	+	deadline := time.Now().Add(timeout)
		151	+	err := probeLoop(ctx, deadline, time.Second, 2*time.Second, func(attemptCtx context.Context) (bool, error) {
		152	+	req, reqErr := http.NewRequestWithContext(attemptCtx, http.MethodGet, url, nil)
		153	+	if reqErr != nil {
		154	+	return false, reqErr
		155	+	}
		156	+
		157	+	resp, err := client.Do(req)
		158	+	if err != nil {
		159	+	// Connection refused, timeout, etc. -- not ready yet.
		160	+	return false, nil
		161	+	}
		162	+	resp.Body.Close()
		163	+	// Any HTTP response means the server is alive and processing.
		164	+	return true, nil
		165	+	})
		166	+	if errors.Is(err, errProbeTimeout) {
		167	+	return fmt.Errorf("HTTP alive probe timeout after %s: no HTTP response from %s", timeout, url)
		168	+	}
		169	+	return err
		170	+	}

M internal/usecase/container/service.go +34 -5


@@ -1330,7 +1330,7 @@ func (s *Service) startLogCollection(ctx context.Context, containerID, domainNam
1330	1330		return
1331	1331		}
1332	1332
1333		-	if err := s.logWriter.StartLogging(ctx, containerID, domainName, logStream); err != nil {
	1333	+	if err := s.logWriter.StartLogging(context.WithoutCancel(ctx), containerID, domainName, logStream); err != nil {
1334	1334		log.Warn().Err(err).Msg("failed to start container log collection")
1335	1335		logStream.Close()
1336	1336		}

@@ -2297,8 +2297,9 @@ func (s *Service) waitForReady(ctx context.Context, containerID string, containe
2297	2297		// readinessCascade auto-detects the strongest available readiness signal:
2298	2298		// 1. Docker healthcheck (if present) → wait for healthy status
2299	2299		// 2. HTTP probe (if gordon.health label set) → GET until 2xx/3xx
2300		-	// 3. TCP probe (if port info available) → connect until accepted
2301		-	// 4. Delay fallback (last resort) → waitForReadyByDelay
	2300	+	// 3. Default HTTP probe (GET / on exposed port) → wait for 2xx/3xx
	2301	+	// 4. TCP probe (if port info available) → connect until accepted
	2302	+	// 5. Delay fallback (last resort) → waitForReadyByDelay
2302	2303		func (s Service) readinessCascade(ctx context.Context, containerID string, containerConfig domain.ContainerConfig, cfg Config) error {
2303	2304		log := zerowrap.FromCtx(ctx)
2304	2305

@@ -2317,12 +2318,17 @@ func (s *Service) readinessCascade(ctx context.Context, containerID string, cont
2317	2318		return probeErr
2318	2319		}
2319	2320
2320		-	// 3. TCP probe (if port info available)
	2321	+	// 3. Default HTTP probe on root path (if port info available)
	2322	+	if probed, probeErr := s.tryDefaultHTTPProbe(ctx, containerID, containerConfig, cfg); probed {
	2323	+	return probeErr
	2324	+	}
	2325	+
	2326	+	// 4. TCP probe (if port info available)
2321	2327		if probed, probeErr := s.tryTCPProbe(ctx, containerID, containerConfig, cfg); probed {
2322	2328		return probeErr
2323	2329		}
2324	2330
2325		-	// 4. Delay fallback
	2331	+	// 5. Delay fallback
2326	2332		log.Info().Msg("readiness cascade: using delay fallback")
2327	2333		return s.waitForReadyByDelay(ctx, containerID)
2328	2334		}

@@ -2352,6 +2358,29 @@ func (s *Service) tryHTTPProbe(ctx context.Context, containerID string, containe
2352	2358		return true, httpProbe(ctx, url, timeout)
2353	2359		}
2354	2360
	2361	+	// tryDefaultHTTPProbe attempts an HTTP GET on "/" using the container's
	2362	+	// exposed port. This is the default readiness check for web containers —
	2363	+	// it verifies the app is actually serving HTTP, not just accepting TCP.
	2364	+	// Returns (true, err) if the probe was attempted, (false, nil) if skipped.
	2365	+	func (s Service) tryDefaultHTTPProbe(ctx context.Context, containerID string, containerConfig domain.ContainerConfig, cfg Config) (bool, error) {
	2366	+	log := zerowrap.FromCtx(ctx)
	2367	+	if containerConfig == nil \|\| len(containerConfig.Ports) == 0 {
	2368	+	return false, nil
	2369	+	}
	2370	+	ip, port, probeErr := s.resolveContainerEndpoint(ctx, containerID)
	2371	+	if probeErr != nil \|\| ip == "" \|\| port <= 0 {
	2372	+	log.Debug().Err(probeErr).Msg("readiness cascade: default HTTP probe skipped, could not resolve container endpoint")
	2373	+	return false, nil
	2374	+	}
	2375	+	url := fmt.Sprintf("http://%s:%d/", ip, port)
	2376	+	timeout := cfg.HTTPProbeTimeout
	2377	+	if timeout == 0 {
	2378	+	timeout = 60 * time.Second
	2379	+	}
	2380	+	log.Info().Str("url", url).Dur("timeout", timeout).Msg("readiness cascade: using default HTTP alive probe")
	2381	+	return true, httpAliveProbe(ctx, url, timeout)
	2382	+	}
	2383	+
2355	2384		// tryTCPProbe attempts a TCP probe if port info is available.
2356	2385		// Returns (true, err) if the probe was attempted, (false, nil) if skipped.
2357	2386		func (s Service) tryTCPProbe(ctx context.Context, containerID string, containerConfig domain.ContainerConfig, cfg Config) (bool, error) {

 		return
 	}
-	if err := s.logWriter.StartLogging(ctx, containerID, domainName, logStream); err != nil {
+	if err := s.logWriter.StartLogging(context.WithoutCancel(ctx), containerID, domainName, logStream); err != nil {
 		log.Warn().Err(err).Msg("failed to start container log collection")
 		logStream.Close()
 	}
 // readinessCascade auto-detects the strongest available readiness signal:
 //  1. Docker healthcheck (if present) → wait for healthy status
 //  2. HTTP probe (if gordon.health label set) → GET until 2xx/3xx
-//  3. TCP probe (if port info available) → connect until accepted
+//  3. Default HTTP probe (GET / on exposed port) → wait for 2xx/3xx
-//  4. Delay fallback (last resort) → waitForReadyByDelay
+//  4. TCP probe (if port info available) → connect until accepted
+//  5. Delay fallback (last resort) → waitForReadyByDelay
 func (s *Service) readinessCascade(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) error {
 	log := zerowrap.FromCtx(ctx)
 		return probeErr
 	}
-	// 3. TCP probe (if port info available)
+	// 3. Default HTTP probe on root path (if port info available)
+	if probed, probeErr := s.tryDefaultHTTPProbe(ctx, containerID, containerConfig, cfg); probed {
+		return probeErr
+	}
+	// 4. TCP probe (if port info available)
 	if probed, probeErr := s.tryTCPProbe(ctx, containerID, containerConfig, cfg); probed {
 		return probeErr
 	}
-	// 4. Delay fallback
+	// 5. Delay fallback
 	log.Info().Msg("readiness cascade: using delay fallback")
 	return s.waitForReadyByDelay(ctx, containerID)
 }
 	return true, httpProbe(ctx, url, timeout)
 }
+// tryDefaultHTTPProbe attempts an HTTP GET on "/" using the container's
+// exposed port. This is the default readiness check for web containers —
+// it verifies the app is actually serving HTTP, not just accepting TCP.
+// Returns (true, err) if the probe was attempted, (false, nil) if skipped.
+func (s *Service) tryDefaultHTTPProbe(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) (bool, error) {
+	log := zerowrap.FromCtx(ctx)
+	if containerConfig == nil || len(containerConfig.Ports) == 0 {
+		return false, nil
+	}
+	ip, port, probeErr := s.resolveContainerEndpoint(ctx, containerID)
+	if probeErr != nil || ip == "" || port <= 0 {
+		log.Debug().Err(probeErr).Msg("readiness cascade: default HTTP probe skipped, could not resolve container endpoint")
+		return false, nil
+	}
+	url := fmt.Sprintf("http://%s:%d/", ip, port)
+	timeout := cfg.HTTPProbeTimeout
+	if timeout == 0 {
+		timeout = 60 * time.Second
+	}
+	log.Info().Str("url", url).Dur("timeout", timeout).Msg("readiness cascade: using default HTTP alive probe")
+	return true, httpAliveProbe(ctx, url, timeout)
+}
 // tryTCPProbe attempts a TCP probe if port info is available.
 // Returns (true, err) if the probe was attempted, (false, nil) if skipped.
 func (s *Service) tryTCPProbe(ctx context.Context, containerID string, containerConfig *domain.ContainerConfig, cfg Config) (bool, error) {