feat: add TCP and HTTP readiness probes
TCP probe retries every 500ms until port accepts connections. HTTP probe retries GET every 1s, expects 2xx/3xx. Both are universal readiness checks for deploy pipeline.
- commit
- d293fa63b194deda7d288592a0b33b62a473935e
- author
- brice · · 2026-03-02 16:23 UTC
- parent
- 867c255
- signature
- unsigned
2 files · +220 -0
A internal/usecase/container/readiness.go +68 -0
| @@ -0,0 +1,68 @@ | |||
| 1 | + | package container | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "context" | |
| 5 | + | "fmt" | |
| 6 | + | "net" | |
| 7 | + | "net/http" | |
| 8 | + | "time" | |
| 9 | + | ) | |
| 10 | + | ||
| 11 | + | // tcpProbe attempts a TCP connection to addr, retrying every 500ms until | |
| 12 | + | // success or timeout. This is the universal fallback readiness check — | |
| 13 | + | // it verifies the process is at least accepting connections. | |
| 14 | + | func tcpProbe(ctx context.Context, addr string, timeout time.Duration) error { | |
| 15 | + | deadline := time.Now().Add(timeout) | |
| 16 | + | for { | |
| 17 | + | if err := ctx.Err(); err != nil { | |
| 18 | + | return err | |
| 19 | + | } | |
| 20 | + | if time.Now().After(deadline) { | |
| 21 | + | return fmt.Errorf("TCP probe timeout after %s: %s not reachable", timeout, addr) | |
| 22 | + | } | |
| 23 | + | ||
| 24 | + | conn, err := net.DialTimeout("tcp", addr, time.Second) | |
| 25 | + | if err == nil { | |
| 26 | + | conn.Close() | |
| 27 | + | return nil | |
| 28 | + | } | |
| 29 | + | ||
| 30 | + | select { | |
| 31 | + | case <-time.After(500 * time.Millisecond): | |
| 32 | + | case <-ctx.Done(): | |
| 33 | + | return ctx.Err() | |
| 34 | + | } | |
| 35 | + | } | |
| 36 | + | } | |
| 37 | + | ||
| 38 | + | // httpProbe performs HTTP GET requests to url, retrying every 1s until a | |
| 39 | + | // 2xx/3xx response or timeout. Used when gordon.health label is set. | |
| 40 | + | func httpProbe(ctx context.Context, url string, timeout time.Duration) error { | |
| 41 | + | client := &http.Client{Timeout: 2 * time.Second} | |
| 42 | + | deadline := time.Now().Add(timeout) | |
| 43 | + | var lastStatus int | |
| 44 | + | ||
| 45 | + | for { | |
| 46 | + | if err := ctx.Err(); err != nil { | |
| 47 | + | return err | |
| 48 | + | } | |
| 49 | + | if time.Now().After(deadline) { | |
| 50 | + | return fmt.Errorf("HTTP probe timeout after %s: last status %d from %s", timeout, lastStatus, url) | |
| 51 | + | } | |
| 52 | + | ||
| 53 | + | resp, err := client.Get(url) | |
| 54 | + | if err == nil { | |
| 55 | + | lastStatus = resp.StatusCode | |
| 56 | + | resp.Body.Close() | |
| 57 | + | if lastStatus >= 200 && lastStatus < 400 { | |
| 58 | + | return nil | |
| 59 | + | } | |
| 60 | + | } | |
| 61 | + | ||
| 62 | + | select { | |
| 63 | + | case <-time.After(time.Second): | |
| 64 | + | case <-ctx.Done(): | |
| 65 | + | return ctx.Err() | |
| 66 | + | } | |
| 67 | + | } | |
| 68 | + | } | |
| @@ -0,0 +1,68 @@ | |||||
| 1 | + | package container | |||
| 2 | + | ||||
| 3 | + | import ( | |||
| 4 | + | "context" | |||
| 5 | + | "fmt" | |||
| 6 | + | "net" | |||
| 7 | + | "net/http" | |||
| 8 | + | "time" | |||
| 9 | + | ) | |||
| 10 | + | ||||
| 11 | + | // tcpProbe attempts a TCP connection to addr, retrying every 500ms until | |||
| 12 | + | // success or timeout. This is the universal fallback readiness check — | |||
| 13 | + | // it verifies the process is at least accepting connections. | |||
| 14 | + | func tcpProbe(ctx context.Context, addr string, timeout time.Duration) error { | |||
| 15 | + | deadline := time.Now().Add(timeout) | |||
| 16 | + | for { | |||
| 17 | + | if err := ctx.Err(); err != nil { | |||
| 18 | + | return err | |||
| 19 | + | } | |||
| 20 | + | if time.Now().After(deadline) { | |||
| 21 | + | return fmt.Errorf("TCP probe timeout after %s: %s not reachable", timeout, addr) | |||
| 22 | + | } | |||
| 23 | + | ||||
| 24 | + | conn, err := net.DialTimeout("tcp", addr, time.Second) | |||
| 25 | + | if err == nil { | |||
| 26 | + | conn.Close() | |||
| 27 | + | return nil | |||
| 28 | + | } | |||
| 29 | + | ||||
| 30 | + | select { | |||
| 31 | + | case <-time.After(500 * time.Millisecond): | |||
| 32 | + | case <-ctx.Done(): | |||
| 33 | + | return ctx.Err() | |||
| 34 | + | } | |||
| 35 | + | } | |||
| 36 | + | } | |||
| 37 | + | ||||
| 38 | + | // httpProbe performs HTTP GET requests to url, retrying every 1s until a | |||
| 39 | + | // 2xx/3xx response or timeout. Used when gordon.health label is set. | |||
| 40 | + | func httpProbe(ctx context.Context, url string, timeout time.Duration) error { | |||
| 41 | + | client := &http.Client{Timeout: 2 * time.Second} | |||
| 42 | + | deadline := time.Now().Add(timeout) | |||
| 43 | + | var lastStatus int | |||
| 44 | + | ||||
| 45 | + | for { | |||
| 46 | + | if err := ctx.Err(); err != nil { | |||
| 47 | + | return err | |||
| 48 | + | } | |||
| 49 | + | if time.Now().After(deadline) { | |||
| 50 | + | return fmt.Errorf("HTTP probe timeout after %s: last status %d from %s", timeout, lastStatus, url) | |||
| 51 | + | } | |||
| 52 | + | ||||
| 53 | + | resp, err := client.Get(url) | |||
| 54 | + | if err == nil { | |||
| 55 | + | lastStatus = resp.StatusCode | |||
| 56 | + | resp.Body.Close() | |||
| 57 | + | if lastStatus >= 200 && lastStatus < 400 { | |||
| 58 | + | return nil | |||
| 59 | + | } | |||
| 60 | + | } | |||
| 61 | + | ||||
| 62 | + | select { | |||
| 63 | + | case <-time.After(time.Second): | |||
| 64 | + | case <-ctx.Done(): | |||
| 65 | + | return ctx.Err() | |||
| 66 | + | } | |||
| 67 | + | } | |||
| 68 | + | } | |||
A internal/usecase/container/readiness_test.go +152 -0
| @@ -0,0 +1,152 @@ | |||
| 1 | + | package container | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "context" | |
| 5 | + | "net" | |
| 6 | + | "net/http" | |
| 7 | + | "net/http/httptest" | |
| 8 | + | "sync/atomic" | |
| 9 | + | "testing" | |
| 10 | + | "time" | |
| 11 | + | ||
| 12 | + | "github.com/stretchr/testify/assert" | |
| 13 | + | "github.com/stretchr/testify/require" | |
| 14 | + | ) | |
| 15 | + | ||
| 16 | + | // --- TCP Probe Tests --- | |
| 17 | + | ||
| 18 | + | func TestTCPProbe_Success(t *testing.T) { | |
| 19 | + | ln, err := net.Listen("tcp", "127.0.0.1:0") | |
| 20 | + | require.NoError(t, err) | |
| 21 | + | defer ln.Close() | |
| 22 | + | ||
| 23 | + | // Accept connections in background so the probe succeeds | |
| 24 | + | go func() { | |
| 25 | + | for { | |
| 26 | + | conn, err := ln.Accept() | |
| 27 | + | if err != nil { | |
| 28 | + | return | |
| 29 | + | } | |
| 30 | + | conn.Close() | |
| 31 | + | } | |
| 32 | + | }() | |
| 33 | + | ||
| 34 | + | ctx := testContext() | |
| 35 | + | err = tcpProbe(ctx, ln.Addr().String(), 5*time.Second) | |
| 36 | + | assert.NoError(t, err) | |
| 37 | + | } | |
| 38 | + | ||
| 39 | + | func TestTCPProbe_Timeout(t *testing.T) { | |
| 40 | + | // Use a port that nothing listens on (port 1 is privileged, won't have a listener) | |
| 41 | + | ctx := testContext() | |
| 42 | + | err := tcpProbe(ctx, "127.0.0.1:1", 200*time.Millisecond) | |
| 43 | + | assert.Error(t, err) | |
| 44 | + | assert.Contains(t, err.Error(), "TCP probe timeout") | |
| 45 | + | } | |
| 46 | + | ||
| 47 | + | func TestTCPProbe_ContextCancelled(t *testing.T) { | |
| 48 | + | ctx, cancel := context.WithCancel(testContext()) | |
| 49 | + | cancel() | |
| 50 | + | err := tcpProbe(ctx, "127.0.0.1:1", 5*time.Second) | |
| 51 | + | assert.Error(t, err) | |
| 52 | + | } | |
| 53 | + | ||
| 54 | + | func TestTCPProbe_DelayedListener(t *testing.T) { | |
| 55 | + | ln, err := net.Listen("tcp", "127.0.0.1:0") | |
| 56 | + | require.NoError(t, err) | |
| 57 | + | // Close initially so probe fails at first | |
| 58 | + | addr := ln.Addr().String() | |
| 59 | + | ln.Close() | |
| 60 | + | ||
| 61 | + | // Re-open after 1 second | |
| 62 | + | go func() { | |
| 63 | + | time.Sleep(time.Second) | |
| 64 | + | newLn, err := net.Listen("tcp", addr) | |
| 65 | + | if err != nil { | |
| 66 | + | return | |
| 67 | + | } | |
| 68 | + | defer newLn.Close() | |
| 69 | + | for { | |
| 70 | + | conn, err := newLn.Accept() | |
| 71 | + | if err != nil { | |
| 72 | + | return | |
| 73 | + | } | |
| 74 | + | conn.Close() | |
| 75 | + | } | |
| 76 | + | }() | |
| 77 | + | ||
| 78 | + | ctx := testContext() | |
| 79 | + | err = tcpProbe(ctx, addr, 5*time.Second) | |
| 80 | + | assert.NoError(t, err) | |
| 81 | + | } | |
| 82 | + | ||
| 83 | + | // --- HTTP Probe Tests --- | |
| 84 | + | ||
| 85 | + | func TestHTTPProbe_Success(t *testing.T) { | |
| 86 | + | srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | |
| 87 | + | if r.URL.Path == "/healthz" { | |
| 88 | + | w.WriteHeader(200) | |
| 89 | + | return | |
| 90 | + | } | |
| 91 | + | w.WriteHeader(404) | |
| 92 | + | })) | |
| 93 | + | defer srv.Close() | |
| 94 | + | ||
| 95 | + | ctx := testContext() | |
| 96 | + | err := httpProbe(ctx, srv.URL+"/healthz", 5*time.Second) | |
| 97 | + | assert.NoError(t, err) | |
| 98 | + | } | |
| 99 | + | ||
| 100 | + | func TestHTTPProbe_ServerError_ThenSuccess(t *testing.T) { | |
| 101 | + | var calls int32 | |
| 102 | + | srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | |
| 103 | + | n := atomic.AddInt32(&calls, 1) | |
| 104 | + | if n < 3 { | |
| 105 | + | w.WriteHeader(503) | |
| 106 | + | return | |
| 107 | + | } | |
| 108 | + | w.WriteHeader(200) | |
| 109 | + | })) | |
| 110 | + | defer srv.Close() | |
| 111 | + | ||
| 112 | + | ctx := testContext() | |
| 113 | + | err := httpProbe(ctx, srv.URL+"/healthz", 10*time.Second) | |
| 114 | + | assert.NoError(t, err) | |
| 115 | + | assert.GreaterOrEqual(t, atomic.LoadInt32(&calls), int32(3)) | |
| 116 | + | } | |
| 117 | + | ||
| 118 | + | func TestHTTPProbe_Timeout(t *testing.T) { | |
| 119 | + | srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | |
| 120 | + | w.WriteHeader(503) | |
| 121 | + | })) | |
| 122 | + | defer srv.Close() | |
| 123 | + | ||
| 124 | + | ctx := testContext() | |
| 125 | + | err := httpProbe(ctx, srv.URL+"/healthz", 500*time.Millisecond) | |
| 126 | + | assert.Error(t, err) | |
| 127 | + | assert.Contains(t, err.Error(), "HTTP probe timeout") | |
| 128 | + | } | |
| 129 | + | ||
| 130 | + | func TestHTTPProbe_ContextCancelled(t *testing.T) { | |
| 131 | + | srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | |
| 132 | + | w.WriteHeader(503) | |
| 133 | + | })) | |
| 134 | + | defer srv.Close() | |
| 135 | + | ||
| 136 | + | ctx, cancel := context.WithCancel(testContext()) | |
| 137 | + | cancel() | |
| 138 | + | err := httpProbe(ctx, srv.URL+"/healthz", 5*time.Second) | |
| 139 | + | assert.Error(t, err) | |
| 140 | + | } | |
| 141 | + | ||
| 142 | + | func TestHTTPProbe_RedirectIsSuccess(t *testing.T) { | |
| 143 | + | srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | |
| 144 | + | w.WriteHeader(302) | |
| 145 | + | })) | |
| 146 | + | defer srv.Close() | |
| 147 | + | ||
| 148 | + | ctx := testContext() | |
| 149 | + | // Disable follow redirects for the test — the probe should treat 3xx as success | |
| 150 | + | err := httpProbe(ctx, srv.URL+"/healthz", 5*time.Second) | |
| 151 | + | assert.NoError(t, err) | |
| 152 | + | } | |
| @@ -0,0 +1,152 @@ | |||||
| 1 | + | package container | |||
| 2 | + | ||||
| 3 | + | import ( | |||
| 4 | + | "context" | |||
| 5 | + | "net" | |||
| 6 | + | "net/http" | |||
| 7 | + | "net/http/httptest" | |||
| 8 | + | "sync/atomic" | |||
| 9 | + | "testing" | |||
| 10 | + | "time" | |||
| 11 | + | ||||
| 12 | + | "github.com/stretchr/testify/assert" | |||
| 13 | + | "github.com/stretchr/testify/require" | |||
| 14 | + | ) | |||
| 15 | + | ||||
| 16 | + | // --- TCP Probe Tests --- | |||
| 17 | + | ||||
| 18 | + | func TestTCPProbe_Success(t *testing.T) { | |||
| 19 | + | ln, err := net.Listen("tcp", "127.0.0.1:0") | |||
| 20 | + | require.NoError(t, err) | |||
| 21 | + | defer ln.Close() | |||
| 22 | + | ||||
| 23 | + | // Accept connections in background so the probe succeeds | |||
| 24 | + | go func() { | |||
| 25 | + | for { | |||
| 26 | + | conn, err := ln.Accept() | |||
| 27 | + | if err != nil { | |||
| 28 | + | return | |||
| 29 | + | } | |||
| 30 | + | conn.Close() | |||
| 31 | + | } | |||
| 32 | + | }() | |||
| 33 | + | ||||
| 34 | + | ctx := testContext() | |||
| 35 | + | err = tcpProbe(ctx, ln.Addr().String(), 5*time.Second) | |||
| 36 | + | assert.NoError(t, err) | |||
| 37 | + | } | |||
| 38 | + | ||||
| 39 | + | func TestTCPProbe_Timeout(t *testing.T) { | |||
| 40 | + | // Use a port that nothing listens on (port 1 is privileged, won't have a listener) | |||
| 41 | + | ctx := testContext() | |||
| 42 | + | err := tcpProbe(ctx, "127.0.0.1:1", 200*time.Millisecond) | |||
| 43 | + | assert.Error(t, err) | |||
| 44 | + | assert.Contains(t, err.Error(), "TCP probe timeout") | |||
| 45 | + | } | |||
| 46 | + | ||||
| 47 | + | func TestTCPProbe_ContextCancelled(t *testing.T) { | |||
| 48 | + | ctx, cancel := context.WithCancel(testContext()) | |||
| 49 | + | cancel() | |||
| 50 | + | err := tcpProbe(ctx, "127.0.0.1:1", 5*time.Second) | |||
| 51 | + | assert.Error(t, err) | |||
| 52 | + | } | |||
| 53 | + | ||||
| 54 | + | func TestTCPProbe_DelayedListener(t *testing.T) { | |||
| 55 | + | ln, err := net.Listen("tcp", "127.0.0.1:0") | |||
| 56 | + | require.NoError(t, err) | |||
| 57 | + | // Close initially so probe fails at first | |||
| 58 | + | addr := ln.Addr().String() | |||
| 59 | + | ln.Close() | |||
| 60 | + | ||||
| 61 | + | // Re-open after 1 second | |||
| 62 | + | go func() { | |||
| 63 | + | time.Sleep(time.Second) | |||
| 64 | + | newLn, err := net.Listen("tcp", addr) | |||
| 65 | + | if err != nil { | |||
| 66 | + | return | |||
| 67 | + | } | |||
| 68 | + | defer newLn.Close() | |||
| 69 | + | for { | |||
| 70 | + | conn, err := newLn.Accept() | |||
| 71 | + | if err != nil { | |||
| 72 | + | return | |||
| 73 | + | } | |||
| 74 | + | conn.Close() | |||
| 75 | + | } | |||
| 76 | + | }() | |||
| 77 | + | ||||
| 78 | + | ctx := testContext() | |||
| 79 | + | err = tcpProbe(ctx, addr, 5*time.Second) | |||
| 80 | + | assert.NoError(t, err) | |||
| 81 | + | } | |||
| 82 | + | ||||
| 83 | + | // --- HTTP Probe Tests --- | |||
| 84 | + | ||||
| 85 | + | func TestHTTPProbe_Success(t *testing.T) { | |||
| 86 | + | srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | |||
| 87 | + | if r.URL.Path == "/healthz" { | |||
| 88 | + | w.WriteHeader(200) | |||
| 89 | + | return | |||
| 90 | + | } | |||
| 91 | + | w.WriteHeader(404) | |||
| 92 | + | })) | |||
| 93 | + | defer srv.Close() | |||
| 94 | + | ||||
| 95 | + | ctx := testContext() | |||
| 96 | + | err := httpProbe(ctx, srv.URL+"/healthz", 5*time.Second) | |||
| 97 | + | assert.NoError(t, err) | |||
| 98 | + | } | |||
| 99 | + | ||||
| 100 | + | func TestHTTPProbe_ServerError_ThenSuccess(t *testing.T) { | |||
| 101 | + | var calls int32 | |||
| 102 | + | srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | |||
| 103 | + | n := atomic.AddInt32(&calls, 1) | |||
| 104 | + | if n < 3 { | |||
| 105 | + | w.WriteHeader(503) | |||
| 106 | + | return | |||
| 107 | + | } | |||
| 108 | + | w.WriteHeader(200) | |||
| 109 | + | })) | |||
| 110 | + | defer srv.Close() | |||
| 111 | + | ||||
| 112 | + | ctx := testContext() | |||
| 113 | + | err := httpProbe(ctx, srv.URL+"/healthz", 10*time.Second) | |||
| 114 | + | assert.NoError(t, err) | |||
| 115 | + | assert.GreaterOrEqual(t, atomic.LoadInt32(&calls), int32(3)) | |||
| 116 | + | } | |||
| 117 | + | ||||
| 118 | + | func TestHTTPProbe_Timeout(t *testing.T) { | |||
| 119 | + | srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | |||
| 120 | + | w.WriteHeader(503) | |||
| 121 | + | })) | |||
| 122 | + | defer srv.Close() | |||
| 123 | + | ||||
| 124 | + | ctx := testContext() | |||
| 125 | + | err := httpProbe(ctx, srv.URL+"/healthz", 500*time.Millisecond) | |||
| 126 | + | assert.Error(t, err) | |||
| 127 | + | assert.Contains(t, err.Error(), "HTTP probe timeout") | |||
| 128 | + | } | |||
| 129 | + | ||||
| 130 | + | func TestHTTPProbe_ContextCancelled(t *testing.T) { | |||
| 131 | + | srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | |||
| 132 | + | w.WriteHeader(503) | |||
| 133 | + | })) | |||
| 134 | + | defer srv.Close() | |||
| 135 | + | ||||
| 136 | + | ctx, cancel := context.WithCancel(testContext()) | |||
| 137 | + | cancel() | |||
| 138 | + | err := httpProbe(ctx, srv.URL+"/healthz", 5*time.Second) | |||
| 139 | + | assert.Error(t, err) | |||
| 140 | + | } | |||
| 141 | + | ||||
| 142 | + | func TestHTTPProbe_RedirectIsSuccess(t *testing.T) { | |||
| 143 | + | srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | |||
| 144 | + | w.WriteHeader(302) | |||
| 145 | + | })) | |||
| 146 | + | defer srv.Close() | |||
| 147 | + | ||||
| 148 | + | ctx := testContext() | |||
| 149 | + | // Disable follow redirects for the test — the probe should treat 3xx as success | |||
| 150 | + | err := httpProbe(ctx, srv.URL+"/healthz", 5*time.Second) | |||
| 151 | + | assert.NoError(t, err) | |||
| 152 | + | } | |||