1package docker
2
3import (
4 "bytes"
5 "context"
6 "errors"
7 "fmt"
8 "io"
9 "math"
10 "strconv"
11 "strings"
12 "time"
13
14 "github.com/bnema/zerowrap"
15 cerrdefs "github.com/containerd/errdefs"
16 "github.com/moby/moby/api/pkg/stdcopy"
17 "github.com/moby/moby/api/types/container"
18 "github.com/moby/moby/client"
19
20 "github.com/bnema/gordon/internal/domain"
21)
22
23// DefaultNetworkProbeImage is the digest-pinned, multi-arch helper image
24// one bounded internal readiness session runs. It provides only a POSIX
25// shell with busybox nc/printf. Pin by digest, never by tag: the helper
26// must be byte-identical across hosts and engines. The image must be
27// pre-provisioned and available offline on the target host, because a
28// probe never pulls.
29const DefaultNetworkProbeImage = "alpine@sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc"
30
31// Bounded resources and windows for the probe helper. They exist only to
32// bound a misbehaving helper, not to size a workload.
33const (
34 probeHelperMemory = 32 << 20
35 probeHelperNanoCPUs = 500_000_000
36 probeHelperPIDs = 32
37 // probeHelperCleanup bounds a force-remove of one helper.
38 probeHelperCleanup = 30 * time.Second
39 probeMaxSeconds = 30
40 // probeHelperOrphanAge is how old a leftover probe helper must be
41 // before a later probe reclaims it. Live helpers last seconds, so this
42 // never touches an in-flight session, including a concurrent one.
43 probeHelperOrphanAge = 10 * time.Minute
44)
45
46// networkProbeArgv is the helper argv: a shell script plus positional
47// arguments, so the target address never appears in a shell string.
48type networkProbeArgv struct {
49 script string
50 args []string
51}
52
53// httpProbeScript sends one HTTP/1.0 request over a raw connection and
54// accepts only a 2xx/3xx status line. It never follows redirects and never
55// consults a proxy. The first line must be an HTTP status line: a
56// non-HTTP greeting that merely starts with 2 or 3 is never readiness. It
57// prints the observed status (000 when none) for diagnostics.
58const httpProbeScript = `path="$1"; tmo="$2"; ip="$3"; port="$4"
59line=$(printf 'GET %s HTTP/1.0\r\nHost: readiness\r\nConnection: close\r\n\r\n' "$path" | nc -w "$tmo" "$ip" "$port" | head -c 128 | head -n1)
60code=$(printf '%s\n' "$line" | awk '$1 ~ /^HTTP\/[0-9]+\.[0-9]+$/ && $2 ~ /^[0-9][0-9][0-9]$/ { print $2 }')
61case "$code" in
62 2??|3??) printf '%s\n' "$code"; exit 0;;
63 [0-9][0-9][0-9]) printf '%s\n' "$code"; exit 1;;
64 *) printf '%s\n' 000; exit 1;;
65esac`
66
67// tcpProbeScript accepts only an accepted connection.
68const tcpProbeScript = `nc -z -w "$1" "$2" "$3"`
69
70// ProbeContainerNetwork runs one bounded readiness session against one
71// declared internal container port. The helper joins only the exact
72// network in the request, targets the exact inspected endpoint IP (never
73// a service alias), and is always force-removed under an independent
74// cleanup context. An infrastructure failure returns an error and must
75// never be retried as an unhealthy attempt; an unhealthy target returns a
76// result with Ready=false.
77func (r *Runtime) ProbeContainerNetwork(ctx context.Context, request domain.ContainerNetworkProbeRequest) (result domain.ContainerNetworkProbeResult, retErr error) {
78 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
79 zerowrap.FieldLayer: "adapter",
80 zerowrap.FieldAdapter: "docker",
81 zerowrap.FieldAction: "ProbeContainerNetwork",
82 "network": request.Network,
83 })
84 log := zerowrap.FromCtx(ctx)
85
86 if err := request.Validate(); err != nil {
87 return domain.ContainerNetworkProbeResult{}, err
88 }
89 sessionCtx, sessionCancel := context.WithTimeout(ctx, request.Timeout)
90 defer sessionCancel()
91 targetIP, err := r.inspectProbeTarget(sessionCtx, request)
92 if err != nil {
93 return domain.ContainerNetworkProbeResult{}, err
94 }
95
96 helperID, cleanup, err := r.createProbeHelper(sessionCtx, request, targetIP)
97 if err != nil {
98 return domain.ContainerNetworkProbeResult{}, err
99 }
100 defer func() {
101 result, retErr = probeCleanupResult(log, cleanup, result, retErr)
102 }()
103
104 if _, err := r.client.ContainerStart(sessionCtx, helperID, client.ContainerStartOptions{}); err != nil {
105 return domain.ContainerNetworkProbeResult{}, log.WrapErr(err, "failed to start network probe helper")
106 }
107
108 timedOut, err := r.waitForProbeHelper(sessionCtx, helperID)
109 if err != nil {
110 return domain.ContainerNetworkProbeResult{}, err
111 }
112 // A bound expiry is a readiness failure, not an infrastructure
113 // failure. Return it before any post-session step, and never classify
114 // it through a context that just expired. Caller cancellation stays an
115 // error so the deployment stops immediately.
116 if timedOut {
117 return probeWaitTimeoutResult(ctx)
118 }
119
120 exitCode, stdout, err := r.probeHelperOutcome(sessionCtx, helperID)
121 if result, handled := classifyProbeStepError(sessionCtx, ctx, err); handled {
122 return result, err
123 }
124 // Revalidate identity before trusting the outcome: a target that
125 // restarted during the session must not report readiness for a
126 // generation that no longer exists. The caller may retry this as a
127 // candidate that is simply not ready yet.
128 finalIP, err := r.inspectProbeTarget(sessionCtx, request)
129 if result, handled := classifyProbeStepError(sessionCtx, ctx, err); handled {
130 return result, err
131 }
132 if finalIP != targetIP {
133 return domain.ContainerNetworkProbeResult{}, fmt.Errorf("deployment: probe target endpoint changed during readiness: %w", domain.ErrAppStateConflict)
134 }
135 if err := ctx.Err(); err != nil {
136 return domain.ContainerNetworkProbeResult{}, err
137 }
138
139 result = domain.ContainerNetworkProbeResult{Ready: exitCode == 0}
140 if request.Protocol == domain.ProbeProtocolHTTP {
141 result.Status = parseProbeStatus(stdout)
142 result.Ready = result.Ready && result.Status >= 200 && result.Status < 400
143 }
144 if !result.Ready {
145 result.Diagnostic = probeDiagnostic(request, result.Status)
146 }
147 return result, nil
148}
149
150func probeWaitTimeoutResult(parentCtx context.Context) (domain.ContainerNetworkProbeResult, error) {
151 if err := parentCtx.Err(); err != nil {
152 return domain.ContainerNetworkProbeResult{}, err
153 }
154 return probeTimeoutFailure(), nil
155}
156
157func classifyProbeStepError(sessionCtx, parentCtx context.Context, err error) (domain.ContainerNetworkProbeResult, bool) {
158 if err == nil {
159 return domain.ContainerNetworkProbeResult{}, false
160 }
161 if probeBoundExpired(sessionCtx, parentCtx) {
162 return probeTimeoutFailure(), true
163 }
164 return domain.ContainerNetworkProbeResult{}, true
165}
166
167// probeTimeoutFailure is the bounded readiness failure of a session whose
168// own bound expired. It is not an infrastructure error: the target may
169// simply be slow, so the caller may retry it as an unhealthy attempt.
170func probeTimeoutFailure() domain.ContainerNetworkProbeResult {
171 return domain.ContainerNetworkProbeResult{Diagnostic: "probe did not complete within its bound"}
172}
173
174// probeBoundExpired reports whether a post-wait probe step failed because
175// the session's own bound expired. A caller cancellation is never a bound
176// expiry: it must stay an error so the deployment stops immediately.
177func probeBoundExpired(sessionCtx, parentCtx context.Context) bool {
178 return parentCtx.Err() == nil && errors.Is(sessionCtx.Err(), context.DeadlineExceeded)
179}
180
181func probeCleanupResult(log zerowrap.Logger, cleanup func() error, result domain.ContainerNetworkProbeResult, retErr error) (domain.ContainerNetworkProbeResult, error) {
182 cleanErr := cleanup()
183 if cleanErr == nil {
184 return result, retErr
185 }
186 log.Warn().Err(cleanErr).Msg("failed to remove network probe helper")
187 if retErr != nil {
188 return domain.ContainerNetworkProbeResult{}, fmt.Errorf("%w: %v; prior probe error: %w", domain.ErrNetworkProbeCleanup, cleanErr, retErr)
189 }
190 return domain.ContainerNetworkProbeResult{}, fmt.Errorf("%w: %v", domain.ErrNetworkProbeCleanup, cleanErr)
191}
192
193// inspectProbeTarget resolves the exact endpoint IP of the target on the
194// requested network and enforces the expected execution start. It fails
195// closed, wrapping domain.ErrContainerNotFound or domain.ErrAppStateConflict
196// when the target is gone, not running, not attached to that network, or
197// has restarted, so the caller can tell "not ready yet" from a broken
198// helper.
199func (r *Runtime) inspectProbeTarget(ctx context.Context, request domain.ContainerNetworkProbeRequest) (string, error) {
200 inspectResult, err := r.client.ContainerInspect(ctx, request.TargetContainerID, client.ContainerInspectOptions{})
201 if err != nil {
202 if cerrdefs.IsNotFound(err) {
203 return "", fmt.Errorf("%w: %s", domain.ErrContainerNotFound, request.TargetContainerID)
204 }
205 return "", fmt.Errorf("deployment: inspect probe target: %w", err)
206 }
207 observed := inspectResult.Container
208 if observed.State == nil || observed.State.Status != container.StateRunning {
209 return "", fmt.Errorf("deployment: probe target %s is not running: %w", request.TargetContainerID, domain.ErrAppStateConflict)
210 }
211 startedAt, _ := time.Parse(time.RFC3339Nano, observed.State.StartedAt)
212 if startedAt.Year() <= 1 {
213 startedAt = time.Time{}
214 }
215 if !startedAt.Equal(request.ExpectedStartedAt) {
216 return "", fmt.Errorf("deployment: probe target %s execution changed during readiness: %w", request.TargetContainerID, domain.ErrAppStateConflict)
217 }
218 if observed.NetworkSettings == nil {
219 return "", fmt.Errorf("deployment: probe target %s has no network settings: %w", request.TargetContainerID, domain.ErrAppStateConflict)
220 }
221 endpoint, ok := observed.NetworkSettings.Networks[request.Network]
222 if !ok || endpoint == nil || !endpoint.IPAddress.IsValid() {
223 return "", fmt.Errorf("deployment: probe target %s is not attached to network %q: %w", request.TargetContainerID, request.Network, domain.ErrAppStateConflict)
224 }
225 return endpoint.IPAddress.String(), nil
226}
227
228// createProbeHelper creates the hardened helper container on only the
229// requested network. It has no host ports, mounts, volumes, environment,
230// or shared networks, runs non-root on a read-only rootfs with all
231// capabilities dropped, and never gains new privileges.
232func (r *Runtime) createProbeHelper(ctx context.Context, request domain.ContainerNetworkProbeRequest, targetIP string) (string, func() error, error) {
233 log := zerowrap.FromCtx(ctx)
234 // Reclaim helpers orphaned by a previous crash before adding another:
235 // in-process cleanup cannot run after SIGKILL, so a later probe is the
236 // only owner that can bound that accumulation.
237 r.sweepOrphanedProbeHelpers(ctx)
238
239 command := buildNetworkProbeCommand(request, targetIP)
240 entrypoint := append([]string{"sh", "-c", command.script, "sh"}, command.args...)
241
242 created, err := r.client.ContainerCreate(ctx, client.ContainerCreateOptions{
243 Config: &container.Config{
244 Image: DefaultNetworkProbeImage,
245 Entrypoint: entrypoint,
246 User: "65534:65534",
247 AttachStdout: true,
248 AttachStderr: true,
249 Labels: map[string]string{
250 domain.LabelPurpose: domain.PurposeNetworkProbe,
251 },
252 },
253 HostConfig: probeHelperHostConfig(request.Network),
254 Name: fmt.Sprintf("gordon-network-probe-%d", time.Now().UTC().UnixNano()),
255 })
256 if err != nil {
257 if cerrdefs.IsNotFound(err) {
258 return "", nil, log.WrapErr(fmt.Errorf(
259 "network probe helper image %s is not available on this host: pre-provision the pinned image, the probe never pulls: %w",
260 DefaultNetworkProbeImage, err), "failed to create network probe helper")
261 }
262 return "", nil, log.WrapErr(err, "failed to create network probe helper")
263 }
264
265 var cleanupErr error
266 var cleaned bool
267 cleanup := func() error {
268 if cleaned {
269 return cleanupErr
270 }
271 cleaned = true
272 // Independent bounded context: the session context may already be
273 // expired, but the helper must still be force-removed.
274 removeCtx, cancel := context.WithTimeout(context.Background(), probeHelperCleanup)
275 defer cancel()
276 cleanupErr = r.RemoveContainer(removeCtx, created.ID, true)
277 return cleanupErr
278 }
279 return created.ID, cleanup, nil
280}
281
282// sweepOrphanedProbeHelpers best-effort removes probe helpers older than
283// probeHelperOrphanAge. It is bounded to the probe purpose label, so it
284// can never touch a workload, a volume archive helper, or an in-flight
285// session. A sweep failure never fails the probe.
286func (r *Runtime) sweepOrphanedProbeHelpers(ctx context.Context) {
287 log := zerowrap.FromCtx(ctx)
288 list, err := r.client.ContainerList(ctx, client.ContainerListOptions{
289 All: true,
290 Filters: client.Filters{}.Add("label", domain.LabelPurpose+"="+domain.PurposeNetworkProbe),
291 })
292 if err != nil {
293 log.Debug().Err(err).Msg("network probe orphan sweep: list failed")
294 return
295 }
296 cutoff := time.Now().Add(-probeHelperOrphanAge)
297 for _, orphan := range list.Items {
298 if time.Unix(orphan.Created, 0).After(cutoff) {
299 continue // live session, leave it alone
300 }
301 if err := ctx.Err(); err != nil {
302 return
303 }
304 removeCtx, cancel := context.WithTimeout(ctx, probeHelperCleanup)
305 removeErr := r.RemoveContainer(removeCtx, orphan.ID, true)
306 cancel()
307 if removeErr != nil {
308 log.Debug().Err(removeErr).Msg("network probe orphan sweep: remove failed")
309 continue
310 }
311 log.Info().Str("container", orphan.ID).Msg("reclaimed orphaned network probe helper")
312 }
313}
314
315// probeHelperHostConfig is the smallest host configuration able to reach
316// the target's private network and run a shell probe. Every field that
317// would widen the helper's reach is left at its zero value.
318func probeHelperHostConfig(network string) *container.HostConfig {
319 pids := int64(probeHelperPIDs)
320 return &container.HostConfig{
321 AutoRemove: false,
322 NetworkMode: container.NetworkMode(network),
323 ReadonlyRootfs: true,
324 SecurityOpt: []string{"no-new-privileges:true"},
325 CapDrop: []string{"ALL"},
326 CapAdd: []string{},
327 Resources: container.Resources{
328 Memory: probeHelperMemory,
329 NanoCPUs: probeHelperNanoCPUs,
330 PidsLimit: &pids,
331 },
332 }
333}
334
335// waitForProbeHelper blocks until the helper exits or the bound expires.
336// A bound expiry is reported as a non-fatal timeout: the target may be
337// hanging, which is a readiness failure, not an infrastructure failure.
338func (r *Runtime) waitForProbeHelper(ctx context.Context, helperID string) (bool, error) {
339 waitResult := r.client.ContainerWait(ctx, helperID, client.ContainerWaitOptions{Condition: container.WaitConditionNotRunning})
340 select {
341 case err := <-waitResult.Error:
342 if err != nil {
343 if ctx.Err() != nil {
344 return true, nil
345 }
346 return false, fmt.Errorf("deployment: wait for network probe helper: %w", err)
347 }
348 return false, nil
349 case <-waitResult.Result:
350 return false, nil
351 case <-ctx.Done():
352 return true, nil
353 }
354}
355
356// probeHelperOutcome reads the helper's exit code and bounded stdout.
357func (r *Runtime) probeHelperOutcome(ctx context.Context, helperID string) (int, string, error) {
358 log := zerowrap.FromCtx(ctx)
359 logs, err := r.client.ContainerLogs(ctx, helperID, client.ContainerLogsOptions{ShowStdout: true, ShowStderr: true})
360 if err != nil {
361 return 0, "", log.WrapErr(err, "failed to read network probe helper logs")
362 }
363 defer func() { _ = logs.Close() }()
364
365 var stdout cappedProbeBuffer
366 if _, err := stdcopy.StdCopy(&stdout, io.Discard, logs); err != nil {
367 return 0, "", log.WrapErr(err, "failed to demux network probe helper logs")
368 }
369
370 inspectResult, err := r.client.ContainerInspect(ctx, helperID, client.ContainerInspectOptions{})
371 if err != nil {
372 return 0, "", fmt.Errorf("deployment: inspect network probe helper: %w", err)
373 }
374 if inspectResult.Container.State == nil {
375 return 0, "", fmt.Errorf("deployment: network probe helper has no state: %w", domain.ErrAppStateConflict)
376 }
377 return inspectResult.Container.State.ExitCode, stdout.String(), nil
378}
379
380type cappedProbeBuffer struct {
381 bytes.Buffer
382}
383
384func (b *cappedProbeBuffer) Write(p []byte) (int, error) {
385 const limit = 64
386 remaining := limit - b.Len()
387 if remaining > 0 {
388 _, _ = b.Buffer.Write(p[:min(len(p), remaining)])
389 }
390 return len(p), nil
391}
392
393// buildNetworkProbeCommand builds the helper argv for the requested
394// protocol. The nc bound is kept strictly shorter than the attempt bound,
395// so the helper exits while the caller context is still alive; it is
396// rounded to whole seconds because nc -w takes seconds, floored at one,
397// and clamped so one attempt can never outlive its bound.
398func buildNetworkProbeCommand(request domain.ContainerNetworkProbeRequest, targetIP string) networkProbeArgv {
399 seconds := int(math.Ceil(request.Timeout.Seconds())) - 1
400 if seconds < 1 {
401 seconds = 1
402 }
403 if seconds > probeMaxSeconds {
404 seconds = probeMaxSeconds
405 }
406 port := strconv.Itoa(request.Port)
407 if request.Protocol == domain.ProbeProtocolHTTP {
408 return networkProbeArgv{
409 script: httpProbeScript,
410 args: []string{request.Path, strconv.Itoa(seconds), targetIP, port},
411 }
412 }
413 return networkProbeArgv{
414 script: tcpProbeScript,
415 args: []string{strconv.Itoa(seconds), targetIP, port},
416 }
417}
418
419// parseProbeStatus extracts the HTTP status the helper observed; 0 when
420// none was read.
421func parseProbeStatus(stdout string) int {
422 status, err := strconv.Atoi(strings.TrimSpace(stdout))
423 if err != nil || status < 0 || status > 999 {
424 return 0
425 }
426 return status
427}
428
429// probeDiagnostic describes an unhealthy attempt without leaking addresses
430// or command output.
431func probeDiagnostic(request domain.ContainerNetworkProbeRequest, status int) string {
432 if request.Protocol == domain.ProbeProtocolHTTP {
433 if status > 0 {
434 return fmt.Sprintf("http status %d", status)
435 }
436 return "no http response"
437 }
438 return "tcp connection refused or timed out"
439}