2b9f5c9 refactor(registryproxy): simplify the proxy store, guest timing contract and size parsing · ermite-runner

M docs/cache-and-artifacts.md +18 -16

@@ -1,6 +1,6 @@
1		# Cache and artifacts	1		# Cache and artifacts
2			2
3	-	The runner serves the GitHub Actions cache and artifacts v4 to jobs. Both use one channel per sandbox.	3	+	The runner serves the GitHub Actions cache, artifacts v4 and a Docker Hub mirror to jobs. All three use one channel per sandbox.
4			4
5		## Channel	5		## Channel
6			6

@@ -84,11 +84,11 @@ Works with `actions/cache`, `setup-go` and `setup-node` with `cache:`, and anyth
84			84
85		The runner provides an anonymous, pull-only Docker Hub mirror on the same private	85		The runner provides an anonymous, pull-only Docker Hub mirror on the same private
86		channel. Each job's dockerd uses `http://127.0.0.1:9089` as a registry mirror.	86		channel. Each job's dockerd uses `http://127.0.0.1:9089` as a registry mirror.
87	-	Other registries are not cached; there is no HTTPS interception or guest network	87	+	Other registries are not cached. There is no HTTPS interception, and the mirror
88	-	policy change. Guest credentials are never forwarded upstream. Private pulls	88	+	needs no network policy rule. Guest credentials are never forwarded upstream. Private pulls
89		fall back to Docker Hub directly, where dockerd can use the job's credentials.	89		fall back to Docker Hub directly, where dockerd can use the job's credentials.
90			90
91	-	Verified sha256 blobs and manifests live under `<state>/registry/`. Tags are	91	+	Verified sha256 blobs and manifests live under `<state dir>/registry/`. Tags are
92		revalidated by an upstream HEAD on each request, with the client's Accept headers	92		revalidated by an upstream HEAD on each request, with the client's Accept headers
93		reduced to the known manifest media types. If Docker Hub is unreachable, times	93		reduced to the known manifest media types. If Docker Hub is unreachable, times
94		out, returns a server error or rate-limits (429), a previously cached tag can be	94		out, returns a server error or rate-limits (429), a previously cached tag can be

@@ -111,24 +111,26 @@ have at most 8 registry requests in progress; one more gets 503 at once.
111			111
112		Downloads have no total time limit, but they fail after 30 s without progress.	112		Downloads have no total time limit, but they fail after 30 s without progress.
113		Auth, manifest and HEAD requests are limited to 10 s. Auth and manifest requests	113		Auth, manifest and HEAD requests are limited to 10 s. Auth and manifest requests
114	-	never follow redirects; blob requests follow the CDN redirect without the bearer	114	+	never follow redirects; blob requests, HEAD included, follow the CDN redirect
115	-	token. Stored blobs support range requests. Cold blob HEAD requests fetch only	115	+	without the bearer token. Stored blobs support range requests. Cold blob HEAD
116	-	upstream metadata, not the layer body.	116	+	requests fetch only upstream metadata, not the layer body.
117			117
118		`ERMITE_RUNNER_REGISTRY_CACHE_MAX` defaults to `10G`. Blobs, manifests and tag	118		`ERMITE_RUNNER_REGISTRY_CACHE_MAX` defaults to `10G`. Blobs, manifests and tag
119	-	files (charged 4 KiB each) share one least-recently-used eviction; an entry that	119	+	files share one least-recently-used eviction; each is charged at least 4 KiB, so
120	-	is being served is never evicted. This cap is independent of	120	+	the number of files stays below the cap divided by 4 KiB. An entry that is being served is
			121	+	never evicted. This cap is independent of
121		`ERMITE_RUNNER_CACHE_MAX`. A zero size in any unit (`0`, `0M`, `0G`) disables only	122		`ERMITE_RUNNER_CACHE_MAX`. A zero size in any unit (`0`, `0M`, `0G`) disables only
122		the proxy. `ERMITE_RUNNER_CACHE=0` disables the whole channel and therefore also	123		the proxy. `ERMITE_RUNNER_CACHE=0` disables the whole channel and therefore also
123		the proxy (logged at startup). Dockerd retains its mirror setting and falls back	124		the proxy (logged at startup). Dockerd retains its mirror setting and falls back
124	-	to Docker Hub if the mirror is unavailable. No deployment is needed to clear	125	+	to Docker Hub if the mirror is unavailable. To clear the cache, stop the runner
125	-	entries: stop the runner before removing `<state>/registry/`.	126	+	and remove `<state dir>/registry/`.
126			127
127	-	Jobs with neither cache permissions nor artifacts still get the loopback channel	128	+	Jobs with neither cache permissions nor artifacts get the loopback channel for
128	-	for dockerd, but no `ACTIONS_CACHE_URL`, `ACTIONS_RESULTS_URL`, or	129	+	dockerd when the proxy is on, but no `ACTIONS_CACHE_URL`, `ACTIONS_RESULTS_URL`, or
129		`ACTIONS_RUNTIME_TOKEN` is exposed to their steps. Registry routes need no token:	130		`ACTIONS_RUNTIME_TOKEN` is exposed to their steps. Registry routes need no token:
130		only public content is served over a channel exclusive to that sandbox.	131		only public content is served over a channel exclusive to that sandbox.
131			132
132	-	The host logs one `registry manifest` line per request with `repo`, `ref`, `hit`	133	+	The host logs one `registry manifest` line per request with `job_id`, `repo`,
133	-	and `ms`. Blob misses log `registry blob` at debug level with `digest`, `bytes`	134	+	`ref`, `hit` and `ms`. Blob misses log `registry blob` at debug level with
134	-	and `ms`.	135	+	`job_id`, `digest`, `bytes` and `ms`; `job_id` is the job that started the
			136	+	download.

M docs/development.md +2 -1

 | `ERMITE_RUNNER_IT_E2E=1` | End-to-end job tests in `internal/e2e` |
 | `ERMITE_RUNNER_IT_NET=1` | Packet-level firewall test |
 | `ERMITE_RUNNER_IT_DOCKER_API=1` | Docker Engine calls against any daemon, including rootless |
+| `ERMITE_RUNNER_IT_REGISTRY=1` | Live Docker Hub pulls through the registry proxy with `curl` (`TestDockerHubCurl`). Needs internet access |
 | `ERMITE_RUNNER_IT_NODE_DIR` | Directory with `node-v24.21.0-linux-x64.tar.gz`, for offline hosts |
 | `ERMITE_RUNNER_TEST_IMAGE` | Image used by the container and end-to-end tests |
 its CPU budget to reproduce a small host without changing the boot sequence:
 ```sh
-docker run --name runner-timing --cpus=1 \
+docker run -d --name runner-timing --cpus=1 \
   --device /dev/kvm --device /dev/net/tun --cap-add NET_ADMIN \
   -v runner-timing-state:/var/lib/ermite-runner \
   --env-file runner.env -e ERMITE_RUNNER_LOG_FORMAT=json ermite-runner:<tag>

M docs/images.md +1 -0


@@ -29,6 +29,7 @@ scripts/build-image.sh tmp/image # builds the OCI image and tmp/image/base.ex
29	- `/home/runner/work` and `/opt/hostedtoolcache`, owned by `runner`. The tool cache holds the toolchains listed in `images/ubuntu-26.04/toolcache.lock` for the image architecture.	29		- `/home/runner/work` and `/opt/hostedtoolcache`, owned by `runner`. The tool cache holds the toolchains listed in `images/ubuntu-26.04/toolcache.lock` for the image architecture.
30	- `bash` and `sh`. `python3` and a `python` command for `shell: python`.	30		- `bash` and `sh`. `python3` and a `python` command for `shell: python`.
31	- `dockerd` and passwordless `sudo` for `runner`, for jobs that use Docker.	31		- `dockerd` and passwordless `sudo` for `runner`, for jobs that use Docker.
		32	+	- No `registry-mirrors` in `/etc/docker/daemon.json`. The runner starts dockerd with `--registry-mirror` (its Docker Hub mirror), and dockerd refuses to start when an option is set both as a flag and in that file.
32	- Common tools: git, curl, jq, build-essential, ca-certificates, tzdata, zip and unzip.	33		- Common tools: git, curl, jq, build-essential, ca-certificates, tzdata, zip and unzip.
33		34
34	`pwsh` is not in the image. A step that asks for it fails with `pwsh: command not found`.	35		`pwsh` is not in the image. A step that asks for it fails with `pwsh: command not found`.

M docs/service.md +4 -3


@@ -67,9 +67,10 @@ The security boundary of a job is its microVM or container, not the unit.
67			67
68		## Job timing	68		## Job timing
69			69
70	-	The existing `job finished` line carries integer milliseconds (all `*_ms`	70	+	The `job finished` line carries integer milliseconds (all `*_ms` fields).
71	-	fields). Phases that did not run are omitted; driver timings are available only	71	+	Phases that did not run are omitted, and the driver (`sb_*`) timings appear only
72	-	when sandbox start succeeds. Guest phases are omitted when they did not run.	72	+	when the sandbox started. `resolve_ms` includes `archive_miss_ms`, and both are
			73	+	part of `run_ms`.
73			74
74		\| Fields \| Meaning \|	75		\| Fields \| Meaning \|
75		\| --- \| --- \|	76		\| --- \| --- \|

M host_cmd.go +17 -20

 	default:
 		return nil, nil, fmt.Errorf("%s: %q is not 0 or 1", envCache, os.Getenv(envCache))
 	}
-	max, err := cachesvc.ParseMax(os.Getenv(cachesvc.EnvMax))
+	// The cache API cannot be turned off by its size: 0 is refused.
+	max, err := cachesvc.ParseSize(os.Getenv(cachesvc.EnvMax), cachesvc.DefaultMax)
+	if err == nil && max == 0 {
+		err = errors.New("0 is not a size cap; " + envCache + "=0 turns the cache off")
+	}
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, fmt.Errorf("%s: %w", cachesvc.EnvMax, err)
 	}
 	archives, err := setupArchives(ctx, scfg.StateDir)
 	if err != nil {
 // service, so it is only used when that is on.
 func setupArchives(ctx context.Context, stateDir string) (*actionscache.Cache, error) {
 	log := zerowrap.FromCtx(ctx)
-	max, err := actionscache.ParseMax(os.Getenv(actionscache.EnvMax))
+	max, err := cachesvc.ParseSize(os.Getenv(actionscache.EnvMax), actionscache.DefaultMax)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("%s: %w", actionscache.EnvMax, err)
 	}
 	if max == 0 {
 		log.Info().Msg("the action archive cache is off (" + actionscache.EnvMax + "=0)")
 	return c, nil
 }
+// envRegistryMax is the size cap of the Docker Hub mirror (default
+// defaultRegistryMax); 0 turns it off.
 const envRegistryMax = "ERMITE_RUNNER_REGISTRY_CACHE_MAX"
+const defaultRegistryMax = 10 << 30
 // setupRegistry opens the Docker Hub mirror. The caller only calls this when
 // the cache channel is enabled; its capacity is independent of the cache API.
 func setupRegistry(ctx context.Context, stateDir string) (http.Handler, error) {
 	log := zerowrap.FromCtx(ctx)
-	value := strings.TrimSpace(os.Getenv(envRegistryMax))
+	max, err := cachesvc.ParseSize(os.Getenv(envRegistryMax), defaultRegistryMax)
-	if zeroSize(value) {
-		log.Info().Msg("registry proxy off (" + envRegistryMax + "=" + value + ")")
-		return nil, nil
-	}
-	if value == "" {
-		value = "10G"
-	}
-	max, err := cachesvc.ParseMax(value)
 	if err != nil {
 		return nil, fmt.Errorf("%s: %w", envRegistryMax, err)
 	}
+	if max == 0 {
+		log.Info().Msg("registry proxy off (" + envRegistryMax + "=0)")
+		return nil, nil
+	}
 	p, err := registryproxy.New(registryproxy.Config{Dir: filepath.Join(stateDir, "registry"), Max: max, Log: log})
 	if err != nil {
 		return nil, err
 	return p, nil
 }
-// zeroSize reports a size of zero in any unit: 0, 00, 0M, 0G, 0GiB...
-func zeroSize(v string) bool {
-	digits := strings.TrimRight(v, "kKmMgGtTiIbB")
-	return digits != "" && strings.Trim(digits, "0") == ""
-}
 // envCache=0 turns the cache service off.
 const envCache = "ERMITE_RUNNER_CACHE"
 		},
 		Features: hcfg.ServerFeaturesFunc(url),
 		Cache:    cache,
-		Registry: cache != nil && cache.RegistryEnabled(),
 		Archives: archives,
 		Results:  results,
 		Artifacts: func(tok string) ermiteconnect.JobArtifactServiceClient {

M images/ubuntu-26.04/install-toolcache.sh +17 -5

 #!/bin/sh
 # SPDX-License-Identifier: MIT
+# Installs the toolchains of a lock file (tool version arch sha256 url per
+# line) for the image architecture into the @actions/tool-cache layout.
 set -eu
+lock=${1:?usage: install-toolcache.sh LOCK_FILE}
 case "$(dpkg --print-architecture)" in
     amd64) arch=x64 ;;
     arm64) arch=arm64 ;;
 esac
 cache=${RUNNER_TOOL_CACHE:-/opt/hostedtoolcache}
 tmp=$(mktemp -d)
-trap 'rm -rf "$tmp"' EXIT HUP INT TERM
+trap 'rm -rf "$tmp"' EXIT
-while read -r tool version entry_arch sha url; do
+trap 'exit 130' INT TERM
+installed=0
+# The last line is read even without a final newline.
+while read -r tool version entry_arch sha url || [ -n "$tool" ]; do
     case "$tool" in ''|'#'*) continue ;; esac
-    [ "$entry_arch" = "$arch" ] || continue
     case "$tool" in go|node) ;; *) echo "unsupported tool: $tool" >&2; exit 1 ;; esac
+    case "$entry_arch" in x64|arm64) ;; *) echo "unsupported arch for $tool $version: $entry_arch" >&2; exit 1 ;; esac
+    [ "$entry_arch" = "$arch" ] || continue
     case "$url" in
         *.tar.gz) compression=z ;;
         *.tar.xz) compression=J ;;
         *) echo "unsupported archive: $url" >&2; exit 1 ;;
     esac
-    curl -fsSL "$url" -o "$tmp/archive"
+    curl -fsSL --retry 3 "$url" -o "$tmp/archive"
     printf '%s  %s\n' "$sha" "$tmp/archive" | sha256sum -c -
     dest="$cache/$tool/$version/$arch"
     mkdir -p "$dest"
     tar -x"$compression"f "$tmp/archive" -C "$dest" --strip-components=1 --no-same-owner
     touch "$dest.complete"
     rm "$tmp/archive"
-done < "${1:?usage: install-toolcache.sh LOCK_FILE}"
+    installed=$((installed + 1))
+done < "$lock"
+if [ "$installed" -eq 0 ]; then
+    echo "no toolchain for $arch in $lock" >&2
+    exit 1
+fi
 chown -R runner:runner "$cache"

M internal/actionscache/cache.go +5 -35

 // verify re-hashes the archive of key against its sum. Any mismatch (or a
 // missing file) drops the entry and returns ErrNotFound.
-func (c *Cache) verify(key string) (duration time.Duration, err error) {
+func (c *Cache) verify(key string) (time.Duration, error) {
 	start := time.Now()
-	defer func() { duration = time.Since(start) }()
 	sb, err := os.ReadFile(c.sumPath(key))
 	if err != nil {
 		c.drop(key)
-		return 0, ErrNotFound
+		return time.Since(start), ErrNotFound
 	}
 	want, sizeS, _ := strings.Cut(strings.TrimSpace(string(sb)), " ")
 	size, perr := strconv.ParseInt(sizeS, 10, 64)
 			_ = f.Close()
 		}
 		c.drop(key)
-		return 0, ErrNotFound
+		return time.Since(start), ErrNotFound
 	}
 	defer func() { _ = f.Close() }()
 	h := sha256.New()
 	if err != nil || n != size || hex.EncodeToString(h.Sum(nil)) != want {
 		c.log.Warn().Str("key", key).Msg("action archive failed its integrity check; dropped")
 		c.drop(key)
-		return 0, ErrNotFound
+		return time.Since(start), ErrNotFound
 	}
-	return 0, nil
+	return time.Since(start), nil
 }
 func (c *Cache) drop(key string) {
 	return c.total
 }
-// ParseMax reads ERMITE_RUNNER_ACTIONS_CACHE_MAX: empty is DefaultMax, 0 is off
-// (returned as 0), otherwise a number of bytes or a size with a K, M, G or T
-// suffix (optionally followed by iB or B).
-func ParseMax(v string) (int64, error) {
-	v = strings.TrimSpace(v)
-	if v == "" {
-		return DefaultMax, nil
-	}
-	s := strings.TrimRight(strings.ToUpper(v), "IB")
-	mult := int64(1)
-	if n := len(s); n > 0 {
-		switch s[n-1] {
-		case 'K':
-			mult, s = 1<<10, s[:n-1]
-		case 'M':
-			mult, s = 1<<20, s[:n-1]
-		case 'G':
-			mult, s = 1<<30, s[:n-1]
-		case 'T':
-			mult, s = 1<<40, s[:n-1]
-		}
-	}
-	n, err := strconv.ParseInt(s, 10, 64)
-	if err != nil || n < 0 || n > (1<<62)/mult {
-		return 0, fmt.Errorf("%s: %q is not a size (examples: 2G, 500M, 0)", EnvMax, v)
-	}
-	return n * mult, nil
-}
 // Dir is the directory of the cache (tests).
 func (c *Cache) Dir() string { return c.dir }

M internal/actionscache/cache_test.go +0 -13


@@ -383,16 +383,3 @@ func TestKeyNeverUsesARef(t *testing.T) {
383		t.Error("shape")	383	t.Error("shape")
384		}	384	}
385		}	385	}
386	-
387	-	func TestParseMax(t *testing.T) {
388	-	for in, want := range map[string]int64{"": DefaultMax, "0": 0, "500M": 500 << 20, "2G": 2 << 30, "2GiB": 2 << 30, "1024": 1024} {
389	-	if got, err := ParseMax(in); err != nil \|\| got != want {
390	-	t.Errorf("%q: %d %v", in, got, err)
391	-	}
392	-	}
393	-	for _, in := range []string{"-1", "x", "G"} {
394	-	if _, err := ParseMax(in); err == nil {
395	-	t.Errorf("%q accepted", in)
396	-	}
397	-	}
398	-	}

M internal/cachesvc/handler.go +16 -9

 func (s *Service) handler(job string) http.Handler {
 	mux := http.NewServeMux()
 	if s.registry != nil {
-		registry := s.serveRegistry(make(chan struct{}, s.registryPerChannel), registryproxy.NewQuota(s.registryDownloads))
+		registry := s.serveRegistry(job, make(chan struct{}, s.registryPerChannel), registryproxy.NewQuota(RegistryDownloadsPerChannel))
-		mux.HandleFunc("GET /v2/", registry)
+		mux.HandleFunc("GET "+RegistryPath, registry)
-		mux.HandleFunc("HEAD /v2/", registry)
+		mux.HandleFunc("HEAD "+RegistryPath, registry)
 	}
 	// The routes are those of the toolkit's v1 client (cacheHttpClient.ts):
 	//   GET   cache?keys=k1,k2&version=v  :89  lookup
 	})
 }
-// channelHandler is the handler of a sandbox's channel: the cache API and the
+// channelHandler is the handler of a sandbox's channel: the cache API, the
-// action archives, and when results is not nil everything else goes to it (the results service of the
+// action archives and the registry, and when results is not nil everything else goes to it (the results service of the
 // job: Twirp and blob paths). The results service gets the same progress
 // deadlines as the cache: its bodies and replies must keep moving.
 func (s *Service) channelHandler(job string, results http.Handler) http.Handler {
 		return cache
 	}
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if strings.HasPrefix(r.URL.Path, APIPath) || strings.HasPrefix(r.URL.Path, ArchivePath) || strings.HasPrefix(r.URL.Path, "/v2/") {
+		if strings.HasPrefix(r.URL.Path, APIPath) || strings.HasPrefix(r.URL.Path, ArchivePath) || strings.HasPrefix(r.URL.Path, RegistryPath) {
 			cache.ServeHTTP(w, r)
 			return
 		}
 // Unwrap lets http.ResponseController reach the real writer.
 func (p *progressWriter) Unwrap() http.ResponseWriter { return p.ResponseWriter }
+var _ registryproxy.KeepAliver = (*progressWriter)(nil)
 // KeepAlive moves the deadlines without writing. The registry proxy calls it
 // while the client waits for a shared upstream download that is progressing,
 // before the first byte of the reply. The read deadline moves too: once it
 // slots caps the requests of this channel in progress at once; one over the
 // cap gets 503 at once, which dockerd retries or answers by pulling from
 // Docker Hub, so the channel's connections stay free for the cache. quota is
-// the channel's share of the proxy's cold downloads.
+// the channel's share of the proxy's cold downloads. job, when not empty, is
-func (s *Service) serveRegistry(slots chan struct{}, quota *registryproxy.Quota) http.HandlerFunc {
+// the channel's job, which the registry logs.
+func (s *Service) serveRegistry(job string, slots chan struct{}, quota *registryproxy.Quota) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		select {
 		case slots <- struct{}{}:
 		}
 		rc := http.NewResponseController(w)
 		defer func() { _ = rc.SetWriteDeadline(deadline(s.progress, nil)) }()
-		r = r.WithContext(registryproxy.WithQuota(r.Context(), quota))
+		ctx := registryproxy.WithQuota(r.Context(), quota)
+		if job != "" {
+			ctx = registryproxy.WithJob(ctx, job)
+		}
+		r = r.WithContext(ctx)
 		s.registry.ServeHTTP(&progressWriter{ResponseWriter: w, rc: rc, idle: s.progress}, r)
 	}
 }

A internal/cachesvc/registry_deadline_test.go +127 -0

@@ -0,0 +1,127 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package cachesvc
	4	+
	5	+	import (
	6	+	"bufio"
	7	+	"fmt"
	8	+	"io"
	9	+	"net"
	10	+	"net/http"
	11	+	"strings"
	12	+	"testing"
	13	+	"time"
	14	+
	15	+	"bnema.dev/ermite-runner/internal/registryproxy"
	16	+	)
	17	+
	18	+	const registryProgress = 300 * time.Millisecond
	19	+
	20	+	// registryRig serves a fake registry behind the service's real HTTP server, so
	21	+	// that the connection deadlines apply.
	22	+	func registryRig(t testing.T, h http.HandlerFunc) rig {
	23	+	t.Helper()
	24	+	return newRigWith(t, Config{Max: 1 << 20, MinCharge: 1, ProgressTimeout: registryProgress, Registry: h})
	25	+	}
	26	+
	27	+	// A registry download that keeps moving may last longer than the progress
	28	+	// period set when the request started.
	29	+	func TestRegistryProgressingTransferSucceeds(t *testing.T) {
	30	+	const chunks = 14
	31	+	r := registryRig(t, func(w http.ResponseWriter, _ *http.Request) {
	32	+	w.Header().Set("Content-Length", fmt.Sprint(chunks*1024))
	33	+	for range chunks {
	34	+	if _, err := w.Write([]byte(strings.Repeat("x", 1024))); err != nil {
	35	+	return
	36	+	}
	37	+	_ = http.NewResponseController(w).Flush()
	38	+	time.Sleep(registryProgress / 6)
	39	+	}
	40	+	})
	41	+	start := time.Now()
	42	+	resp, err := http.Get(r.srv.URL + RegistryPath + "library/alpine/blobs/x")
	43	+	if err != nil {
	44	+	t.Fatal(err)
	45	+	}
	46	+	defer func() { _ = resp.Body.Close() }()
	47	+	b, err := io.ReadAll(resp.Body)
	48	+	if err != nil \|\| len(b) != chunks*1024 {
	49	+	t.Fatalf("read %d bytes: %v", len(b), err)
	50	+	}
	51	+	if d := time.Since(start); d < 2*registryProgress {
	52	+	t.Fatalf("transfer took %v, not longer than the progress period", d)
	53	+	}
	54	+	}
	55	+
	56	+	// The proxy keeps a client alive while it waits for a shared upstream
	57	+	// download, before the first byte; the read deadline moves too, so the request
	58	+	// context stays alive.
	59	+	func TestRegistryKeepAliveWhileWaiting(t *testing.T) {
	60	+	r := registryRig(t, func(w http.ResponseWriter, req *http.Request) {
	61	+	k, ok := w.(registryproxy.KeepAliver)
	62	+	if !ok {
	63	+	t.Error("the registry writer is no KeepAliver")
	64	+	return
	65	+	}
	66	+	for range 12 { // two progress periods
	67	+	time.Sleep(registryProgress / 6)
	68	+	k.KeepAlive()
	69	+	if req.Context().Err() != nil {
	70	+	t.Error("request context ended while kept alive")
	71	+	return
	72	+	}
	73	+	}
	74	+	_, _ = io.WriteString(w, "verified")
	75	+	})
	76	+	resp, err := http.Get(r.srv.URL + RegistryPath + "library/alpine/blobs/x")
	77	+	if err != nil {
	78	+	t.Fatal(err)
	79	+	}
	80	+	defer func() { _ = resp.Body.Close() }()
	81	+	if b, err := io.ReadAll(resp.Body); err != nil \|\| string(b) != "verified" {
	82	+	t.Fatalf("%q %v", b, err)
	83	+	}
	84	+	}
	85	+
	86	+	// A client that stops reading is cut after one progress period.
	87	+	func TestRegistryStalledReaderIsCut(t *testing.T) {
	88	+	result := make(chan error, 1)
	89	+	r := registryRig(t, func(w http.ResponseWriter, _ *http.Request) {
	90	+	chunk := []byte(strings.Repeat("x", 64<<10))
	91	+	deadline := time.Now().Add(5 * time.Second)
	92	+	for time.Now().Before(deadline) {
	93	+	if _, err := w.Write(chunk); err != nil {
	94	+	result <- err
	95	+	return
	96	+	}
	97	+	}
	98	+	result <- nil
	99	+	})
	100	+	conn, err := net.Dial("tcp", strings.TrimPrefix(r.srv.URL, "http://"))
	101	+	if err != nil {
	102	+	t.Fatal(err)
	103	+	}
	104	+	defer func() { _ = conn.Close() }()
	105	+	if tc, ok := conn.(*net.TCPConn); ok {
	106	+	_ = tc.SetReadBuffer(4096)
	107	+	}
	108	+	if _, err := fmt.Fprint(conn, "GET "+RegistryPath+"library/alpine/blobs/x HTTP/1.1\r\nHost: x\r\n\r\n"); err != nil {
	109	+	t.Fatal(err)
	110	+	}
	111	+	resp, err := http.ReadResponse(bufio.NewReader(conn), nil)
	112	+	if err != nil {
	113	+	t.Fatal(err)
	114	+	}
	115	+	if resp.StatusCode != http.StatusOK {
	116	+	t.Fatal(resp.StatusCode)
	117	+	}
	118	+	// Read nothing more.
	119	+	select {
	120	+	case err := <-result:
	121	+	if err == nil {
	122	+	t.Fatal("a stalled reader was not cut")
	123	+	}
	124	+	case <-time.After(10 * time.Second):
	125	+	t.Fatal("handler still blocked")
	126	+	}
	127	+	}

A internal/cachesvc/registry_limits_test.go +195 -0

@@ -0,0 +1,195 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package cachesvc
	4	+
	5	+	import (
	6	+	"bytes"
	7	+	"context"
	8	+	"crypto/sha256"
	9	+	"fmt"
	10	+	"io"
	11	+	"net/http"
	12	+	"net/http/httptest"
	13	+	"strings"
	14	+	"sync"
	15	+	"sync/atomic"
	16	+	"testing"
	17	+	"time"
	18	+
	19	+	"github.com/bnema/zerowrap"
	20	+
	21	+	"bnema.dev/ermite-runner/internal/registryproxy"
	22	+	)
	23	+
	24	+	// The registry requests of one channel in progress at once are capped: one
	25	+	// over the cap gets 503 at once, the cache API of that channel still answers,
	26	+	// and another channel has its own slots.
	27	+	func TestRegistryPerChannelCap(t *testing.T) {
	28	+	const limit = 2
	29	+	var inside atomic.Int32
	30	+	release := make(chan struct{})
	31	+	svc, err := New(Config{Dir: t.TempDir(), RegistryPerChannel: limit, Registry: http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
	32	+	inside.Add(1)
	33	+	<-release
	34	+	_, _ = io.WriteString(w, "ok")
	35	+	})})
	36	+	if err != nil {
	37	+	t.Fatal(err)
	38	+	}
	39	+	t.Cleanup(func() { _ = svc.CloseAll() })
	40	+	var once sync.Once
	41	+	unblock := func() { once.Do(func() { close(release) }) }
	42	+	t.Cleanup(unblock)
	43	+	a, b := channel(t, svc, "sb-a", "job-a"), channel(t, svc, "sb-b", "job-b")
	44	+	get := func(c *http.Client) int {
	45	+	resp, err := c.Get("http://cache" + RegistryPath + "library/alpine/blobs/x")
	46	+	if err != nil {
	47	+	t.Error(err)
	48	+	return 0
	49	+	}
	50	+	_, _ = io.Copy(io.Discard, resp.Body)
	51	+	_ = resp.Body.Close()
	52	+	return resp.StatusCode
	53	+	}
	54	+	var wg sync.WaitGroup
	55	+	codes := make(chan int, limit)
	56	+	for range limit {
	57	+	wg.Go(func() { codes <- get(a) })
	58	+	}
	59	+	waitUntil(t, func() bool { return inside.Load() == limit })
	60	+	if code := get(a); code != http.StatusServiceUnavailable {
	61	+	t.Fatalf("over the cap: %d", code)
	62	+	}
	63	+	if code := lookupStatus(t, a, ""); code != http.StatusForbidden {
	64	+	t.Fatalf("cache API behind the registry cap: %d", code)
	65	+	}
	66	+	other := make(chan int, 1)
	67	+	go func() { other <- get(b) }()
	68	+	waitUntil(t, func() bool { return inside.Load() == limit+1 })
	69	+	unblock()
	70	+	wg.Wait()
	71	+	close(codes)
	72	+	for code := range codes {
	73	+	if code != http.StatusOK {
	74	+	t.Fatalf("within the cap: %d", code)
	75	+	}
	76	+	}
	77	+	if code := <-other; code != http.StatusOK {
	78	+	t.Fatalf("other channel: %d", code)
	79	+	}
	80	+	if code := get(a); code != http.StatusOK {
	81	+	t.Fatalf("slots not released: %d", code)
	82	+	}
	83	+	}
	84	+
	85	+	// stallingHub fakes Docker Hub for a real registryproxy: a manifest, and blobs
	86	+	// whose download stalls until the test ends. It counts the blob downloads
	87	+	// started.
	88	+	func stallingHub(t testing.T) (httptest.Server, *atomic.Int32, string) {
	89	+	t.Helper()
	90	+	const manifest = `{"schemaVersion":2}`
	91	+	digest := fmt.Sprintf("sha256:%x", sha256.Sum256([]byte(manifest)))
	92	+	var started atomic.Int32
	93	+	done := make(chan struct{})
	94	+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	95	+	switch {
	96	+	case r.URL.Path == "/token":
	97	+	_, _ = io.WriteString(w, `{"token":"public"}`)
	98	+	case strings.Contains(r.URL.Path, "/manifests/"):
	99	+	w.Header().Set("Content-Type", "application/vnd.oci.image.index.v1+json")
	100	+	w.Header().Set("Docker-Content-Digest", digest)
	101	+	_, _ = io.WriteString(w, manifest)
	102	+	default:
	103	+	started.Add(1)
	104	+	w.Header().Set("Content-Length", "1000000")
	105	+	_, _ = io.WriteString(w, "partial")
	106	+	w.(http.Flusher).Flush()
	107	+	select {
	108	+	case <-r.Context().Done():
	109	+	case <-done:
	110	+	}
	111	+	}
	112	+	}))
	113	+	t.Cleanup(srv.Close)
	114	+	t.Cleanup(func() { close(done) }) // before srv.Close, which waits for the handlers
	115	+	return srv, &started, digest
	116	+	}
	117	+
	118	+	// Each channel has its own quota of cold downloads: a channel that has
	119	+	// started RegistryDownloadsPerChannel of them waits for its own to end, while
	120	+	// another channel starts its download at once.
	121	+	func TestRegistryQuotaPerChannel(t *testing.T) {
	122	+	hub, started, _ := stallingHub(t)
	123	+	proxy, err := registryproxy.New(registryproxy.Config{Dir: t.TempDir(), Max: 1 << 30, Upstream: hub.URL, AuthURL: hub.URL + "/token",
	124	+	SlotWait: time.Hour, AbandonGrace: 10 * time.Millisecond})
	125	+	if err != nil {
	126	+	t.Fatal(err)
	127	+	}
	128	+	svc, err := New(Config{Dir: t.TempDir(), Registry: proxy})
	129	+	if err != nil {
	130	+	t.Fatal(err)
	131	+	}
	132	+	t.Cleanup(func() { _ = svc.CloseAll() })
	133	+	ctx, cancel := context.WithCancel(context.Background())
	134	+	defer cancel()
	135	+	var wg sync.WaitGroup
	136	+	defer wg.Wait()
	137	+	defer cancel()
	138	+	pull := func(c *http.Client, i int) {
	139	+	req, _ := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("http://cache%slibrary/alpine/blobs/sha256:%x", RegistryPath, sha256.Sum256(fmt.Appendf(nil, "blob-%d", i))), nil)
	140	+	wg.Go(func() {
	141	+	if resp, err := c.Do(req); err == nil {
	142	+	_ = resp.Body.Close()
	143	+	}
	144	+	})
	145	+	}
	146	+	a, b := channel(t, svc, "sb-a", "job-a"), channel(t, svc, "sb-b", "job-b")
	147	+	for i := range RegistryDownloadsPerChannel + 1 {
	148	+	pull(a, i)
	149	+	}
	150	+	waitUntil(t, func() bool { return started.Load() == RegistryDownloadsPerChannel })
	151	+	time.Sleep(50 * time.Millisecond) // the one over A's quota would have started by now
	152	+	if n := started.Load(); n != RegistryDownloadsPerChannel {
	153	+	t.Fatalf("channel A started %d downloads, its quota is %d", n, RegistryDownloadsPerChannel)
	154	+	}
	155	+	pull(b, 100)
	156	+	waitUntil(t, func() bool { return started.Load() == RegistryDownloadsPerChannel+1 })
	157	+	}
	158	+
	159	+	// The registry lines of a channel carry its job.
	160	+	func TestRegistryLogsJob(t *testing.T) {
	161	+	hub, _, digest := stallingHub(t)
	162	+	var logs bytes.Buffer
	163	+	var mu sync.Mutex
	164	+	proxy, err := registryproxy.New(registryproxy.Config{Dir: t.TempDir(), Max: 1 << 20, Upstream: hub.URL, AuthURL: hub.URL + "/token",
	165	+	Log: zerowrap.New(zerowrap.Config{Level: "info", Format: "json", Output: writerFunc(func(p []byte) (int, error) {
	166	+	mu.Lock()
	167	+	defer mu.Unlock()
	168	+	return logs.Write(p)
	169	+	})})})
	170	+	if err != nil {
	171	+	t.Fatal(err)
	172	+	}
	173	+	svc, err := New(Config{Dir: t.TempDir(), Registry: proxy})
	174	+	if err != nil {
	175	+	t.Fatal(err)
	176	+	}
	177	+	t.Cleanup(func() { _ = svc.CloseAll() })
	178	+	resp, err := channel(t, svc, "sb-a", "job-a").Get("http://cache" + RegistryPath + "library/alpine/manifests/" + digest)
	179	+	if err != nil {
	180	+	t.Fatal(err)
	181	+	}
	182	+	_ = resp.Body.Close()
	183	+	if resp.StatusCode != http.StatusOK {
	184	+	t.Fatal(resp.StatusCode)
	185	+	}
	186	+	mu.Lock()
	187	+	defer mu.Unlock()
	188	+	if !strings.Contains(logs.String(), `"job_id":"job-a"`) \|\| !strings.Contains(logs.String(), `"message":"registry manifest"`) {
	189	+	t.Fatalf("logs: %s", logs.String())
	190	+	}
	191	+	}
	192	+
	193	+	type writerFunc func(p []byte) (int, error)
	194	+
	195	+	func (f writerFunc) Write(p []byte) (int, error) { return f(p) }

D internal/cachesvc/registry_progress_test.go +0 -235

@@ -1,235 +0,0 @@
1	-	// SPDX-License-Identifier: MIT
2	-	package cachesvc
3	-
4	-	import (
5	-	"bufio"
6	-	"fmt"
7	-	"io"
8	-	"net"
9	-	"net/http"
10	-	"strings"
11	-	"sync"
12	-	"sync/atomic"
13	-	"testing"
14	-	"time"
15	-
16	-	"bnema.dev/ermite-runner/internal/registryproxy"
17	-	)
18	-
19	-	const registryProgress = 150 * time.Millisecond
20	-
21	-	// registryRig serves a fake registry behind the service's real HTTP server, so
22	-	// that the connection deadlines apply.
23	-	func registryRig(t testing.T, h http.HandlerFunc) rig {
24	-	t.Helper()
25	-	return newRigWith(t, Config{Max: 1 << 20, MinCharge: 1, ProgressTimeout: registryProgress, Registry: h})
26	-	}
27	-
28	-	// A registry download that keeps moving may last longer than the
29	-	// progress period set when the request started.
30	-	func TestRegistryProgressingTransferSucceeds(t *testing.T) {
31	-	const chunks = 8
32	-	r := registryRig(t, func(w http.ResponseWriter, _ *http.Request) {
33	-	w.Header().Set("Content-Length", fmt.Sprint(chunks*1024))
34	-	for range chunks {
35	-	if _, err := w.Write([]byte(strings.Repeat("x", 1024))); err != nil {
36	-	return
37	-	}
38	-	_ = http.NewResponseController(w).Flush()
39	-	time.Sleep(registryProgress / 3)
40	-	}
41	-	})
42	-	start := time.Now()
43	-	resp, err := http.Get(r.srv.URL + "/v2/library/alpine/blobs/x")
44	-	if err != nil {
45	-	t.Fatal(err)
46	-	}
47	-	defer func() { _ = resp.Body.Close() }()
48	-	b, err := io.ReadAll(resp.Body)
49	-	if err != nil \|\| len(b) != chunks*1024 {
50	-	t.Fatalf("read %d bytes: %v", len(b), err)
51	-	}
52	-	if d := time.Since(start); d < 2*registryProgress {
53	-	t.Fatalf("transfer took %v, not longer than the progress period", d)
54	-	}
55	-	}
56	-
57	-	// The proxy keeps a client alive while it waits for a shared upstream
58	-	// download, before the first byte; the read deadline moves too, so the request
59	-	// context stays alive.
60	-	func TestRegistryKeepAliveWhileWaiting(t *testing.T) {
61	-	r := registryRig(t, func(w http.ResponseWriter, req *http.Request) {
62	-	k, ok := w.(interface{ KeepAlive() })
63	-	if !ok {
64	-	t.Error("no KeepAlive on the registry writer")
65	-	return
66	-	}
67	-	for range 6 {
68	-	time.Sleep(registryProgress / 3)
69	-	k.KeepAlive()
70	-	if req.Context().Err() != nil {
71	-	t.Error("request context ended while kept alive")
72	-	return
73	-	}
74	-	}
75	-	_, _ = io.WriteString(w, "verified")
76	-	})
77	-	resp, err := http.Get(r.srv.URL + "/v2/library/alpine/blobs/x")
78	-	if err != nil {
79	-	t.Fatal(err)
80	-	}
81	-	defer func() { _ = resp.Body.Close() }()
82	-	if b, err := io.ReadAll(resp.Body); err != nil \|\| string(b) != "verified" {
83	-	t.Fatalf("%q %v", b, err)
84	-	}
85	-	}
86	-
87	-	// The registry requests of one channel in progress at once are capped: one
88	-	// over the cap gets 503 at once, the cache API of that channel still answers,
89	-	// and another channel has its own slots.
90	-	func TestRegistryPerChannelCap(t *testing.T) {
91	-	const limit = 2
92	-	var inside atomic.Int32
93	-	release := make(chan struct{})
94	-	svc, err := New(Config{Dir: t.TempDir(), RegistryPerChannel: limit, Registry: http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
95	-	inside.Add(1)
96	-	<-release
97	-	_, _ = io.WriteString(w, "ok")
98	-	})})
99	-	if err != nil {
100	-	t.Fatal(err)
101	-	}
102	-	t.Cleanup(func() { _ = svc.CloseAll() })
103	-	var once sync.Once
104	-	unblock := func() { once.Do(func() { close(release) }) }
105	-	t.Cleanup(unblock)
106	-	a, b := channel(t, svc, "sb-a", "job-a"), channel(t, svc, "sb-b", "job-b")
107	-	get := func(c *http.Client) int {
108	-	resp, err := c.Get("http://cache/v2/library/alpine/blobs/x")
109	-	if err != nil {
110	-	t.Error(err)
111	-	return 0
112	-	}
113	-	_, _ = io.Copy(io.Discard, resp.Body)
114	-	_ = resp.Body.Close()
115	-	return resp.StatusCode
116	-	}
117	-	var wg sync.WaitGroup
118	-	codes := make(chan int, limit)
119	-	for range limit {
120	-	wg.Go(func() { codes <- get(a) })
121	-	}
122	-	waitUntil(t, func() bool { return inside.Load() == limit })
123	-	if code := get(a); code != http.StatusServiceUnavailable {
124	-	t.Fatalf("over the cap: %d", code)
125	-	}
126	-	if code := lookupStatus(t, a, ""); code != http.StatusForbidden {
127	-	t.Fatalf("cache API behind the registry cap: %d", code)
128	-	}
129	-	other := make(chan int, 1)
130	-	go func() { other <- get(b) }()
131	-	waitUntil(t, func() bool { return inside.Load() == limit+1 })
132	-	unblock()
133	-	wg.Wait()
134	-	close(codes)
135	-	for code := range codes {
136	-	if code != http.StatusOK {
137	-	t.Fatalf("within the cap: %d", code)
138	-	}
139	-	}
140	-	if code := <-other; code != http.StatusOK {
141	-	t.Fatalf("other channel: %d", code)
142	-	}
143	-	if code := get(a); code != http.StatusOK {
144	-	t.Fatalf("slots not released: %d", code)
145	-	}
146	-	}
147	-
148	-	// Each channel hands the registry its own download quota, the same one for
149	-	// all of its requests.
150	-	func TestRegistryQuotaPerChannel(t *testing.T) {
151	-	var mu sync.Mutex
152	-	seen := map[string]map[*registryproxy.Quota]bool{}
153	-	svc, err := New(Config{Dir: t.TempDir(), Registry: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
154	-	q := registryproxy.QuotaOf(r.Context())
155	-	if q == nil {
156	-	t.Error("no quota on the registry request")
157	-	}
158	-	mu.Lock()
159	-	ch := r.Header.Get("X-Channel")
160	-	if seen[ch] == nil {
161	-	seen[ch] = map[*registryproxy.Quota]bool{}
162	-	}
163	-	seen[ch][q] = true
164	-	mu.Unlock()
165	-	})})
166	-	if err != nil {
167	-	t.Fatal(err)
168	-	}
169	-	t.Cleanup(func() { _ = svc.CloseAll() })
170	-	for _, ch := range []string{"a", "b"} {
171	-	c := channel(t, svc, "sb-"+ch, "job-"+ch)
172	-	for range 3 {
173	-	req, _ := http.NewRequest(http.MethodGet, "http://cache/v2/library/alpine/blobs/x", nil)
174	-	req.Header.Set("X-Channel", ch)
175	-	resp, err := c.Do(req)
176	-	if err != nil {
177	-	t.Fatal(err)
178	-	}
179	-	_ = resp.Body.Close()
180	-	}
181	-	}
182	-	mu.Lock()
183	-	defer mu.Unlock()
184	-	if len(seen["a"]) != 1 \|\| len(seen["b"]) != 1 {
185	-	t.Fatalf("quotas per channel: a=%d b=%d", len(seen["a"]), len(seen["b"]))
186	-	}
187	-	for qa := range seen["a"] {
188	-	if seen["b"][qa] {
189	-	t.Fatal("two channels share a quota")
190	-	}
191	-	}
192	-	}
193	-
194	-	// A client that stops reading is cut after one progress period.
195	-	func TestRegistryStalledReaderIsCut(t *testing.T) {
196	-	result := make(chan error, 1)
197	-	r := registryRig(t, func(w http.ResponseWriter, _ *http.Request) {
198	-	chunk := []byte(strings.Repeat("x", 64<<10))
199	-	deadline := time.Now().Add(5 * time.Second)
200	-	for time.Now().Before(deadline) {
201	-	if _, err := w.Write(chunk); err != nil {
202	-	result <- err
203	-	return
204	-	}
205	-	}
206	-	result <- nil
207	-	})
208	-	conn, err := net.Dial("tcp", strings.TrimPrefix(r.srv.URL, "http://"))
209	-	if err != nil {
210	-	t.Fatal(err)
211	-	}
212	-	defer func() { _ = conn.Close() }()
213	-	if tc, ok := conn.(*net.TCPConn); ok {
214	-	_ = tc.SetReadBuffer(4096)
215	-	}
216	-	if _, err := fmt.Fprint(conn, "GET /v2/library/alpine/blobs/x HTTP/1.1\r\nHost: x\r\n\r\n"); err != nil {
217	-	t.Fatal(err)
218	-	}
219	-	resp, err := http.ReadResponse(bufio.NewReader(conn), nil)
220	-	if err != nil {
221	-	t.Fatal(err)
222	-	}
223	-	if resp.StatusCode != http.StatusOK {
224	-	t.Fatal(resp.StatusCode)
225	-	}
226	-	// Read nothing more.
227	-	select {
228	-	case err := <-result:
229	-	if err == nil {
230	-	t.Fatal("a stalled reader was not cut")
231	-	}
232	-	case <-time.After(10 * time.Second):
233	-	t.Fatal("handler still blocked")
234	-	}
235	-	}

M internal/cachesvc/registry_test.go +4 -1

 // SPDX-License-Identifier: MIT
 package cachesvc
 import (
 	"testing"
 )
+// The registry is served under RegistryPath on the cache handler and the
+// channel, never by the results service; without one, those paths are 404.
 func TestRegistryRoute(t *testing.T) {
 	for _, on := range []bool{false, true} {
 		cfg := Config{}
 		})
 		for _, h := range []http.Handler{r.svc.Handler(), r.svc.channelHandler("job", results)} {
 			w := httptest.NewRecorder()
-			h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/v2/", nil))
+			h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, RegistryPath, nil))
 			if on {
 				if w.Code != http.StatusOK || w.Body.String() != "registry" {
 					t.Fatalf("%d %s", w.Code, w.Body)

M internal/cachesvc/service.go +20 -21

 	// progress at once, so that one sandbox's pulls leave room for its cache and
 	// artifact traffic. dockerd pulls 3 layers at once by default.
 	DefaultRegistryPerChannel = 8
-	// DefaultRegistryDownloadsPerChannel bounds the cold Docker Hub downloads
+	// RegistryDownloadsPerChannel bounds the cold Docker Hub downloads one
-	// one channel may have started at once (dockerd's default
+	// channel may have started at once (dockerd's default
 	// max-concurrent-downloads), so that one sandbox cannot take every slot of
 	// the registry proxy.
-	DefaultRegistryDownloadsPerChannel = 3
+	RegistryDownloadsPerChannel = 3
+	// RegistryPath is the prefix of the Docker registry API on the channel.
+	RegistryPath = "/v2/"
 )
 // Config configures a Service.
 	ProgressTimeout              time.Duration
 	ConnsPerListener, Conns      int
 	RegistryPerChannel           int
-	RegistryDownloadsPerChannel  int
 	MinCharge                    int64
 	MaxPending, MaxPendingPerJob int
 	// hookChunk, when set, is called by an upload once its range is admitted and
 	registry http.Handler
 	// registryPerChannel caps the registry requests in progress per handler.
 	registryPerChannel int
-	// registryDownloads is the cold download quota of each channel.
+	hookChunk          func(id int64)
-	registryDownloads int
+	progress           time.Duration
-	hookChunk         func(id int64)
+	perListener        int
-	progress          time.Duration
+	connsGlobal        chan struct{}
-	perListener       int
-	connsGlobal       chan struct{}
 }
 type listener struct {
 	if cfg.RegistryPerChannel <= 0 {
 		cfg.RegistryPerChannel = DefaultRegistryPerChannel
 	}
-	if cfg.RegistryDownloadsPerChannel <= 0 {
-		cfg.RegistryDownloadsPerChannel = DefaultRegistryDownloadsPerChannel
-	}
 	s := &Service{store: st, sign: sg, now: cfg.Now, log: cfg.Log, channels: map[string]*listener{}, stop: make(chan struct{}),
-		archives: cfg.Archives, registry: cfg.Registry, hookChunk: cfg.hookChunk, progress: cfg.ProgressTimeout, perListener: cfg.ConnsPerListener, connsGlobal: make(chan struct{}, cfg.Conns), registryPerChannel: cfg.RegistryPerChannel, registryDownloads: cfg.RegistryDownloadsPerChannel}
+		archives: cfg.Archives, registry: cfg.Registry, hookChunk: cfg.hookChunk, progress: cfg.ProgressTimeout, perListener: cfg.ConnsPerListener, connsGlobal: make(chan struct{}, cfg.Conns), registryPerChannel: cfg.RegistryPerChannel}
 	s.wg.Add(1)
 	go s.housekeeping()
 	return s, nil
 	return s.store.flush()
 }
-// ParseMax reads ERMITE_RUNNER_CACHE_MAX: a number of bytes or a size with a K,
+// ParseSize reads a size: a number of bytes or a number with a K, M, G or T
-// M, G or T suffix (optionally followed by iB or B). Empty means DefaultMax.
+// suffix (powers of 1024), optionally followed by iB or B, such as 10G, 500M
-func ParseMax(v string) (int64, error) {
+// or 2GiB. Surrounding spaces are ignored; empty means def. Zero in any unit
+// (0, 0G, 0GiB) is 0. The error names no variable: the caller prefixes it.
+func ParseSize(v string, def int64) (int64, error) {
+	v = strings.TrimSpace(v)
 	if v == "" {
-		return DefaultMax, nil
+		return def, nil
 	}
 	s := []byte(v)
-	for len(s) > 0 && (s[len(s)-1] == 'B' || s[len(s)-1] == 'b' || s[len(s)-1] == 'i') {
+	for len(s) > 0 && strings.IndexByte("bBiI", s[len(s)-1]) >= 0 {
 		s = s[:len(s)-1]
 	}
 	mult := int64(1)
 		}
 	}
 	n, err := strconv.ParseInt(string(s), 10, 64)
-	if err != nil || n <= 0 || n > (1<<62)/mult {
+	if err != nil || n < 0 || n > (1<<62)/mult {
-		return 0, fmt.Errorf("%s: %q is not a size (examples: 10G, 500M)", EnvMax, v)
+		return 0, fmt.Errorf("%q is not a size (examples: 10G, 500M)", v)
 	}
 	return n * mult, nil
 }

M internal/cachesvc/service_test.go +10 -5

 	}
 }
-func TestParseMax(t *testing.T) {
+func TestParseSize(t *testing.T) {
-	for in, want := range map[string]int64{"": DefaultMax, "10G": 10 << 30, "500m": 500 << 20, "1GiB": 1 << 30, "2048": 2048, "3TB": 3 << 40} {
+	const def = 7
-		if got, err := ParseMax(in); err != nil || got != want {
+	for in, want := range map[string]int64{"": def, "  ": def, "10G": 10 << 30, "500m": 500 << 20, "1GiB": 1 << 30, "2048": 2048, "3TB": 3 << 40, " 2G ": 2 << 30,
+		"0": 0, "00": 0, "0G": 0, "0GiB": 0, "2GIB": 2 << 30} {
+		if got, err := ParseSize(in, def); err != nil || got != want {
 			t.Errorf("%q: %d %v", in, got, err)
 		}
 	}
-	for _, in := range []string{"0", "-1", "G", "ten", "1.5G", "99999999999T"} {
+	for _, in := range []string{"-1", "G", "ten", "1.5G", "99999999999T"} {
-		if _, err := ParseMax(in); err == nil {
+		_, err := ParseSize(in, def)
+		if err == nil {
 			t.Errorf("%q accepted", in)
+		} else if strings.Contains(err.Error(), "ERMITE_") {
+			t.Errorf("%q: the error names a variable: %v", in, err)
 		}
 	}
 }

M internal/e2e/registry_test.go +4 -2

 // SPDX-License-Identifier: MIT
 package e2e
 import (
 	"testing"
 	"time"
+	"github.com/bnema/zerowrap"
 	"bnema.dev/ermite-runner/internal/cachesvc"
 	"bnema.dev/ermite-runner/internal/execstream/execpb"
 	"bnema.dev/ermite-runner/internal/registryproxy"
-	"github.com/bnema/zerowrap"
 )
 type registryLogs struct {
     steps:
       - run: echo alpine-job
 `)
+		// A registry-only job: the host sends the port and no token.
 		spec.CachePort = cachesvc.DefaultPort
-		spec.CacheToken = svc.Token("e2e", cachesvc.Scope{}, time.Hour)
 		s := runJob(t, d, spec, nil)
 		if s.Result == nil || s.Result.Conclusion != execpb.StepConclusion_STEP_CONCLUSION_SUCCESS {
 			t.Fatalf("pull job %d: %v\n%s", i, s.Result, strings.Join(s.Logs, "\n"))

M internal/e2e/timing_test.go +3 -1

 	"testing"
 	"time"
+	"github.com/bnema/zerowrap"
 	ermite "bnema.dev/ermite-api/gen/ermite"
 	"bnema.dev/ermite-api/gen/ermite/ermiteconnect"
 	"bnema.dev/ermite-runner/internal/host"
 	"bnema.dev/ermite-runner/internal/host/hosttest"
-	"github.com/bnema/zerowrap"
 )
 type timingWriter struct {

M internal/guest/cacheurl_test.go +3 -3


@@ -121,11 +121,11 @@ func TestStartCacheRelay(t *testing.T) {
121		}	121		}
122		}	122		}
123			123
124	-	// No cache service, no relay and no variables; a port that is taken leaves the	124	+	// No port, no relay and no variables; a port that is taken leaves the job
125	-	// job without a cache instead of failing it.	125	+	// without a cache instead of failing it.
126		func TestStartCacheRelayWithoutServiceOrPort(t *testing.T) {	126		func TestStartCacheRelayWithoutServiceOrPort(t *testing.T) {
127		target := Target{Network: "unix", Addr: filepath.Join(t.TempDir(), "exec.sock")}	127		target := Target{Network: "unix", Addr: filepath.Join(t.TempDir(), "exec.sock")}
128	-	for _, spec := range []*execpb.JobSpec{{}, {CachePort: 9089}, {CacheToken: "tok"}} {	128	+	for _, spec := range []*execpb.JobSpec{{}, {CacheToken: "tok"}, {CacheToken: "tok", CacheMode: "write"}} {
129		if got := startCacheRelay(context.Background(), target, spec, noBridge); got != (cacheRelay{}) {	129		if got := startCacheRelay(context.Background(), target, spec, noBridge); got != (cacheRelay{}) {
130		t.Errorf("%+v: relay %+v", spec, got)	130		t.Errorf("%+v: relay %+v", spec, got)
131		}	131		}

M internal/guest/dockerd.go +3 -1

 // SPDX-License-Identifier: MIT
 package guest
 import (
-	"bnema.dev/ermite-runner/internal/sandbox"
 	"fmt"
+	"bnema.dev/ermite-runner/internal/sandbox"
 )
 func defaultDockerdArgs() []string {

M internal/guest/init.go +12 -12

 // replace the fields.
 type InitConfig struct {
 	Version string
-	bootMs  map[string]int64
 	// Cmdline is the kernel command line; empty reads /proc/cmdline.
 	Cmdline string
 	// Getpid reports the process id; Init only acts as a VM's PID 1 when it is 1.
 // powering the VM off, or exiting with the executor's code in a container.
 func Init(ctx context.Context, cfg InitConfig) int {
 	entry := time.Now()
-	cfg.bootMs = make(map[string]int64)
+	// bootMs is the boot phase timing handed to the executor (ERMITE_BOOT_MS),
+	// which reports it in its Hello.
+	bootMs := make(map[string]int64)
 	log := zerowrap.FromCtx(ctx)
 	// The kernel starts a VM's init with umask 0, which dockerd, the executor
 	// and every step would inherit: files and t.TempDir() directories would be
 		if b, err := os.ReadFile("/proc/uptime"); err == nil {
 			if fields := strings.Fields(string(b)); len(fields) > 0 {
 				if seconds, err := strconv.ParseFloat(fields[0], 64); err == nil && seconds >= 0 {
-					cfg.bootMs["kernel"] = max(0, int64(seconds*1000)-time.Since(entry).Milliseconds())
+					bootMs["kernel"] = max(0, int64(seconds*1000)-time.Since(entry).Milliseconds())
 				}
 			}
 		}
 			powerOff()
 			return 1
 		}
-		cfg.bootMs["setup"] = time.Since(start).Milliseconds()
+		bootMs["setup"] = time.Since(start).Milliseconds()
 	}
-	code := runPayload(ctx, cfg, mode, args)
+	code := runPayload(ctx, cfg, mode, args, bootMs)
 	if inVM {
 		powerOff()
 	}
 // with an empty environment.
 const defaultPath = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
-func runPayload(ctx context.Context, cfg InitConfig, mode string, args map[string]string) int {
+// runPayload starts dockerd and the executor and waits for them. It adds its
+// boot phases to bootMs, which the executor gets.
+func runPayload(ctx context.Context, cfg InitConfig, mode string, args map[string]string, bootMs map[string]int64) int {
 	log := zerowrap.FromCtx(ctx)
 	if os.Getenv("PATH") == "" {
 		_ = os.Setenv("PATH", defaultPath)
 		return 1
 	}
-	if cfg.bootMs == nil {
-		cfg.bootMs = make(map[string]int64)
-	}
 	var dockerStart time.Time
 	var dockerPid, containerdPid int
 	_, statErr := os.Stat("/usr/bin/dockerd")
 			// A containerd that never came up still cost its wait: keep it in the
 			// breakdown. A missing binary returns at once (0 ms) and is omitted.
 			if ms := time.Since(start).Milliseconds(); ok || ms > 0 {
-				cfg.bootMs["containerd"] = ms
+				bootMs["containerd"] = ms
 			}
 			if ok {
 				containerdPid = pid
 			timeout = dockerReadyTimeout
 		}
 		err := waitDockerReady(ctx, log, sock, dockerPid, timeout, sigs)
-		cfg.bootMs["dockerd"] = time.Since(dockerStart).Milliseconds()
+		bootMs["dockerd"] = time.Since(dockerStart).Milliseconds()
 		var exited *dockerdExitError
 		var abort *abortBootError
 		switch {
 		}
 	}
-	execEnv := append(os.Environ(), "ERMITE_CONNECT="+target, "ERMITE_SANDBOX="+mode, "ERMITE_VERSION="+cfg.Version, "ERMITE_BOOT_MS="+formatBootMs(cfg.bootMs))
+	execEnv := append(os.Environ(), "ERMITE_CONNECT="+target, "ERMITE_SANDBOX="+mode, "ERMITE_VERSION="+cfg.Version, "ERMITE_BOOT_MS="+formatBootMs(bootMs))
 	execPid, err := spawn(cfg.ExecArgs, execEnv)
 	if err != nil {
 		log.Error().Err(err).Msg("init: start executor")

M internal/guest/jobexec.go +5 -5


@@ -353,7 +353,7 @@ func (c cacheRelay) close() {
353		func startCacheRelay(ctx context.Context, target Target, spec *execpb.JobSpec, bridge bridgeLookup) cacheRelay {	353		func startCacheRelay(ctx context.Context, target Target, spec *execpb.JobSpec, bridge bridgeLookup) cacheRelay {
354		port := spec.GetCachePort()	354		port := spec.GetCachePort()
355		url := cacheURL(port)	355		url := cacheURL(port)
356	-	if url == "" \|\| spec.GetCacheToken() == "" {	356	+	if url == "" {
357		return cacheRelay{}	357		return cacheRelay{}
358		}	358		}
359		log := zerowrap.FromCtx(ctx)	359		log := zerowrap.FromCtx(ctx)

@@ -365,10 +365,10 @@ func startCacheRelay(ctx context.Context, target Target, spec *execpb.JobSpec, b
365		return cacheRelay{}	365		return cacheRelay{}
366		}	366		}
367		results := spec.GetResults()	367		results := spec.GetResults()
368	-	// Registry-only jobs need dockerd's loopback relay, but no action runtime	368	+	// A registry-only job (no cache scope, no results service: the host sends
369	-	// URLs or token. Leave those fields empty so neither node nor docker	369	+	// no token) needs dockerd's loopback relay, but no action runtime URLs or
370	-	// actions receive ACTIONS_* runtime variables.	370	+	// token: neither node nor docker actions get ACTIONS_* variables.
371	-	if !results && spec.GetCacheMode() == "" {	371	+	if spec.GetCacheToken() == "" \|\| (!results && spec.GetCacheMode() == "") {
372		return cacheRelay{relay: r}	372		return cacheRelay{relay: r}
373		}	373		}
374		c := cacheRelay{relay: r, url: url, token: spec.GetCacheToken(), mode: spec.GetCacheMode(),	374		c := cacheRelay{relay: r, url: url, token: spec.GetCacheToken(), mode: spec.GetCacheMode(),

M internal/guest/registry_test.go +11 -3

 // SPDX-License-Identifier: MIT
 package guest
 import (
-	"bnema.dev/ermite-runner/internal/execstream/execpb"
 	"context"
 	"strings"
 	"testing"
+	"bnema.dev/ermite-runner/internal/execstream/execpb"
 )
 func TestDefaultDockerdMirror(t *testing.T) {
 		t.Fatal(args)
 	}
 }
+// A registry-only job (a port, no token, no mode, no results) gets the
+// loopback relay for dockerd and no runtime environment for its steps.
 func TestRegistryOnlyRelayHasNoRuntimeEnvironment(t *testing.T) {
 	target := serveCache(t, t.TempDir())
-	got := startCacheRelay(context.Background(), target, &execpb.JobSpec{CachePort: freePort(t), CacheToken: "unscoped"}, noBridge)
+	got := startCacheRelay(context.Background(), target, &execpb.JobSpec{CachePort: freePort(t)}, noBridge)
 	defer got.close()
 	if got.relay == nil {
 		t.Fatal("missing loopback relay")
 	}
-	if got.url != "" || got.containerURL != "" || got.token != "" || got.mode != "" || got.results || got.bridge != nil {
+	if got.token != "" {
+		t.Fatalf("token %q for a registry-only job", got.token)
+	}
+	if got.url != "" || got.containerURL != "" || got.mode != "" || got.results || got.bridge != nil {
 		t.Fatalf("runtime environment leaked: %+v", got)
 	}
 }

M internal/host/archives.go +11 -7

 // busy runner does not trip the forge's rate limit (codeload 429).
 //
 // It only ever improves a resolution: an action that cannot be cached (no cache,
-// no channel for this job, a ref that did not resolve to a sha, a download that
+// no cache scope for this job, a ref that did not resolve to a sha, a download that
 // failed, a full disk) keeps its original URL and token, and the sandbox
 // downloads it as before, with its retries and its messages. The sandbox prints
 // "Download action repository" in either case, so a hit reads like a miss in the
 		return
 	}
 	if _, ok, err := cacheScope(j.job, j.contexts); err != nil || !ok {
-		return // no channel for this job: the guest has no relay either
+		return // action archives are served only to jobs with a cache scope
 	}
 	ttl := time.Duration(j.timeoutMinutes())*time.Minute + cacheTokenMargin
 	for i, a := range orig {
 		if !ok {
 			continue
 		}
+		// fetched: this call ran the download itself. An invalid key, a wait
+		// that was cancelled or a joined download that failed are no misses.
+		var fetched bool
 		var fetchTime time.Duration
 		var bytes int64
 		hit, verify, err := rt.Archives.Ensure(ctx, key, func(ctx context.Context, w io.Writer) error {
+			fetched = true
 			start := time.Now()
 			cw := &archiveCountWriter{Writer: w}
 			err := j.fetchArchive(ctx, a, origin, cw)
 			bytes += cw.n
 			return err
 		})
-		if !hit {
+		if fetched {
 			j.mu.Lock()
-			j.timeline.add("archive_miss_ms", fetchTime)
+			j.timeline.addMs("archive_miss_ms", fetchTime)
-			j.timeline["archive_misses"]++
+			j.timeline.add("archive_misses", 1)
-			j.timeline["archive_miss_bytes"] += bytes
+			j.timeline.add("archive_miss_bytes", bytes)
 			j.mu.Unlock()
 		}
 		log := j.log.With().Str("action", a.NameWithOwner).Str("sha", a.ResolvedSha).Logger()
 			continue
 		}
 		event := log.Info().Bool("hit", hit).Int64("verify_ms", verify.Milliseconds())
-		if !hit {
+		if fetched {
 			event = event.Int64("fetch_ms", fetchTime.Milliseconds()).Int64("bytes", bytes)
 		}
 		event.Msg("action archive cache")

M internal/host/archives_test.go +46 -2


@@ -6,6 +6,7 @@ import (
6		"bytes"	6		"bytes"
7		"context"	7		"context"
8		"encoding/json"	8		"encoding/json"
			9	+	"io"
9		"net/http"	10		"net/http"
10		"net/http/httptest"	11		"net/http/httptest"
11		"path/filepath"	12		"path/filepath"

@@ -60,7 +61,7 @@ func newArchiveRig(t testing.T) archiveRig {
60		func (a archiveRig) job(id string) jobRun {	61		func (a archiveRig) job(id string) jobRun {
61		j := &jobRun{r: a.r, log: zerowrap.Logger{}, ctx: context.Background(),	62		j := &jobRun{r: a.r, log: zerowrap.Logger{}, ctx: context.Background(),
62		job: &ermite.Job{JobId: id, Ref: "refs/heads/main"},	63		job: &ermite.Job{JobId: id, Ref: "refs/heads/main"},
63	-	contexts: `{"github":{"repository":"o/r"}}`}	64	+	contexts: `{"github":{"repository":"o/r"}}`, timeline: timeline{}}
64		return j	65		return j
65		}	66		}
66			67

@@ -130,7 +131,7 @@ func TestCacheArchivesFallsBack(t *testing.T) {
130		}	131		}
131		}	132		}
132			133
133	-	// No cache channel for the job (no repository): the guest has no relay.	134	+	// No cache scope for the job (no repository): archives are not served.
134		o, c = a.action(strings.Repeat("d", 40))	135		o, c = a.action(strings.Repeat("d", 40))
135		j := a.job("j")	136		j := a.job("j")
136		j.contexts = `{}`	137		j.contexts = `{}`

@@ -182,3 +183,46 @@ func TestArchiveTimingLogs(t *testing.T) {
182		t.Fatal(j.timeline)	183		t.Fatal(j.timeline)
183		}	184		}
184		}	185		}
			186	+
			187	+	// Only a download the job ran itself is a miss: a failed one counts, a job
			188	+	// whose wait for another job's download is cancelled does not.
			189	+	func TestArchiveMissesCountFetches(t *testing.T) {
			190	+	a := newArchiveRig(t)
			191	+	a.status.Store(http.StatusTooManyRequests)
			192	+	failed := a.job("failed")
			193	+	o, c := a.action(strings.Repeat("a", 40))
			194	+	failed.cacheArchives(context.Background(), []ermite.ResolvedAction{o}, []execpb.ResolvedAction{c})
			195	+	if failed.timeline["archive_misses"] != 1 {
			196	+	t.Fatalf("failed fetch: %v", failed.timeline)
			197	+	}
			198	+
			199	+	a.status.Store(http.StatusOK)
			200	+	// A joiner whose wait is cancelled while another job downloads.
			201	+	release := make(chan struct{})
			202	+	started, done := make(chan struct{}), make(chan struct{})
			203	+	sha := strings.Repeat("b", 40)
			204	+	key, ok := actionscache.Key(originOf(a.srv.URL), "x/y", sha)
			205	+	if !ok {
			206	+	t.Fatal("key")
			207	+	}
			208	+	go func() {
			209	+	defer close(done)
			210	+	_, _, _ = a.r.Archives.Ensure(context.Background(), key, func(_ context.Context, w io.Writer) error {
			211	+	close(started)
			212	+	<-release
			213	+	_, err := io.WriteString(w, "tarball")
			214	+	return err
			215	+	})
			216	+	}()
			217	+	<-started
			218	+	ctx, cancel := context.WithCancel(context.Background())
			219	+	cancel()
			220	+	joiner := a.job("joiner")
			221	+	o, c = a.action(sha)
			222	+	joiner.cacheArchives(ctx, []ermite.ResolvedAction{o}, []execpb.ResolvedAction{c})
			223	+	close(release)
			224	+	<-done // the download writes under t.TempDir: let it finish before cleanup
			225	+	if _, ok := joiner.timeline["archive_misses"]; ok {
			226	+	t.Fatalf("cancelled joiner: %v", joiner.timeline)
			227	+	}
			228	+	}

M internal/host/cache.go +8 -4

 // port it has in the guest, the job's ACTIONS_RUNTIME_TOKEN, the cache mode and
 // whether the channel serves the results service. The channel exists when the
 // runner serves a cache and the job has a cache scope, the results service,
-// or the runner has a registry proxy. Without it, 0, "", "" and false.
+// or the cache serves a registry proxy. Without it, 0, "", "" and false.
-// A job without a cache scope gets an empty mode and a token that the cache
+// A job with the results service and no cache scope gets an empty mode and a
-// refuses. Registry-only jobs expose no runtime environment variables to steps.
+// token that the cache refuses. A registry-only job (neither) gets the port
+// alone: no token, and so no runtime environment variables for its steps.
 func (j *jobRun) runtimeFor(rt *Runner) (port uint32, token, mode string, results bool) {
 	if rt.Cache == nil {
 		return 0, "", "", false
 			mode = "write"
 		}
 	}
-	if !ok && !j.results && !rt.Registry {
+	if !ok && !j.results {
+		if rt.Cache.RegistryEnabled() {
+			return cachesvc.DefaultPort, "", "", false
+		}
 		return 0, "", "", false
 	}
 	var scopes []string

M internal/host/fake_test.go +6 -1

 	blockStart chan struct{}     // when set, Start waits on it
 	specs      []sandbox.JobSpec // what each Start was asked for
 	iso        sandbox.Isolation
+	bootMs     map[string]int64 // the guest's Hello.boot_ms; nil: dockerd=2
 }
 func (d *fakeDriver) Mode() sandbox.Mode           { return sandbox.Container }
 	}
 	d.started++
 	sc := d.script
+	bootMs := d.bootMs
+	if bootMs == nil {
+		bootMs = map[string]int64{"dockerd": 2}
+	}
 	d.mu.Unlock()
 	host, guest := net.Pipe()
 	s.mu.Lock()
 	go func() {
 		defer func() { _ = guest.Close() }()
 		c := execstream.NewConn(guest)
-		_ = c.Send(&execpb.Frame{Body: &execpb.Frame_Hello{Hello: &execpb.Hello{Version: "test", Sandbox: "container", BootMs: map[string]int64{"dockerd": 2}}}})
+		_ = c.Send(&execpb.Frame{Body: &execpb.Frame_Hello{Hello: &execpb.Hello{Version: "test", Sandbox: "container", BootMs: bootMs}}})
 		// NamedValue frames (secrets, variables) come before Run.
 		vals := &seenValues{secrets: map[string]string{}, vars: map[string]string{}}
 		var f *execpb.Frame

M internal/host/job.go +4 -10

 	cancelKind execpb.CancelKind // kind of the first cancel request
 	stopping   bool              // the runner is shutting down
+	mu    sync.Mutex
+	steps map[int32]*ermite.StepUpdate
+	// timeline is the job's phase timing, guarded by mu.
 	timeline timeline
-	mu       sync.Mutex
-	steps    map[int32]*ermite.StepUpdate
 	// What the guest has told us, for the validation of the next frame: the
 	// guest is untrusted code, its step numbers and lines are checked before
 	// they reach the server (see violate).
 		Hello: func(h *execpb.Hello) {
 			j.mu.Lock()
 			defer j.mu.Unlock()
-			if j.timeline == nil {
+			j.timeline.addGuestBoot(h.GetBootMs())
-				j.timeline = make(timeline)
-			}
-			for k, v := range h.BootMs {
-				if v >= 0 {
-					j.timeline["guest_"+k+"_ms"] = v
-				}
-			}
 		},
 		StepStarted: j.stepStarted, StepCompleted: j.stepCompleted,
 		Log:             j.logLine,

M internal/host/registry_test.go +31 -23

 // SPDX-License-Identifier: MIT
 package host
 import (
+	"testing"
 	"bnema.dev/ermite-api/gen/ermite"
 	"bnema.dev/ermite-runner/internal/cachesvc"
-	"net/http"
+	"bnema.dev/ermite-runner/internal/registryproxy"
-	"net/http/httptest"
-	"strings"
-	"testing"
 )
+// A job with neither a cache scope nor the results service gets the channel
+// for the registry proxy alone: its port and no token. Without the proxy, or
+// without the cache, it gets no channel.
 func TestRuntimeForRegistryOnly(t *testing.T) {
-	svc, err := cachesvc.New(cachesvc.Config{Dir: t.TempDir()})
+	proxy, err := registryproxy.New(registryproxy.Config{Dir: t.TempDir(), Max: 1 << 20})
 	if err != nil {
 		t.Fatal(err)
 	}
-	defer func() { _ = svc.CloseAll() }()
+	withRegistry, err := cachesvc.New(cachesvc.Config{Dir: t.TempDir(), Registry: proxy})
-	j := &jobRun{job: &ermite.Job{JobId: "j1", TimeoutMinutes: 5}}
+	if err != nil {
-	port, tok, mode, results := j.runtimeFor(&Runner{Cache: svc, Registry: true})
+		t.Fatal(err)
-	if port != cachesvc.DefaultPort || tok == "" || mode != "" || results {
+	}
-		t.Fatalf("%d %q %q %v", port, tok, mode, results)
+	defer func() { _ = withRegistry.CloseAll() }()
+	without, err := cachesvc.New(cachesvc.Config{Dir: t.TempDir()})
+	if err != nil {
+		t.Fatal(err)
 	}
-	job, scopes, _, err := svc.RuntimeToken(tok)
+	defer func() { _ = without.CloseAll() }()
-	if err != nil || job != "j1" || strings.Contains(scopes, "Actions.Results") {
+	// No repository in the contexts: the job has no cache scope.
-		t.Fatalf("%q %q %v", job, scopes, err)
+	j := &jobRun{job: &ermite.Job{JobId: "j1", TimeoutMinutes: 5}}
+	port, tok, mode, results := j.runtimeFor(&Runner{Cache: withRegistry})
+	if port != cachesvc.DefaultPort || tok != "" || mode != "" || results {
+		t.Fatalf("registry only: %d %q %q %v", port, tok, mode, results)
 	}
-	req := httptest.NewRequest(http.MethodPost, cachesvc.APIPath+"caches", strings.NewReader(`{"key":"k","version":"v","cacheSize":1}`))
+	if p, _, _, _ := j.runtimeFor(&Runner{Cache: without}); p != 0 {
-	req.Header.Set("Authorization", "Bearer "+tok)
+		t.Fatal("proxy off still supplies a channel")
-	w := httptest.NewRecorder()
-	svc.Handler().ServeHTTP(w, req)
-	if w.Code != http.StatusForbidden {
-		t.Fatalf("unscoped token allowed cache: %d", w.Code)
 	}
-	if p, _, _, _ := j.runtimeFor(&Runner{Cache: svc}); p != 0 {
+	if p, _, _, _ := j.runtimeFor(&Runner{}); p != 0 {
-		t.Fatal("proxy off still supplies channel")
+		t.Fatal("cache off still supplies a channel")
 	}
-	if p, _, _, _ := j.runtimeFor(&Runner{Registry: true}); p != 0 {
+	// A job with a cache scope still gets its token, proxy or not.
-		t.Fatal("cache off still supplies channel")
+	scoped := &jobRun{job: &ermite.Job{JobId: "j1", Ref: "refs/heads/main", TimeoutMinutes: 5}, contexts: `{"github":{"repository":"o/r"}}`}
+	if _, tok, mode, _ := scoped.runtimeFor(&Runner{Cache: withRegistry}); tok == "" || mode != "write" {
+		t.Fatalf("scoped job: %q %q", tok, mode)
 	}
 }

M internal/host/resolve.go +2 -2


@@ -116,8 +116,8 @@ func (j jobRun) resolveActions(ctx context.Context, req execpb.ResolveRequest)
116		defer func() {	116		defer func() {
117		j.mu.Lock()	117		j.mu.Lock()
118		defer j.mu.Unlock()	118		defer j.mu.Unlock()
119	-	j.timeline.add("resolve_ms", time.Since(start))	119	+	j.timeline.addMs("resolve_ms", time.Since(start))
120	-	j.timeline["resolve_calls"]++	120	+	j.timeline.add("resolve_calls", 1)
121		}()	121		}()
122		fail := func(format string, a ...any) *execpb.ResolveResponse {	122		fail := func(format string, a ...any) *execpb.ResolveResponse {
123		return &execpb.ResolveResponse{Error: fmt.Sprintf(format, a...)}	123		return &execpb.ResolveResponse{Error: fmt.Sprintf(format, a...)}

M internal/host/resolve_test.go +1 -1


@@ -48,7 +48,7 @@ func (c resolveStub) ResolveActions(_ context.Context, r connect.Request[ermit
48		}	48		}
49			49
50		func resolveJob(c jobClient) *jobRun {	50		func resolveJob(c jobClient) *jobRun {
51	-	j := &jobRun{client: c, ctx: context.Background(), job: &ermite.Job{JobId: "j1"}, log: zerowrap.Logger{},	51	+	j := &jobRun{client: c, ctx: context.Background(), job: &ermite.Job{JobId: "j1"}, log: zerowrap.Logger{}, timeline: timeline{},
52		t: Timing{CallTimeout: time.Second, BackoffMin: time.Millisecond, BackoffMax: time.Millisecond}}	52		t: Timing{CallTimeout: time.Second, BackoffMin: time.Millisecond, BackoffMax: time.Millisecond}}
53		j.setOrigins(`{"github":{"server_url":"https://ermite.example"}}`, "https://ermite.example", "")	53		j.setOrigins(`{"github":{"server_url":"https://ermite.example"}}`, "https://ermite.example", "")
54		return j	54		return j

M internal/host/results_test.go +1 -1


@@ -485,7 +485,7 @@ func TestAnnotationsWaitingForTheServerAreBounded(t *testing.T) {
485		tm := fastTiming()	485		tm := fastTiming()
486		ctx, cancel := context.WithCancel(context.Background())	486		ctx, cancel := context.WithCancel(context.Background())
487		defer cancel()	487		defer cancel()
488	-	j := &jobRun{client: cl, t: tm, ctx: ctx, job: &ermite.Job{JobId: "j1"}, steps: map[int32]*ermite.StepUpdate{},	488	+	j := &jobRun{client: cl, t: tm, ctx: ctx, job: &ermite.Job{JobId: "j1"}, steps: map[int32]*ermite.StepUpdate{}, timeline: timeline{},
489		annSeen: map[int32]bool{}, sumSeen: map[int32]bool{}, features: jobFeatures{annotations: true},	489		annSeen: map[int32]bool{}, sumSeen: map[int32]bool{}, features: jobFeatures{annotations: true},
490		cancelCh: make(chan *execpb.Cancel, 1), log: zerowrap.New(zerowrap.Config{Level: "debug", Format: "json", Output: logs})}	490		cancelCh: make(chan *execpb.Cancel, 1), log: zerowrap.New(zerowrap.Config{Level: "debug", Format: "json", Output: logs})}
491		j.outbox = newOutbox(cl, "j1", tm, func(error) {}, j.log)	491		j.outbox = newOutbox(cl, "j1", tm, func(error) {}, j.log)

M internal/host/runner.go +1 -3

 	// which the guest reaches on its loopback (cachesvc.DefaultPort), and each
 	// job gets a token scoped to its repository and refs.
 	Cache *cachesvc.Service
-	// Registry enables the runtime channel for anonymous registry-only jobs.
-	Registry bool
 	// Archives, when not nil (and Cache is), is the host's cache of action
 	// archives: the host downloads them and the sandbox reads them from the cache
 	// channel (see cacheArchives). ArchiveHTTP is its HTTP client (tests); nil
 		r: r, job: job, client: r.Job(job.JobToken), t: t, ctx: jctx, cancel: cancel,
 		log:        zerowrap.Logger{Logger: log.With().Str("job_id", job.JobId).Int64("run_id", job.RunId).Int32("attempt", job.Attempt).Logger()},
 		receivedAt: r.clock(), cancelCh: make(chan *execpb.Cancel, 1), steps: map[int32]*ermite.StepUpdate{},
-		annSeen: map[int32]bool{}, sumSeen: map[int32]bool{},
+		annSeen: map[int32]bool{}, sumSeen: map[int32]bool{}, timeline: timeline{},
 	}
 	jr.log.Info().Str("job_key", job.JobKey).Msg("job assigned")
 	r.active.Add(1)

M internal/host/runner_test.go +9 -15

 	if err := stop(); err != nil {
 		t.Fatal(err)
 	}
-	for _, line := range strings.Split(h.logs.String(), "\n") {
+	fields := jobFinished(t, h.logs.String())
-		var fields map[string]any
+	ms := map[string]float64{}
-		if json.Unmarshal([]byte(line), &fields) != nil || fields["message"] != "job finished" {
+	for _, key := range []string{"ack_ms", "sandbox_ms", "run_ms", "stop_ms", "complete_ms", "total_ms", "sb_socket_ms", "guest_dockerd_ms"} {
-			continue
+		value, ok := fields[key].(float64)
-		}
+		if !ok || value < 0 {
-		for _, key := range []string{"ack_ms", "sandbox_ms", "run_ms", "stop_ms", "complete_ms", "total_ms", "sb_socket_ms", "guest_dockerd_ms"} {
+			t.Errorf("%s = %v", key, fields[key])
-			value, ok := fields[key].(float64)
-			if !ok || value < 0 {
-				t.Errorf("%s = %v", key, fields[key])
-			}
-		}
-		if fields["total_ms"].(float64) < fields["sandbox_ms"].(float64)+fields["run_ms"].(float64) {
-			t.Error("total shorter than sandbox + run")
 		}
+		ms[key] = value
 	}
-	if !strings.Contains(h.logs.String(), `"message":"job finished"`) {
+	if ms["total_ms"] < ms["sandbox_ms"]+ms["run_ms"] {
-		t.Fatalf("no job finished log: %s", h.logs.String())
+		t.Error("total shorter than sandbox + run")
 	}
 	s := h.srv
 	s.mu.Lock()

M internal/host/shutdown_test.go +1 -1


@@ -81,7 +81,7 @@ func TestReportBudgetFollowsTheShutdownChannel(t *testing.T) {
81		cl := &stubClient{}	81		cl := &stubClient{}
82		ctx, cancel := context.WithCancel(context.Background())	82		ctx, cancel := context.WithCancel(context.Background())
83		defer cancel()	83		defer cancel()
84	-	j := &jobRun{client: cl, t: tm, ctx: ctx, job: &ermite.Job{JobId: "j1"}, steps: map[int32]*ermite.StepUpdate{},	84	+	j := &jobRun{client: cl, t: tm, ctx: ctx, job: &ermite.Job{JobId: "j1"}, steps: map[int32]*ermite.StepUpdate{}, timeline: timeline{},
85		cancelCh: make(chan *execpb.Cancel, 1), log: zerowrap.New(zerowrap.Config{Level: "debug", Format: "json", Output: &logBuf{}})}	85		cancelCh: make(chan *execpb.Cancel, 1), log: zerowrap.New(zerowrap.Config{Level: "debug", Format: "json", Output: &logBuf{}})}
86		j.outbox = newOutbox(cl, "j1", tm, func(error) {}, j.log)	86		j.outbox = newOutbox(cl, "j1", tm, func(error) {}, j.log)
87		j.outbox.start(ctx)	87		j.outbox.start(ctx)

M internal/host/timeline.go +34 -21


@@ -4,36 +4,49 @@ package host
4			4
5		import "time"	5		import "time"
6			6
7	-	// timeline is protected by jobRun.mu.	7	+	// timeline is the per-phase timing of a job, logged on its "job finished"
			8	+	// line: durations in milliseconds (the *_ms keys) and counters. A jobRun
			9	+	// creates it once, in startJob; it is guarded by jobRun.mu.
8		type timeline map[string]int64	10		type timeline map[string]int64
9			11
10	-	func (t *timeline) set(key string, d time.Duration) {	12	+	// setMs records d as the duration of a phase, in whole milliseconds.
11	-	if *t == nil {	13	+	func (t timeline) setMs(key string, d time.Duration) { t[key] = max(0, d.Milliseconds()) }
12	-	*t = make(timeline)	14	+
13	-	}	15	+	// addMs adds d to the summed duration of a phase that may run several times.
14	-	(*t)[key] = max(0, d.Milliseconds())	16	+	func (t timeline) addMs(key string, d time.Duration) { t[key] += max(0, d.Milliseconds()) }
15	-	}	17	+
16	-	func (t *timeline) add(key string, d time.Duration) {	18	+	// add adds n to a counter.
17	-	if *t == nil {	19	+	func (t timeline) add(key string, n int64) { t[key] += n }
18	-	*t = make(timeline)	20	+
19	-	}	21	+	// guestBootKeys are the boot phases the guest may report in its Hello: the
20	-	(*t)[key] += max(0, d.Milliseconds())	22	+	// host logs these as guest_<key>_ms and ignores any other key, as it does not
21	-	}	23	+	// trust the guest.
22	-	func (t timeline) fields() map[string]any {	24	+	var guestBootKeys = []string{"kernel", "setup", "containerd", "dockerd"}
23	-	fields := make(map[string]any, len(t))	25	+
24	-	for k, v := range t {	26	+	// addGuestBoot records the guest's boot phases: known keys with a value of
25	-	fields[k] = v	27	+	// at least 0.
			28	+	func (t timeline) addGuestBoot(ms map[string]int64) {
			29	+	for _, k := range guestBootKeys {
			30	+	if v, ok := ms[k]; ok && v >= 0 {
			31	+	t["guest_"+k+"_ms"] = v
			32	+	}
26		}	33		}
27	-	return fields
28		}	34		}
			35	+
29		func (j *jobRun) timeSet(key string, d time.Duration) {	36		func (j *jobRun) timeSet(key string, d time.Duration) {
30		j.mu.Lock()	37		j.mu.Lock()
31		defer j.mu.Unlock()	38		defer j.mu.Unlock()
32	-	j.timeline.set(key, d)	39	+	j.timeline.setMs(key, d)
33		}	40		}
			41	+
			42	+	// timingFields sets total_ms and returns a copy of the timeline for the log.
34		func (j *jobRun) timingFields() map[string]any {	43		func (j *jobRun) timingFields() map[string]any {
35		j.mu.Lock()	44		j.mu.Lock()
36		defer j.mu.Unlock()	45		defer j.mu.Unlock()
37	-	j.timeline.set("total_ms", time.Since(j.receivedAt))	46	+	j.timeline.setMs("total_ms", time.Since(j.receivedAt))
38	-	return j.timeline.fields()	47	+	fields := make(map[string]any, len(j.timeline))
			48	+	for k, v := range j.timeline {
			49	+	fields[k] = v
			50	+	}
			51	+	return fields
39		}	52		}

M internal/host/timeline_test.go +64 -16


@@ -3,24 +3,72 @@
3		package host	3		package host
4			4
5		import (	5		import (
			6	+	"encoding/json"
			7	+	"fmt"
			8	+	"strings"
6		"testing"	9		"testing"
7	-	"time"	10	+
			11	+	ermite "bnema.dev/ermite-api/gen/ermite"
8		)	12		)
9			13
10	-	func TestTimeline(t *testing.T) {	14	+	// jobFinished returns the fields of the one "job finished" line of logs.
11	-	var timeline timeline	15	+	func jobFinished(t *testing.T, logs string) map[string]any {
12	-	timeline.set("run_ms", 3*time.Millisecond)	16	+	t.Helper()
13	-	timeline.add("run_ms", 2*time.Millisecond)	17	+	var found map[string]any
14	-	fields := timeline.fields()	18	+	for _, line := range strings.Split(logs, "\n") {
15	-	if fields["run_ms"] != int64(5) {	19	+	var fields map[string]any
16	-	t.Fatal(fields)	20	+	if json.Unmarshal([]byte(line), &fields) != nil \|\| fields["message"] != "job finished" {
17	-	}	21	+	continue
18	-	fields["run_ms"] = int64(99)	22	+	}
19	-	if timeline.fields()["run_ms"] != int64(5) {	23	+	if found != nil {
20	-	t.Fatal("shared fields")	24	+	t.Fatal("two job finished lines")
21	-	}	25	+	}
22	-	timeline.set("run_ms", time.Millisecond)	26	+	found = fields
23	-	if timeline.fields()["run_ms"] != int64(1) {	27	+	}
24	-	t.Fatal(timeline)	28	+	if found == nil {
			29	+	t.Fatalf("no job finished line: %s", logs)
			30	+	}
			31	+	return found
			32	+	}
			33	+
			34	+	// The guest is untrusted: of its Hello.boot_ms, the job finished line keeps
			35	+	// only the known phases with a value of at least 0, whatever else it sends.
			36	+	func TestHostileGuestBootTimings(t *testing.T) {
			37	+	h := newHarness(t, simpleScript("hello"))
			38	+	hostile := map[string]int64{
			39	+	"kernel": 5,
			40	+	"dockerd": -7, // negative: dropped
			41	+	"setup": 0,
			42	+	"total": 1, // would overwrite a host field as guest_total_ms
			43	+	"": 3,
			44	+	"x\"y": 4,
			45	+	"../../etc": 5,
			46	+	"containerd": 9,
			47	+	}
			48	+	for i := range 500 {
			49	+	hostile[fmt.Sprint("junk", i)] = int64(i)
			50	+	}
			51	+	h.driver.bootMs = hostile
			52	+	h.srv.jobs = []*ermite.Job{testJob("j1")}
			53	+	stop := h.run(t)
			54	+	eventually(t, "CompleteJob", func() bool { return h.srv.completeCount() == 1 })
			55	+	if err := stop(); err != nil {
			56	+	t.Fatal(err)
			57	+	}
			58	+	fields := jobFinished(t, h.logs.String())
			59	+	var guest []string
			60	+	for k := range fields {
			61	+	if strings.HasPrefix(k, "guest_") {
			62	+	guest = append(guest, k)
			63	+	}
			64	+	}
			65	+	want := map[string]float64{"guest_kernel_ms": 5, "guest_setup_ms": 0, "guest_containerd_ms": 9}
			66	+	if len(guest) != len(want) {
			67	+	t.Fatalf("guest fields %v, want %v", guest, want)
			68	+	}
			69	+	for k, v := range want {
			70	+	if got, ok := fields[k].(float64); !ok \|\| got != v {
			71	+	t.Errorf("%s = %v, want %v", k, fields[k], v)
			72	+	}
25		}	73		}
26		}	74		}

M internal/registryproxy/accept.go +1 -0

@@ -1,4 +1,5 @@
1	// SPDX-License-Identifier: MIT	1		// SPDX-License-Identifier: MIT
		2	+
2	package registryproxy	3		package registryproxy
3		4
4	import (	5		import (

A internal/registryproxy/accept_test.go +112 -0

@@ -0,0 +1,112 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package registryproxy
	4	+
	5	+	import (
	6	+	"fmt"
	7	+	"io"
	8	+	"net/http"
	9	+	"net/http/httptest"
	10	+	"strings"
	11	+	"sync"
	12	+	"testing"
	13	+	)
	14	+
	15	+	// Accept is reduced to the known manifest types, so its variants map to
	16	+	// a bounded set of keys.
	17	+	func TestNormalizeAccept(t *testing.T) {
	18	+	const index, v2 = "application/vnd.oci.image.index.v1+json", "application/vnd.docker.distribution.manifest.v2+json"
	19	+	all := normalizeAccept("")
	20	+	if strings.Count(all, ",") != len(manifestTypes)-1 {
	21	+	t.Fatalf("default key %q", all)
	22	+	}
	23	+	for _, tc := range []struct{ in, want string }{
	24	+	{index + ";q=0.9, " + strings.ToUpper(v2) + " , " + index, v2 + "," + index},
	25	+	{" " + index + " ;q=0.1", index},
	26	+	{"text/html, /", all},
	27	+	{"", all},
	28	+	} {
	29	+	if got := normalizeAccept(tc.in); got != tc.want {
	30	+	t.Errorf("normalizeAccept(%q) = %q, want %q", tc.in, got, tc.want)
	31	+	}
	32	+	}
	33	+	}
	34	+
	35	+	// The Accept variants a job sends for one tag reach upstream with every known
	36	+	// type they name, and make at most one tag file per subset of the known types.
	37	+	// dockerd's legacy client sends one Accept line per type; the 8 KiB bound
	38	+	// applies to the joined lines.
	39	+	func TestAcceptTagFiles(t *testing.T) {
	40	+	types := []string{
	41	+	"application/vnd.oci.image.index.v1+json",
	42	+	"application/vnd.docker.distribution.manifest.list.v2+json",
	43	+	"application/vnd.docker.distribution.manifest.v2+json",
	44	+	}
	45	+	for _, tc := range []struct {
	46	+	name string
	47	+	accepts func(i int) []string // the Accept lines of request i
	48	+	requests int
	49	+	maxTags int
	50	+	code int
	51	+	}{
	52	+	{"multi-line", func(int) []string { return types }, 2, 1, http.StatusOK},
	53	+	{"variants", func(i int) []string {
	54	+	switch i % 3 {
	55	+	case 1:
	56	+	return []string{fmt.Sprintf("x/only-junk-%d", i)}
	57	+	case 2:
	58	+	return []string{fmt.Sprintf("%s;q=0.%d, x/junk-%d, %s", types[0], i%10, i, types[1])}
	59	+	}
	60	+	return []string{fmt.Sprintf("%s;q=0.%d, x/junk-%d", types[0], i%10, i)}
	61	+	}, 300, 3, http.StatusOK},
	62	+	{"oversized", func(int) []string {
	63	+	lines := make([]string, 200)
	64	+	for i := range lines {
	65	+	lines[i] = strings.Repeat("x", 50)
	66	+	}
	67	+	return lines
	68	+	}, 1, 0, http.StatusNotFound},
	69	+	} {
	70	+	t.Run(tc.name, func(t *testing.T) {
	71	+	var mu sync.Mutex
	72	+	var seen []string
	73	+	srv := upstreamWith(t, func(w http.ResponseWriter, r *http.Request) {
	74	+	mu.Lock()
	75	+	seen = append(seen, strings.Join(r.Header.Values("Accept"), ","))
	76	+	mu.Unlock()
	77	+	w.Header().Set("Content-Type", types[0])
	78	+	w.Header().Set("Docker-Content-Digest", sum(manifestBody))
	79	+	if r.Method == http.MethodGet {
	80	+	_, _ = io.WriteString(w, manifestBody)
	81	+	}
	82	+	})
	83	+	p := proxyWith(t, srv, Config{})
	84	+	for i := range tc.requests {
	85	+	r := httptest.NewRequest(http.MethodGet, "/v2/library/alpine/manifests/3", nil)
	86	+	lines := tc.accepts(i)
	87	+	for _, line := range lines {
	88	+	r.Header.Add("Accept", line)
	89	+	}
	90	+	w := httptest.NewRecorder()
	91	+	p.ServeHTTP(w, r)
	92	+	if w.Code != tc.code {
	93	+	t.Fatalf("request %d: %d", i, w.Code)
	94	+	}
	95	+	if tc.code != http.StatusOK {
	96	+	continue
	97	+	}
	98	+	mu.Lock()
	99	+	got := seen[len(seen)-1]
	100	+	mu.Unlock()
	101	+	for _, typ := range types {
	102	+	if strings.Contains(strings.Join(lines, ","), typ) && !strings.Contains(got, typ) {
	103	+	t.Errorf("upstream Accept %q lacks %s", got, typ)
	104	+	}
	105	+	}
	106	+	}
	107	+	if n := accounted(t, p.s); n > tc.maxTags \|\| (tc.maxTags > 0 && n == 0) {
	108	+	t.Fatalf("%d tag files for %d requests", n, tc.requests)
	109	+	}
	110	+	})
	111	+	}
	112	+	}

M internal/registryproxy/clients.go +28 -49

 // SPDX-License-Identifier: MIT
 package registryproxy
 import (
 // upstreamClients returns the metadata client (auth, manifests: a total
 // timeout and no redirects) and the blob client (no total timeout, a response
 // header timeout, and redirects that drop the bearer when leaving the host).
-func upstreamClients(c Config) (*http.Client, *http.Client) {
+func upstreamClients(requestTimeout time.Duration) (metadata, blobs *http.Client) {
-	base := http.Client{}
+	metadata = &http.Client{
-	if c.Client != nil {
+		Timeout:       requestTimeout,
-		base = *c.Client
+		CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
 	}
-	metadata := base
+	blobs = &http.Client{
-	metadata.Timeout = c.RequestTimeout
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
-	metadata.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }
+			if len(via) >= 10 {
-	blobs := base
+				return errors.New("too many registry redirects")
-	blobs.Timeout = 0
+			}
-	transport := base.Transport
+			if len(via) > 0 && req.URL.Host != via[0].URL.Host {
-	if transport == nil {
+				req.Header.Del("Authorization")
-		transport = http.DefaultTransport
+			}
+			return nil
+		},
 	}
-	if tr, ok := transport.(*http.Transport); ok {
+	if tr, ok := http.DefaultTransport.(*http.Transport); ok {
 		tr = tr.Clone()
-		tr.ResponseHeaderTimeout = c.RequestTimeout
+		tr.ResponseHeaderTimeout = requestTimeout
 		blobs.Transport = tr
 	}
-	blobs.CheckRedirect = func(req *http.Request, via []*http.Request) error {
+	return metadata, blobs
-		if len(via) >= 10 {
-			return errors.New("too many registry redirects")
-		}
-		if len(via) > 0 && req.URL.Host != via[0].URL.Host {
-			req.Header.Del("Authorization")
-		}
-		return nil
-	}
-	return &metadata, &blobs
 }
 // idleBody serializes timer reset/stop against expiry. Cancellation interrupts
 	}
 	return n, err
 }
 func (b *idleBody) Close() error {
 	b.mu.Lock()
 	b.expired = true
 	b.cancel()
 	return b.ReadCloser.Close()
 }
+// blob GETs a blob. Each attempt has only a header deadline (RequestTimeout)
+// and then a body progress deadline (BlobIdleTimeout), and no total transfer
+// deadline: the response body is an idleBody that owns the attempt's context.
 func (u *upstream) blob(ctx context.Context, name, digest string) (*http.Response, error) {
-	// Token acquisition has a total timeout; the subsequent blob request has only
+	return u.authorized(ctx, name, func(tok string) (*http.Response, error) {
-	// a header/body progress deadline and no total transfer deadline.
-	authCtx, authCancel := context.WithTimeout(ctx, u.requestTimeout)
-	tok, err := u.token(authCtx, name, false)
-	authCancel()
-	if err != nil {
-		return nil, err
-	}
-	for attempt := 0; attempt < 2; attempt++ {
 		bodyCtx, cancel := context.WithCancel(ctx)
 		lease := &idleBody{cancel: cancel, idle: u.blobIdle}
 		lease.timer = time.AfterFunc(u.requestTimeout, func() { lease.mu.Lock(); lease.expired = true; cancel(); lease.mu.Unlock() })
-		req, err := http.NewRequestWithContext(bodyCtx, http.MethodGet, u.base+"/v2/"+name+"/blobs/"+digest, nil)
+		req, err := http.NewRequestWithContext(bodyCtx, http.MethodGet, u.base+"/v2/"+name+"/"+kindBlob.dir()+"/"+digest, nil)
 		if err != nil {
 			lease.timer.Stop()
 			cancel()
 			return nil, err
 		}
 		lease.ReadCloser = resp.Body
+		resp.Body = lease
 		if resp.StatusCode == http.StatusOK {
 			lease.mu.Lock()
 			if !lease.expired {
 				lease.timer.Reset(u.blobIdle)
 			}
 			lease.mu.Unlock()
-			resp.Body = lease
-			return resp, nil
 		}
-		_ = lease.Close()
+		return resp, nil
-		if (resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden) && attempt == 0 {
+	})
-			authCtx, authCancel := context.WithTimeout(ctx, u.requestTimeout)
-			tok, err = u.token(authCtx, name, true)
-			authCancel()
-			if err != nil {
-				return nil, err
-			}
-			continue
-		}
-		return nil, upstreamError{resp.StatusCode}
-	}
-	return nil, upstreamError{http.StatusUnauthorized}
 }

M internal/registryproxy/flights_test.go +123 -306

@@ -1,8 +1,8 @@
1		// SPDX-License-Identifier: MIT	1		// SPDX-License-Identifier: MIT
			2	+
2		package registryproxy	3		package registryproxy
3			4
4		import (	5		import (
5	-	"bytes"
6		"context"	6		"context"
7		"fmt"	7		"fmt"
8		"io"	8		"io"

@@ -19,49 +19,37 @@ import (
19		"github.com/bnema/zerowrap"	19		"github.com/bnema/zerowrap"
20		)	20		)
21			21
22	-	// stallingUpstream serves a blob's first bytes and then holds the body until	22	+	// Concurrent requests for one digest share a single fetch.
23	-	// the request is cancelled; it counts started and cancelled downloads.	23	+	func TestSingleFlight(t *testing.T) {
24	-	type stallingUpstream struct {	24	+	s := mustOpen(t, storeCfg(t.TempDir(), 1<<20))
25	-	srv *httptest.Server	25	+	var calls atomic.Int32
26	-	started, cancelled atomic.Int32	26	+	var wg sync.WaitGroup
27	-	}	27	+	start := make(chan struct{})
28	-		28	+	for range 20 {
29	-	func newStallingUpstream(t testing.T) stallingUpstream {	29	+	wg.Go(func() {
30	-	t.Helper()	30	+	<-start
31	-	u := &stallingUpstream{}	31	+	f, _, _, err := s.ensure(context.Background(), kindBlob, sum("content"), func(_ context.Context, w io.Writer) (string, error) {
32	-	u.srv = upstreamWith(t, func(w http.ResponseWriter, r *http.Request) {	32	+	calls.Add(1)
33	-	u.started.Add(1)	33	+	_, err := io.WriteString(w, "content")
34	-	w.Header().Set("Content-Length", "1000000")	34	+	return "", err
35	-	_, _ = io.WriteString(w, "partial")	35	+	}, keepalive{})
36	-	w.(http.Flusher).Flush()	36	+	if err != nil {
37	-	select {	37	+	t.Error(err)
38	-	case <-r.Context().Done():	38	+	return
39	-	u.cancelled.Add(1)	39	+	}
40	-	case <-time.After(10 * time.Second):	40	+	_ = f.Close()
41	-	}	41	+	})
42	-	})	42	+	}
43	-	return u	43	+	close(start)
44	-	}	44	+	wg.Wait()
45	-		45	+	if calls.Load() != 1 {
46	-	func blobPath(i int) string { return "/v2/library/alpine/blobs/" + sum(fmt.Sprint("blob-", i)) }	46	+	t.Fatalf("fetches %d", calls.Load())
47	-		47	+	}
48	-	// serveCancelled starts a request and cancels it once its download has begun.
49	-	func serveCancelled(t testing.T, p Proxy, u *stallingUpstream, path string, started int32) {
50	-	t.Helper()
51	-	ctx, cancel := context.WithCancel(context.Background())
52	-	done := make(chan struct{})
53	-	go func() {
54	-	defer close(done)
55	-	p.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, path, nil).WithContext(ctx))
56	-	}()
57	-	waitFor(t, p.s, "the download to start", func() bool { return u.started.Load() >= started })
58	-	cancel()
59	-	<-done
60		}	48		}
61			49
62	-	// MAJOR 1a: a guest that requests many digests and disconnects leaves no	50	+	// A guest that requests many digests and disconnects leaves no download
63	-	// download behind: once a flight has no waiter for the grace period, it is	51	+	// behind: once a flight has no waiter for the grace period, it is cancelled
64	-	// cancelled upstream and its temporary file removed.	52	+	// upstream and its temporary file removed.
65		func TestAbandonedDownloadsCancelled(t *testing.T) {	53		func TestAbandonedDownloadsCancelled(t *testing.T) {
66		u := newStallingUpstream(t)	54		u := newStallingUpstream(t)
67		const n = 20	55		const n = 20

@@ -82,7 +70,7 @@ func TestAbandonedDownloadsCancelled(t *testing.T) {
82		})	70		})
83		}	71		}
84			72
85	-	// MAJOR 1a: a request that joins within the grace period keeps the download.	73	+	// A request that joins within the grace period keeps the download.
86		func TestRejoinWithinGraceKeepsDownload(t *testing.T) {	74		func TestRejoinWithinGraceKeepsDownload(t *testing.T) {
87		body := strings.Repeat("layer", 4000)	75		body := strings.Repeat("layer", 4000)
88		release := make(chan struct{})	76		release := make(chan struct{})

@@ -109,7 +97,7 @@ func TestRejoinWithinGraceKeepsDownload(t *testing.T) {
109		defer close(first)	97		defer close(first)
110		p.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, path, nil).WithContext(ctx))	98		p.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, path, nil).WithContext(ctx))
111		}()	99		}()
112	-	key := "blobs/" + sum(body)[7:]	100	+	key := kindBlob.key(sum(body)[7:])
113		waitFor(t, p.s, "the first waiter", func() bool { return waiters(p.s, key) == 1 })	101		waitFor(t, p.s, "the first waiter", func() bool { return waiters(p.s, key) == 1 })
114		cancel()	102		cancel()
115		<-first	103		<-first

@@ -124,255 +112,109 @@ func TestRejoinWithinGraceKeepsDownload(t *testing.T) {
124		}	112		}
125		}	113		}
126			114
127	-	// MAJOR 1b: at most MaxDownloads cold blobs download at once; one more waits	115	+	// The shared download does not belong to the client that started it. That
128	-	// for SlotWait and then gets 503 (dockerd retries or pulls from Docker Hub),	116	+	// client disconnects mid-download; a second client, joining within the grace
129	-	// and a slot frees once a download ends.	117	+	// period, still gets the blob and the blob is committed, from one upstream
130	-	func TestColdDownloadCap(t *testing.T) {	118	+	// fetch.
131	-	u := newStallingUpstream(t)	119	+	func TestBlobFlightSurvivesInitiatorDisconnect(t *testing.T) {
132	-	const limit = 3	120	+	body := strings.Repeat("layer", 4000)
133	-	const wait = 50 * time.Millisecond	121	+	half, release := make(chan struct{}), make(chan struct{})
134	-	p := proxyWith(t, u.srv, Config{AbandonGrace: 10 * time.Millisecond, MaxDownloads: limit, SlotWait: wait})	122	+	var releaseOnce sync.Once
135	-	ctx, cancel := context.WithCancel(context.Background())	123	+	unblock := func() { releaseOnce.Do(func() { close(release) }) }
136	-	var wg sync.WaitGroup	124	+	var gets atomic.Int32
137	-	for i := range limit {
138	-	wg.Go(func() {
139	-	p.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, blobPath(i), nil).WithContext(ctx))
140	-	})
141	-	}
142	-	waitFor(t, p.s, "the cap to fill", func() bool { return p.s.downloads == limit })
143	-	start := time.Now()
144	-	w := request(t, p, http.MethodGet, blobPath(limit))
145	-	if w.Code != http.StatusServiceUnavailable \|\| w.Header().Get("Retry-After") == "" {
146	-	t.Fatalf("over the cap: %d", w.Code)
147	-	}
148	-	if d := time.Since(start); d < wait {
149	-	t.Fatalf("503 after %v, before the slot wait", d)
150	-	}
151	-	// Joining a running download takes no slot.
152	-	joined := make(chan int, 1)
153	-	go func() {
154	-	jctx, jcancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
155	-	defer jcancel()
156	-	jw := httptest.NewRecorder()
157	-	p.ServeHTTP(jw, httptest.NewRequest(http.MethodGet, blobPath(0), nil).WithContext(jctx))
158	-	joined <- jw.Code
159	-	}()
160	-	if code := <-joined; code == http.StatusServiceUnavailable {
161	-	t.Fatal("a joiner was refused a slot")
162	-	}
163	-	cancel()
164	-	wg.Wait()
165	-	waitFor(t, p.s, "the slots to free", func() bool { return p.s.downloads == 0 })
166	-	if u.started.Load() != limit {
167	-	t.Fatalf("%d downloads started", u.started.Load())
168	-	}
169	-	}
170	-
171	-	// MAJOR 2: dockerd's legacy client sends one Accept line per type. All of them
172	-	// reach upstream, and they make one tag file.
173	-	func TestMultiLineAccept(t *testing.T) {
174	-	types := []string{
175	-	"application/vnd.oci.image.index.v1+json",
176	-	"application/vnd.docker.distribution.manifest.list.v2+json",
177	-	"application/vnd.docker.distribution.manifest.v2+json",
178	-	}
179	-	var mu sync.Mutex
180	-	var seen []string
181		srv := upstreamWith(t, func(w http.ResponseWriter, r *http.Request) {	125		srv := upstreamWith(t, func(w http.ResponseWriter, r *http.Request) {
182	-	mu.Lock()	126	+	gets.Add(1)
183	-	seen = append(seen, strings.Join(r.Header.Values("Accept"), ","))	127	+	w.Header().Set("Content-Length", fmt.Sprint(len(body)))
184	-	mu.Unlock()	128	+	_, _ = io.WriteString(w, body[:len(body)/2])
185	-	w.Header().Set("Content-Type", types[0])	129	+	w.(http.Flusher).Flush()
186	-	w.Header().Set("Docker-Content-Digest", sum(manifestBody))	130	+	close(half)
187	-	if r.Method == http.MethodGet {	131	+	<-release
188	-	_, _ = io.WriteString(w, manifestBody)	132	+	_, _ = io.WriteString(w, body[len(body)/2:])
189	-	}
190		})	133		})
			134	+	t.Cleanup(unblock) // before srv.Close, which waits for the handler
191		p := proxyWith(t, srv, Config{})	135		p := proxyWith(t, srv, Config{})
192	-	for range 2 {	136	+	path := "/v2/library/alpine/blobs/" + sum(body)
193	-	r := httptest.NewRequest(http.MethodGet, "/v2/library/alpine/manifests/3", nil)
194	-	for _, typ := range types {
195	-	r.Header.Add("Accept", typ)
196	-	}
197	-	w := httptest.NewRecorder()
198	-	p.ServeHTTP(w, r)
199	-	if w.Code != http.StatusOK {
200	-	t.Fatal(w.Code)
201	-	}
202	-	}
203	-	mu.Lock()
204	-	for _, got := range seen {
205	-	for _, typ := range types {
206	-	if !strings.Contains(got, typ) {
207	-	t.Errorf("upstream Accept %q lacks %s", got, typ)
208	-	}
209	-	}
210	-	}
211	-	mu.Unlock()
212	-	if n := accounted(t, p.s); n != 1 {
213	-	t.Fatalf("%d tag files", n)
214	-	}
215	-	// The 8 KiB bound applies to the joined lines.
216	-	r := httptest.NewRequest(http.MethodGet, "/v2/library/alpine/manifests/3", nil)
217	-	for range 200 {
218	-	r.Header.Add("Accept", strings.Repeat("x", 50))
219	-	}
220	-	w := httptest.NewRecorder()
221	-	p.ServeHTTP(w, r)
222	-	if w.Code != http.StatusNotFound {
223	-	t.Fatalf("oversized Accept: %d", w.Code)
224	-	}
225	-	}
226			137
227	-	// MINOR: a rate-limited upstream (429) is unavailable: a known tag is served	138	+	ctx, cancel := context.WithCancel(context.Background())
228	-	// stale; an unknown one is an error.	139	+	first := httptest.NewRecorder()
229	-	func TestRateLimitedServesStaleTag(t *testing.T) {	140	+	done := make(chan struct{})
230	-	f := fake(t)	141	+	go func() {
231	-	p := proxyFor(t, f)	142	+	defer close(done)
232	-	if w := request(t, p, http.MethodGet, "/v2/library/alpine/manifests/3"); w.Code != http.StatusOK {	143	+	p.ServeHTTP(first, httptest.NewRequest(http.MethodGet, path, nil).WithContext(ctx))
233	-	t.Fatal(w.Code)	144	+	}()
234	-	}	145	+	<-half
235	-	f.status.Store(http.StatusTooManyRequests)	146	+	cancel()
236	-	if w := request(t, p, http.MethodGet, "/v2/library/alpine/manifests/3"); w.Code != http.StatusOK \|\| w.Body.String() != manifestBody {	147	+	<-done
237	-	t.Fatalf("stale on 429: %d", w.Code)	148	+	if strings.Contains(first.Body.String(), "layer") {
			149	+	t.Fatal("unverified bytes sent to the initiator")
238		}	150		}
239	-	if w := request(t, p, http.MethodGet, "/v2/library/alpine/manifests/other"); w.Code != http.StatusBadGateway {	151	+	key := kindBlob.key(sum(body)[7:])
240	-	t.Fatalf("unknown on 429: %d", w.Code)	152	+	p.s.mu.Lock()
			153	+	left := waiters(p.s, key)
			154	+	p.s.mu.Unlock()
			155	+	if left != 0 {
			156	+	t.Fatalf("waiters after the disconnect: %d", left)
241		}	157		}
242	-	}
243			158
244	-	// MINOR: eviction walks the LRU, evicts least recently used first, frees down	159	+	second := httptest.NewRecorder()
245	-	// to the low-water mark rather than the bare need, and skips pinned entries.	160	+	done = make(chan struct{})
246	-	func TestEvictionLRUAndLowWater(t *testing.T) {	161	+	go func() {
247	-	now := time.Unix(1_700_000_000, 0)	162	+	defer close(done)
248	-	const max = 2000 // low water: 5% = 100 bytes	163	+	p.ServeHTTP(second, httptest.NewRequest(http.MethodGet, path, nil))
249	-	s, err := openStore(t.TempDir(), max, func() time.Time { return now })	164	+	}()
250	-	if err != nil {	165	+	waitFor(t, p.s, "the second client to join", func() bool { return waiters(p.s, key) == 1 })
251	-	t.Fatal(err)	166	+	unblock()
252	-	}	167	+	<-done
253	-	// 40 blobs of 50 bytes fill the store. Freeing only the need (1 byte) would	168	+	if second.Code != http.StatusOK \|\| second.Body.String() != body {
254	-	// evict one blob; the low-water mark (100 bytes) evicts two.	169	+	t.Fatalf("second client: %d", second.Code)
255	-	const n = 40
256	-	blob := func(i int) string { return fmt.Sprintf("%03d", i) + strings.Repeat("x", 47) }
257	-	for i := range n {
258	-	now = now.Add(time.Second)
259	-	put(t, s, blob(i))
260	-	}
261	-	// Touch 0 and 1, and keep 2 open: the least recently used are then 3, 4...
262	-	for _, i := range []int{0, 1} {
263	-	now = now.Add(time.Second)
264	-	f, _, err := s.open("blobs", sum(blob(i)))
265	-	if err != nil {
266	-	t.Fatal(err)
267	-	}
268	-	_ = f.Close()
269	-	}
270	-	pinned, _, err := s.open("blobs", sum(blob(2)))
271	-	if err != nil {
272	-	t.Fatal(err)
273	-	}
274	-	defer func() { _ = pinned.Close() }()
275	-	s.mu.Lock()
276	-	s.lru.MoveToFront(s.entries["blobs/"+sum(blob(2))[7:]].elem) // oldest of all, but open
277	-	s.mu.Unlock()
278	-	now = now.Add(time.Second)
279	-	put(t, s, "a") // 1 byte over the cap: frees max(1, 100) bytes
280	-	present := func(i int) bool {
281	-	_, err := os.Stat(filepath.Join(s.dir, "blobs", sum(blob(i))[7:]))
282	-	return err == nil
283		}	170		}
284	-	for i := range n {	171	+	if _, err := os.Stat(filepath.Join(p.s.dir, key)); err != nil {
285	-	want := i != 3 && i != 4 // 3 and 4 are the least recently used unpinned	172	+	t.Fatalf("blob not committed: %v", err)
286	-	if present(i) != want {
287	-	t.Errorf("blob %d present=%v", i, present(i))
288	-	}
289		}	173		}
290	-	accounted(t, s)	174	+	if gets.Load() != 1 {
291	-	s.mu.Lock()	175	+	t.Fatalf("%d upstream fetches", gets.Load())
292	-	free := s.max - s.total
293	-	s.mu.Unlock()
294	-	if free < min(lowWater, int64(max)/20)-1 {
295	-	t.Fatalf("freed only to %d", free)
296		}	176		}
297		}	177		}
298			178
299	-	// MINOR: the tag index follows rewrites, and removing a manifest takes exactly	179	+	// A client waiting for a progressing download is kept alive, so the cache
300	-	// its own tags along.	180	+	// channel's progress deadline follows the download.
301	-	func TestTagIndex(t *testing.T) {	181	+	func TestWaiterKeepAlive(t *testing.T) {
302	-	s, err := openStore(t.TempDir(), 1<<20, time.Now)	182	+	body := strings.Repeat("layer", 4000)
303	-	if err != nil {	183	+	srv := upstreamWith(t, slowBlob(body, 10, 30*time.Millisecond))
304	-	t.Fatal(err)	184	+	p := proxyWith(t, srv, Config{KeepAliveInterval: 10 * time.Millisecond})
305	-	}	185	+	w := &keepAliveRecorder{ResponseRecorder: httptest.NewRecorder()}
306	-	manifests := []string{`{"schemaVersion":2,"a":1}`, `{"schemaVersion":2,"b":1}`}	186	+	p.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/v2/library/alpine/blobs/"+sum(body), nil))
307	-	for _, m := range manifests {	187	+	if w.Code != http.StatusOK \|\| w.calls.Load() == 0 {
308	-	f, _, _, err := s.ensure(context.Background(), "manifests", sum(m), func(_ context.Context, w io.Writer) (string, error) {	188	+	t.Fatalf("code %d, %d keepalives", w.Code, w.calls.Load())
309	-	_, err := io.WriteString(w, m)
310	-	return "application/vnd.oci.image.index.v1+json", err
311	-	}, keepalive{})
312	-	if err != nil {
313	-	t.Fatal(err)
314	-	}
315	-	_ = f.Close()
316	-	}
317	-	for _, tag := range []string{"x", "y"} {
318	-	if err := s.setTag("library/alpine", tag, "", sum(manifests[0])); err != nil {
319	-	t.Fatal(err)
320	-	}
321	-	}
322	-	// y moves to the second manifest.
323	-	if err := s.setTag("library/alpine", "y", "", sum(manifests[1])); err != nil {
324	-	t.Fatal(err)
325	-	}
326	-	s.mu.Lock()
327	-	if !s.removeLocked("manifests/" + sum(manifests[0])[7:]) {
328	-	t.Fatal("remove failed")
329	-	}
330	-	n0, n1 := len(s.tagsOf[sum(manifests[0])]), len(s.tagsOf[sum(manifests[1])])
331	-	s.mu.Unlock()
332	-	if s.tag("library/alpine", "x", "") != "" \|\| s.tag("library/alpine", "y", "") != sum(manifests[1]) \|\| n0 != 0 \|\| n1 != 1 {
333	-	t.Fatalf("index: %d %d", n0, n1)
334		}	189		}
335	-	accounted(t, s)
336		}	190		}
337			191
338	-	// NIT: the committed entry is pinned for its waiters, so an eviction between	192	+	// A panic in a download fails its flight and frees its slots; it does not
339	-	// the commit and their open cannot remove it.	193	+	// leave the waiters hanging.
340	-	func TestCommittedEntryPinnedForWaiters(t *testing.T) {	194	+	func TestPanickingFetchFailsFlight(t *testing.T) {
341	-	s, err := openStore(t.TempDir(), 8, time.Now)	195	+	c := storeCfg(t.TempDir(), 1<<20)
342	-	if err != nil {	196	+	c.maxDownloads = 1
343	-	t.Fatal(err)	197	+	s := mustOpen(t, c)
344	-	}	198	+	q := NewQuota(1)
345	-	s.hookCommitted = func(string) {	199	+	ctx, cancel := context.WithTimeout(WithQuota(context.Background(), q), 5*time.Second)
346	-	// Ask for the whole store before the waiters are woken.	200	+	defer cancel()
347	-	s.mu.Lock()	201	+	_, _, _, err := s.ensure(ctx, kindBlob, sum("x"), func(context.Context, io.Writer) (string, error) {
348	-	s.evictLocked(s.max)	202	+	panic("boom")
349	-	s.mu.Unlock()
350	-	}
351	-	f, _, _, err := s.ensure(context.Background(), "blobs", sum("aaaa"), func(_ context.Context, w io.Writer) (string, error) {
352	-	_, err := io.WriteString(w, "aaaa")
353	-	return "", err
354		}, keepalive{})	203		}, keepalive{})
355	-	if err != nil {	204	+	if err == nil \|\| !strings.Contains(err.Error(), "panicked") {
356	-	t.Fatalf("waiter lost its entry: %v", err)	205	+	t.Fatalf("err %v", err)
357		}	206		}
358		s.mu.Lock()	207		s.mu.Lock()
359	-	pins := s.entries["blobs/"+sum("aaaa")[7:]].pins	208	+	defer s.mu.Unlock()
360	-	s.mu.Unlock()	209	+	if s.downloads != 0 \|\| len(s.flights) != 0 \|\| q.used.Load() != 0 \|\| s.pending != 0 {
361	-	if pins != 1 {	210	+	t.Fatalf("leaked: downloads=%d flights=%d quota=%d pending=%d", s.downloads, len(s.flights), q.used.Load(), s.pending)
362	-	t.Fatalf("pins %d: the commit pin was not released to the reader's", pins)
363		}	211		}
364	-	_ = f.Close()	212	+	if ds, _ := os.ReadDir(filepath.Join(s.dir, "tmp")); len(ds) != 0 {
365	-	s.hookCommitted = nil	213	+	t.Fatal("temp file kept")
366	-	s.mu.Lock()
367	-	s.evictLocked(s.max)
368	-	_, kept := s.entries["blobs/"+sum("aaaa")[7:]]
369	-	s.mu.Unlock()
370	-	if kept {
371	-	t.Fatal("unpinned entry not evictable")
372		}	214		}
373		}	215		}
374			216
375	-	// NIT: a request that joins a manifest flight logs hit=false.	217	+	// A request that joins a manifest flight logs hit=false, with its job.
376		func TestJoinerLogsMiss(t *testing.T) {	218		func TestJoinerLogsMiss(t *testing.T) {
377		release := make(chan struct{})	219		release := make(chan struct{})
378		srv := upstreamWith(t, func(w http.ResponseWriter, r *http.Request) {	220		srv := upstreamWith(t, func(w http.ResponseWriter, r *http.Request) {

@@ -386,46 +228,21 @@ func TestJoinerLogsMiss(t *testing.T) {
386		var logs syncBuffer	228		var logs syncBuffer
387		p := proxyWith(t, srv, Config{Log: zerowrap.New(zerowrap.Config{Level: "info", Format: "json", Output: &logs})})	229		p := proxyWith(t, srv, Config{Log: zerowrap.New(zerowrap.Config{Level: "info", Format: "json", Output: &logs})})
388		var wg sync.WaitGroup	230		var wg sync.WaitGroup
389	-	for range 2 {	231	+	for i := range 2 {
390	-	wg.Go(func() { request(t, p, http.MethodGet, "/v2/library/alpine/manifests/3") })	232	+	wg.Go(func() {
			233	+	ctx := WithJob(context.Background(), fmt.Sprint("job-", i))
			234	+	p.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/v2/library/alpine/manifests/3", nil).WithContext(ctx))
			235	+	})
391		}	236		}
392	-	key := "manifests/" + sum(manifestBody)[7:]	237	+	key := kindManifest.key(sum(manifestBody)[7:])
393		waitFor(t, p.s, "two waiters", func() bool { return waiters(p.s, key) == 2 })	238		waitFor(t, p.s, "two waiters", func() bool { return waiters(p.s, key) == 2 })
394		close(release)	239		close(release)
395		wg.Wait()	240		wg.Wait()
396	-	if out := logs.String(); strings.Count(out, `"hit":false`) != 2 \|\| strings.Contains(out, `"hit":true`) {	241	+	out := logs.String()
			242	+	if strings.Count(out, `"hit":false`) != 2 \|\| strings.Contains(out, `"hit":true`) {
397		t.Fatalf("logs: %s", out)	243		t.Fatalf("logs: %s", out)
398		}	244		}
399	-	}	245	+	if !strings.Contains(out, `"job_id":"job-0"`) \|\| !strings.Contains(out, `"job_id":"job-1"`) {
400	-		246	+	t.Fatalf("no job_id on the registry lines: %s", out)
401	-	type syncBuffer struct {
402	-	mu sync.Mutex
403	-	b bytes.Buffer
404	-	}
405	-
406	-	func (s *syncBuffer) Write(p []byte) (int, error) {
407	-	s.mu.Lock()
408	-	defer s.mu.Unlock()
409	-	return s.b.Write(p)
410	-	}
411	-	func (s *syncBuffer) String() string { s.mu.Lock(); defer s.mu.Unlock(); return s.b.String() }
412	-
413	-	// NIT: a .type file without its manifest is removed at startup.
414	-	func TestOrphanTypeRemoved(t *testing.T) {
415	-	dir := t.TempDir()
416	-	if _, err := openStore(dir, 1<<20, time.Now); err != nil {
417	-	t.Fatal(err)
418	-	}
419	-	orphan := filepath.Join(dir, "manifests", sum("gone")[7:]+".type")
420	-	if err := os.WriteFile(orphan, []byte("application/json"), 0600); err != nil {
421	-	t.Fatal(err)
422	-	}
423	-	s, err := openStore(dir, 1<<20, time.Now)
424	-	if err != nil {
425	-	t.Fatal(err)
426	-	}
427	-	if _, err := os.Stat(orphan); !os.IsNotExist(err) {
428	-	t.Fatal("orphan .type kept")
429		}	247		}
430	-	accounted(t, s)
431		}	248		}

A internal/registryproxy/helpers_test.go +375 -0

@@ -0,0 +1,375 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package registryproxy
	4	+
	5	+	import (
	6	+	"bytes"
	7	+	"context"
	8	+	"crypto/sha256"
	9	+	"fmt"
	10	+	"io"
	11	+	"net/http"
	12	+	"net/http/httptest"
	13	+	"os"
	14	+	"path/filepath"
	15	+	"strings"
	16	+	"sync"
	17	+	"sync/atomic"
	18	+	"testing"
	19	+	"time"
	20	+	)
	21	+
	22	+	// TestMain fails the run if any store hit an accounting bug (unpinLocked of
	23	+	// an unpinned entry), whichever test caused it.
	24	+	func TestMain(m *testing.M) {
	25	+	var bugs atomic.Int32
	26	+	defaultBugHook = func(msg string) {
	27	+	bugs.Add(1)
	28	+	fmt.Fprintln(os.Stderr, "registry cache bug:", msg)
	29	+	}
	30	+	code := m.Run()
	31	+	if n := bugs.Load(); n > 0 && code == 0 {
	32	+	fmt.Fprintf(os.Stderr, "FAIL: %d registry cache accounting bugs\n", n)
	33	+	code = 1
	34	+	}
	35	+	os.Exit(code)
	36	+	}
	37	+
	38	+	const manifestBody = `{"schemaVersion":2}`
	39	+	const blobBody = "a layer of bytes"
	40	+
	41	+	func sum(b string) string { return fmt.Sprintf("sha256:%x", sha256.Sum256([]byte(b))) }
	42	+
	43	+	// storeCfg is a complete configuration for a test that opens a store directly.
	44	+	func storeCfg(dir string, max int64) storeConfig {
	45	+	return storeConfig{dir: dir, max: max, now: time.Now, grace: time.Second, maxDownloads: 16, slotWait: time.Second}
	46	+	}
	47	+
	48	+	func mustOpen(t testing.T, c storeConfig) store {
	49	+	t.Helper()
	50	+	s, err := openStore(c)
	51	+	if err != nil {
	52	+	t.Fatal(err)
	53	+	}
	54	+	return s
	55	+	}
	56	+
	57	+	// put stores b as a blob.
	58	+	func put(t testing.T, s store, b string) {
	59	+	t.Helper()
	60	+	f, _, _, err := s.ensure(context.Background(), kindBlob, sum(b), func(_ context.Context, w io.Writer) (string, error) { _, err := io.WriteString(w, b); return "", err }, keepalive{})
	61	+	if err != nil {
	62	+	t.Fatal(err)
	63	+	}
	64	+	_ = f.Close()
	65	+	}
	66	+
	67	+	// putManifest stores m as a manifest of media type media.
	68	+	func putManifest(t testing.T, s store, m, media string) {
	69	+	t.Helper()
	70	+	f, _, _, err := s.ensure(context.Background(), kindManifest, sum(m), func(_ context.Context, w io.Writer) (string, error) {
	71	+	_, err := io.WriteString(w, m)
	72	+	return media, err
	73	+	}, keepalive{})
	74	+	if err != nil {
	75	+	t.Fatal(err)
	76	+	}
	77	+	_ = f.Close()
	78	+	}
	79	+
	80	+	// accounted checks that the store's total is exactly the sum of its entries and
	81	+	// within Max, and returns the number of tag files on disk.
	82	+	func accounted(t testing.T, s store) int {
	83	+	t.Helper()
	84	+	s.mu.Lock()
	85	+	defer s.mu.Unlock()
	86	+	var sum int64
	87	+	tags := 0
	88	+	for k, e := range s.entries {
	89	+	sum += e.size
	90	+	if e.size < minCharge {
	91	+	t.Errorf("%s charged %d, under the minimum", k, e.size)
	92	+	}
	93	+	if e.kind == kindTag {
	94	+	tags++
	95	+	if e.size != minCharge {
	96	+	t.Errorf("tag %s charged %d", k, e.size)
	97	+	}
	98	+	}
	99	+	}
	100	+	if sum != s.total \|\| s.total+s.pending > s.max {
	101	+	t.Fatalf("accounting: sum=%d total=%d pending=%d max=%d", sum, s.total, s.pending, s.max)
	102	+	}
	103	+	files, err := os.ReadDir(filepath.Join(s.dir, kindTag.dir()))
	104	+	if err != nil {
	105	+	t.Fatal(err)
	106	+	}
	107	+	if len(files) != tags {
	108	+	t.Fatalf("%d tag files, %d tag entries", len(files), tags)
	109	+	}
	110	+	return tags
	111	+	}
	112	+
	113	+	// waitFor polls cond (under the store lock) until it holds.
	114	+	func waitFor(t testing.T, s store, what string, cond func() bool) {
	115	+	t.Helper()
	116	+	for end := time.Now().Add(5 * time.Second); ; {
	117	+	s.mu.Lock()
	118	+	ok := cond()
	119	+	s.mu.Unlock()
	120	+	if ok {
	121	+	return
	122	+	}
	123	+	if time.Now().After(end) {
	124	+	t.Fatalf("timed out waiting for %s", what)
	125	+	}
	126	+	time.Sleep(time.Millisecond)
	127	+	}
	128	+	}
	129	+
	130	+	// waiters is the number of requests waiting for the flight of key (-1: none).
	131	+	// The caller holds the store lock.
	132	+	func waiters(s *store, key string) int {
	133	+	if fl, ok := s.flights[key]; ok {
	134	+	return fl.waiters
	135	+	}
	136	+	return -1
	137	+	}
	138	+
	139	+	// upstreamWith fakes Docker Hub: /token answers an anonymous token and h serves
	140	+	// everything else.
	141	+	func upstreamWith(t testing.T, h http.HandlerFunc) httptest.Server {
	142	+	t.Helper()
	143	+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	144	+	if r.URL.Path == "/token" {
	145	+	_, _ = io.WriteString(w, `{"token":"public","expires_in":300}`)
	146	+	return
	147	+	}
	148	+	h(w, r)
	149	+	}))
	150	+	t.Cleanup(srv.Close)
	151	+	return srv
	152	+	}
	153	+
	154	+	func proxyWith(t testing.T, srv httptest.Server, c Config) *Proxy {
	155	+	t.Helper()
	156	+	if c.Dir == "" {
	157	+	c.Dir = t.TempDir()
	158	+	}
	159	+	if c.Max == 0 {
	160	+	c.Max = 8 << 20
	161	+	}
	162	+	c.Upstream, c.AuthURL = srv.URL, srv.URL+"/token"
	163	+	p, err := New(c)
	164	+	if err != nil {
	165	+	t.Fatal(err)
	166	+	}
	167	+	return p
	168	+	}
	169	+
	170	+	// fakeRegistry is a Docker Hub with one manifest and one blob, served from a
	171	+	// CDN behind a redirect. It counts the requests it gets.
	172	+	type fakeRegistry struct {
	173	+	server *httptest.Server
	174	+	heads, gets, blobs, tokens atomic.Int32
	175	+	status atomic.Int32
	176	+	corrupt atomic.Bool
	177	+	}
	178	+
	179	+	func fake(t testing.T) fakeRegistry {
	180	+	t.Helper()
	181	+	f := &fakeRegistry{}
	182	+	cdn := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	183	+	if r.Header.Get("Authorization") != "" {
	184	+	t.Error("bearer leaked to CDN")
	185	+	}
	186	+	w.Header().Set("Content-Length", fmt.Sprint(len(blobBody)))
	187	+	if r.Method != http.MethodHead {
	188	+	b := blobBody
	189	+	if f.corrupt.Load() {
	190	+	b = strings.Repeat("x", len(b))
	191	+	}
	192	+	_, _ = io.WriteString(w, b)
	193	+	}
	194	+	}))
	195	+	t.Cleanup(cdn.Close)
	196	+	f.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	197	+	if r.URL.Path == "/token" {
	198	+	f.tokens.Add(1)
	199	+	if r.Header.Get("Authorization") != "" {
	200	+	t.Error("guest credentials forwarded to auth")
	201	+	}
	202	+	if r.URL.Query().Get("scope") != "repository:library/alpine:pull" {
	203	+	t.Error("wrong scope")
	204	+	}
	205	+	_, _ = io.WriteString(w, `{"token":"public","expires_in":300}`)
	206	+	return
	207	+	}
	208	+	if r.Header.Get("Authorization") != "Bearer public" {
	209	+	t.Errorf("wrong bearer %q", r.Header.Get("Authorization"))
	210	+	}
	211	+	if s := f.status.Load(); s != 0 {
	212	+	w.WriteHeader(int(s))
	213	+	return
	214	+	}
	215	+	if strings.Contains(r.URL.Path, "/manifests/") {
	216	+	w.Header().Set("Content-Type", "application/vnd.oci.image.index.v1+json")
	217	+	w.Header().Set("Docker-Content-Digest", sum(manifestBody))
	218	+	w.Header().Set("Content-Length", fmt.Sprint(len(manifestBody)))
	219	+	if r.Method == http.MethodHead {
	220	+	f.heads.Add(1)
	221	+	} else {
	222	+	f.gets.Add(1)
	223	+	_, _ = io.WriteString(w, manifestBody)
	224	+	}
	225	+	return
	226	+	}
	227	+	if strings.Contains(r.URL.Path, "/blobs/") {
	228	+	if r.Method == http.MethodGet {
	229	+	f.blobs.Add(1)
	230	+	}
	231	+	http.Redirect(w, r, cdn.URL+"/layer", http.StatusTemporaryRedirect)
	232	+	return
	233	+	}
	234	+	http.NotFound(w, r)
	235	+	}))
	236	+	t.Cleanup(f.server.Close)
	237	+	return f
	238	+	}
	239	+
	240	+	func proxyFor(t testing.T, f fakeRegistry) *Proxy {
	241	+	t.Helper()
	242	+	p, err := New(Config{Dir: t.TempDir(), Max: 8 << 20, Upstream: f.server.URL, AuthURL: f.server.URL + "/token"})
	243	+	if err != nil {
	244	+	t.Fatal(err)
	245	+	}
	246	+	return p
	247	+	}
	248	+
	249	+	// request serves one request with guest credentials that must never reach
	250	+	// upstream; headers are name, value pairs.
	251	+	func request(t testing.T, p http.Handler, method, path string, headers ...string) httptest.ResponseRecorder {
	252	+	t.Helper()
	253	+	r := httptest.NewRequest(method, path, nil)
	254	+	r.Header.Set("Authorization", "Bearer guest-secret")
	255	+	for i := 0; i < len(headers); i += 2 {
	256	+	r.Header.Set(headers[i], headers[i+1])
	257	+	}
	258	+	w := httptest.NewRecorder()
	259	+	p.ServeHTTP(w, r)
	260	+	return w
	261	+	}
	262	+
	263	+	// stallingUpstream serves a blob's first bytes and then holds the body until
	264	+	// the request is cancelled; it counts started and cancelled downloads.
	265	+	type stallingUpstream struct {
	266	+	srv *httptest.Server
	267	+	started, cancelled atomic.Int32
	268	+	}
	269	+
	270	+	func newStallingUpstream(t testing.T) stallingUpstream {
	271	+	t.Helper()
	272	+	u := &stallingUpstream{}
	273	+	u.srv = upstreamWith(t, func(w http.ResponseWriter, r *http.Request) {
	274	+	u.started.Add(1)
	275	+	w.Header().Set("Content-Length", "1000000")
	276	+	_, _ = io.WriteString(w, "partial")
	277	+	w.(http.Flusher).Flush()
	278	+	select {
	279	+	case <-r.Context().Done():
	280	+	u.cancelled.Add(1)
	281	+	case <-time.After(10 * time.Second):
	282	+	}
	283	+	})
	284	+	return u
	285	+	}
	286	+
	287	+	func blobPath(i int) string { return "/v2/library/alpine/blobs/" + sum(fmt.Sprint("blob-", i)) }
	288	+
	289	+	// serveCancelled starts a request and cancels it once its download has begun.
	290	+	func serveCancelled(t testing.T, p Proxy, u *stallingUpstream, path string, started int32) {
	291	+	t.Helper()
	292	+	ctx, cancel := context.WithCancel(context.Background())
	293	+	done := make(chan struct{})
	294	+	go func() {
	295	+	defer close(done)
	296	+	p.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, path, nil).WithContext(ctx))
	297	+	}()
	298	+	waitFor(t, p.s, "the download to start", func() bool { return u.started.Load() >= started })
	299	+	cancel()
	300	+	<-done
	301	+	}
	302	+
	303	+	// serveAsync serves path with ctx on its own goroutine, into w (a new
	304	+	// recorder when nil); the status code comes on the returned channel.
	305	+	func serveAsync(p Proxy, ctx context.Context, path string, w keepAliveRecorder) <-chan int {
	306	+	if w == nil {
	307	+	w = &keepAliveRecorder{ResponseRecorder: httptest.NewRecorder()}
	308	+	}
	309	+	code := make(chan int, 1)
	310	+	go func() {
	311	+	p.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil).WithContext(ctx))
	312	+	code <- w.Code
	313	+	}()
	314	+	return code
	315	+	}
	316	+
	317	+	type keepAliveRecorder struct {
	318	+	*httptest.ResponseRecorder
	319	+	calls atomic.Int32
	320	+	}
	321	+
	322	+	func (k *keepAliveRecorder) KeepAlive() { k.calls.Add(1) }
	323	+
	324	+	// waitKeptAlive waits until w has been kept alive at least once.
	325	+	func waitKeptAlive(t testing.T, w keepAliveRecorder) {
	326	+	t.Helper()
	327	+	for end := time.Now().Add(5 * time.Second); w.calls.Load() == 0; time.Sleep(time.Millisecond) {
	328	+	if time.Now().After(end) {
	329	+	t.Fatal("request never kept alive")
	330	+	}
	331	+	}
	332	+	}
	333	+
	334	+	// blockingWriter holds a response in its first Write until release is closed.
	335	+	type blockingWriter struct {
	336	+	*httptest.ResponseRecorder
	337	+	started, release chan struct{}
	338	+	once sync.Once
	339	+	}
	340	+
	341	+	func (b *blockingWriter) Write(p []byte) (int, error) {
	342	+	b.once.Do(func() { close(b.started) })
	343	+	<-b.release
	344	+	return b.ResponseRecorder.Write(p)
	345	+	}
	346	+
	347	+	// slowBlob streams body in chunks, one every gap.
	348	+	func slowBlob(body string, chunks int, gap time.Duration) http.HandlerFunc {
	349	+	return func(w http.ResponseWriter, r *http.Request) {
	350	+	w.Header().Set("Content-Length", fmt.Sprint(len(body)))
	351	+	size := len(body) / chunks
	352	+	for i := 0; i < len(body); i += size {
	353	+	_, _ = io.WriteString(w, body[i:min(i+size, len(body))])
	354	+	w.(http.Flusher).Flush()
	355	+	select {
	356	+	case <-time.After(gap):
	357	+	case <-r.Context().Done():
	358	+	return
	359	+	}
	360	+	}
	361	+	}
	362	+	}
	363	+
	364	+	type syncBuffer struct {
	365	+	mu sync.Mutex
	366	+	b bytes.Buffer
	367	+	}
	368	+
	369	+	func (s *syncBuffer) Write(p []byte) (int, error) {
	370	+	s.mu.Lock()
	371	+	defer s.mu.Unlock()
	372	+	return s.b.Write(p)
	373	+	}
	374	+
	375	+	func (s *syncBuffer) String() string { s.mu.Lock(); defer s.mu.Unlock(); return s.b.String() }

M internal/registryproxy/hub_test.go +3 -1

 // SPDX-License-Identifier: MIT
 package registryproxy
 import (
 	"bytes"
 	"encoding/json"
-	"github.com/bnema/zerowrap"
 	"net/http/httptest"
 	"os"
 	"os/exec"
 	"strings"
 	"testing"
+	"github.com/bnema/zerowrap"
 )
 // An opt-in manual check using curl against a real local HTTP listener, with

D internal/registryproxy/limits_test.go +0 -494

@@ -1,494 +0,0 @@
1	-	// SPDX-License-Identifier: MIT
2	-	package registryproxy
3	-
4	-	import (
5	-	"context"
6	-	"errors"
7	-	"fmt"
8	-	"io"
9	-	"net/http"
10	-	"net/http/httptest"
11	-	"os"
12	-	"path/filepath"
13	-	"strings"
14	-	"sync"
15	-	"sync/atomic"
16	-	"testing"
17	-	"time"
18	-	)
19	-
20	-	// upstreamWith fakes Docker Hub: /token answers an anonymous token and h serves
21	-	// everything else.
22	-	func upstreamWith(t testing.T, h http.HandlerFunc) httptest.Server {
23	-	t.Helper()
24	-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
25	-	if r.URL.Path == "/token" {
26	-	_, _ = io.WriteString(w, `{"token":"public","expires_in":300}`)
27	-	return
28	-	}
29	-	h(w, r)
30	-	}))
31	-	t.Cleanup(srv.Close)
32	-	return srv
33	-	}
34	-
35	-	func proxyWith(t testing.T, srv httptest.Server, c Config) *Proxy {
36	-	t.Helper()
37	-	if c.Dir == "" {
38	-	c.Dir = t.TempDir()
39	-	}
40	-	if c.Max == 0 {
41	-	c.Max = 8 << 20
42	-	}
43	-	c.Upstream, c.AuthURL = srv.URL, srv.URL+"/token"
44	-	p, err := New(c)
45	-	if err != nil {
46	-	t.Fatal(err)
47	-	}
48	-	return p
49	-	}
50	-
51	-	// accounted checks that the store's total is exactly the sum of its entries and
52	-	// within Max, and returns the number of tag files on disk.
53	-	func accounted(t testing.T, s store) int {
54	-	t.Helper()
55	-	s.mu.Lock()
56	-	defer s.mu.Unlock()
57	-	var sum int64
58	-	tags := 0
59	-	for k, e := range s.entries {
60	-	sum += e.size
61	-	if strings.HasPrefix(k, "tags/") {
62	-	tags++
63	-	if e.size != tagCharge {
64	-	t.Errorf("tag %s charged %d", k, e.size)
65	-	}
66	-	}
67	-	}
68	-	if sum != s.total \|\| s.total+s.pending > s.max {
69	-	t.Fatalf("accounting: sum=%d total=%d pending=%d max=%d", sum, s.total, s.pending, s.max)
70	-	}
71	-	files, err := os.ReadDir(filepath.Join(s.dir, "tags"))
72	-	if err != nil {
73	-	t.Fatal(err)
74	-	}
75	-	if len(files) != tags {
76	-	t.Fatalf("%d tag files, %d tag entries", len(files), tags)
77	-	}
78	-	return tags
79	-	}
80	-
81	-	// Accept is reduced to the known manifest types, so its variants map to
82	-	// a bounded set of keys.
83	-	func TestNormalizeAccept(t *testing.T) {
84	-	const index, v2 = "application/vnd.oci.image.index.v1+json", "application/vnd.docker.distribution.manifest.v2+json"
85	-	all := normalizeAccept("")
86	-	for _, tc := range []struct{ in, want string }{
87	-	{index + ";q=0.9, " + strings.ToUpper(v2) + " , " + index, v2 + "," + index},
88	-	{" " + index + " ;q=0.1", index},
89	-	{"text/html, /", all},
90	-	{"", all},
91	-	} {
92	-	if got := normalizeAccept(tc.in); got != tc.want {
93	-	t.Errorf("normalizeAccept(%q) = %q, want %q", tc.in, got, tc.want)
94	-	}
95	-	}
96	-	if strings.Count(all, ",") != len(manifestTypes)-1 {
97	-	t.Fatalf("default key %q", all)
98	-	}
99	-	}
100	-
101	-	// A job varying Accept on one tag creates at most one tag file per
102	-	// subset of the known types, and every tag file is charged.
103	-	func TestTagFilesBoundedByAccept(t *testing.T) {
104	-	f := fake(t)
105	-	p := proxyFor(t, f)
106	-	for i := range 300 {
107	-	accept := fmt.Sprintf("application/vnd.oci.image.index.v1+json;q=0.%d, x/junk-%d", i%10, i)
108	-	switch i % 3 {
109	-	case 1:
110	-	accept = fmt.Sprintf("x/only-junk-%d", i)
111	-	case 2:
112	-	accept += ", application/vnd.docker.distribution.manifest.list.v2+json"
113	-	}
114	-	if w := request(t, p, http.MethodGet, "/v2/library/alpine/manifests/3", "Accept", accept); w.Code != http.StatusOK {
115	-	t.Fatalf("manifest: %d", w.Code)
116	-	}
117	-	}
118	-	if n := accounted(t, p.s); n == 0 \|\| n > 3 {
119	-	t.Fatalf("%d tag files for 300 Accept variants", n)
120	-	}
121	-	}
122	-
123	-	// Tag files share the LRU of the store, so they cannot grow the store
124	-	// past Max.
125	-	func TestTagsChargedAndEvicted(t *testing.T) {
126	-	now := time.Unix(1_700_000_000, 0)
127	-	const media = "application/vnd.oci.image.index.v1+json"
128	-	dir := t.TempDir()
129	-	max := int64(len(manifestBody)+len(media)) + 3*tagCharge
130	-	s, err := openStore(dir, max, func() time.Time { return now })
131	-	if err != nil {
132	-	t.Fatal(err)
133	-	}
134	-	m, _, _, err := s.ensure(context.Background(), "manifests", sum(manifestBody), func(_ context.Context, w io.Writer) (string, error) {
135	-	_, err := io.WriteString(w, manifestBody)
136	-	return media, err
137	-	}, keepalive{})
138	-	if err != nil {
139	-	t.Fatal(err)
140	-	}
141	-	_ = m.Close()
142	-	for i := range 20 {
143	-	now = now.Add(time.Second)
144	-	if err := s.setTag("library/alpine", fmt.Sprint("t", i), "", sum(manifestBody)); err != nil {
145	-	t.Fatalf("tag %d: %v", i, err)
146	-	}
147	-	if n := accounted(t, s); n > 3 {
148	-	t.Fatalf("%d tags over a cap of 3", n)
149	-	}
150	-	}
151	-	if s.tag("library/alpine", "t19", "") != sum(manifestBody) \|\| s.tag("library/alpine", "t0", "") != "" {
152	-	t.Fatal("tags not evicted least recently used first")
153	-	}
154	-	// Reopening charges the tag files again.
155	-	s, err = openStore(dir, max, time.Now)
156	-	if err != nil {
157	-	t.Fatal(err)
158	-	}
159	-	if n := accounted(t, s); n != 3 {
160	-	t.Fatalf("reopened with %d tags", n)
161	-	}
162	-	}
163	-
164	-	// An entry that is open for serving is not evicted (its bytes would
165	-	// stay allocated while uncharged); it can be evicted once closed.
166	-	func TestPinnedBlobSurvivesEviction(t *testing.T) {
167	-	dir := t.TempDir()
168	-	s, err := openStore(dir, 8, time.Now)
169	-	if err != nil {
170	-	t.Fatal(err)
171	-	}
172	-	put(t, s, "aaaa")
173	-	f, _, err := s.open("blobs", sum("aaaa"))
174	-	if err != nil {
175	-	t.Fatal(err)
176	-	}
177	-	fill := func() error {
178	-	g, _, _, err := s.ensure(context.Background(), "blobs", sum("bbbbbbbb"), func(_ context.Context, w io.Writer) (string, error) {
179	-	_, err := io.WriteString(w, "bbbbbbbb")
180	-	return "", err
181	-	}, keepalive{})
182	-	if err == nil {
183	-	_ = g.Close()
184	-	}
185	-	return err
186	-	}
187	-	if err := fill(); !errors.Is(err, errTooLarge) {
188	-	t.Fatalf("fill over a pinned entry: %v", err)
189	-	}
190	-	if _, err := os.Stat(filepath.Join(dir, "blobs", sum("aaaa")[7:])); err != nil {
191	-	t.Fatalf("pinned blob removed: %v", err)
192	-	}
193	-	accounted(t, s)
194	-	if err := f.Close(); err != nil {
195	-	t.Fatal(err)
196	-	}
197	-	if err := fill(); err != nil {
198	-	t.Fatal(err)
199	-	}
200	-	if _, err := os.Stat(filepath.Join(dir, "blobs", sum("aaaa")[7:])); !os.IsNotExist(err) {
201	-	t.Fatal("unpinned blob not evicted")
202	-	}
203	-	accounted(t, s)
204	-	}
205	-
206	-	// blockingWriter holds a response in its first Write until release is closed.
207	-	type blockingWriter struct {
208	-	*httptest.ResponseRecorder
209	-	started, release chan struct{}
210	-	once sync.Once
211	-	}
212	-
213	-	func (b *blockingWriter) Write(p []byte) (int, error) {
214	-	b.once.Do(func() { close(b.started) })
215	-	<-b.release
216	-	return b.ResponseRecorder.Write(p)
217	-	}
218	-
219	-	// A blob that a response is still sending is pinned: a fill that needs
220	-	// its room fails, and succeeds once the response has closed the file.
221	-	func TestEvictionWhileServing(t *testing.T) {
222	-	a, b := strings.Repeat("a", 1024), strings.Repeat("b", 1024)
223	-	srv := upstreamWith(t, func(w http.ResponseWriter, r *http.Request) {
224	-	switch {
225	-	case strings.HasSuffix(r.URL.Path, sum(a)):
226	-	_, _ = io.WriteString(w, a)
227	-	case strings.HasSuffix(r.URL.Path, sum(b)):
228	-	_, _ = io.WriteString(w, b)
229	-	default:
230	-	http.NotFound(w, r)
231	-	}
232	-	})
233	-	p := proxyWith(t, srv, Config{Max: 1500})
234	-	if w := request(t, p, http.MethodGet, "/v2/library/alpine/blobs/"+sum(a)); w.Code != http.StatusOK {
235	-	t.Fatal(w.Code)
236	-	}
237	-	bw := &blockingWriter{ResponseRecorder: httptest.NewRecorder(), started: make(chan struct{}), release: make(chan struct{})}
238	-	done := make(chan struct{})
239	-	go func() {
240	-	defer close(done)
241	-	p.ServeHTTP(bw, httptest.NewRequest(http.MethodGet, "/v2/library/alpine/blobs/"+sum(a), nil))
242	-	}()
243	-	<-bw.started
244	-	if w := request(t, p, http.MethodGet, "/v2/library/alpine/blobs/"+sum(b)); w.Code != http.StatusBadGateway {
245	-	t.Fatalf("fill evicted a blob being served: %d", w.Code)
246	-	}
247	-	if _, err := os.Stat(filepath.Join(p.s.dir, "blobs", sum(a)[7:])); err != nil {
248	-	t.Fatalf("served blob removed: %v", err)
249	-	}
250	-	accounted(t, p.s)
251	-	close(bw.release)
252	-	<-done
253	-	if bw.Code != http.StatusOK \|\| bw.Body.String() != a {
254	-	t.Fatalf("served: %d", bw.Code)
255	-	}
256	-	if w := request(t, p, http.MethodGet, "/v2/library/alpine/blobs/"+sum(b)); w.Code != http.StatusOK \|\| w.Body.String() != b {
257	-	t.Fatalf("after release: %d", w.Code)
258	-	}
259	-	if _, err := os.Stat(filepath.Join(p.s.dir, "blobs", sum(a)[7:])); !os.IsNotExist(err) {
260	-	t.Fatal("released blob not evicted")
261	-	}
262	-	accounted(t, p.s)
263	-	}
264	-
265	-	// slowBlob streams body in chunks, one every gap.
266	-	func slowBlob(body string, chunks int, gap time.Duration) http.HandlerFunc {
267	-	return func(w http.ResponseWriter, r *http.Request) {
268	-	w.Header().Set("Content-Length", fmt.Sprint(len(body)))
269	-	size := len(body) / chunks
270	-	for i := 0; i < len(body); i += size {
271	-	_, _ = io.WriteString(w, body[i:min(i+size, len(body))])
272	-	w.(http.Flusher).Flush()
273	-	select {
274	-	case <-time.After(gap):
275	-	case <-r.Context().Done():
276	-	return
277	-	}
278	-	}
279	-	}
280	-	}
281	-
282	-	// A blob body has no total deadline: a transfer that keeps moving may
283	-	// last far longer than RequestTimeout and BlobIdleTimeout.
284	-	func TestBlobTransferOutlivesRequestTimeout(t *testing.T) {
285	-	body := strings.Repeat("layer", 4000)
286	-	srv := upstreamWith(t, slowBlob(body, 20, 25*time.Millisecond))
287	-	p := proxyWith(t, srv, Config{RequestTimeout: 100 * time.Millisecond, BlobIdleTimeout: 150 * time.Millisecond})
288	-	start := time.Now()
289	-	w := request(t, p, http.MethodGet, "/v2/library/alpine/blobs/"+sum(body))
290	-	if w.Code != http.StatusOK \|\| w.Body.String() != body {
291	-	t.Fatalf("blob: %d", w.Code)
292	-	}
293	-	if d := time.Since(start); d < 300*time.Millisecond {
294	-	t.Fatalf("transfer took %v, not longer than the timeouts", d)
295	-	}
296	-	}
297	-
298	-	// A blob body that stops moving is cut after BlobIdleTimeout, and auth,
299	-	// manifests and HEAD keep their total RequestTimeout.
300	-	func TestStalledUpstreamTimesOut(t *testing.T) {
301	-	body := strings.Repeat("layer", 4000)
302	-	srv := upstreamWith(t, func(w http.ResponseWriter, r *http.Request) {
303	-	if strings.Contains(r.URL.Path, "/blobs/") && r.Method == http.MethodGet {
304	-	w.Header().Set("Content-Length", fmt.Sprint(len(body)))
305	-	_, _ = io.WriteString(w, body[:100])
306	-	w.(http.Flusher).Flush()
307	-	}
308	-	select { // a stalled body, or headers that never come
309	-	case <-r.Context().Done():
310	-	case <-time.After(5 * time.Second):
311	-	}
312	-	})
313	-	p := proxyWith(t, srv, Config{RequestTimeout: 100 * time.Millisecond, BlobIdleTimeout: 150 * time.Millisecond})
314	-	for _, tc := range []struct{ method, path string }{
315	-	{http.MethodGet, "/v2/library/alpine/blobs/" + sum(body)},
316	-	{http.MethodHead, "/v2/library/alpine/blobs/" + sum(body)},
317	-	{http.MethodGet, "/v2/library/alpine/manifests/3"},
318	-	} {
319	-	start := time.Now()
320	-	if w := request(t, p, tc.method, tc.path); w.Code != http.StatusBadGateway {
321	-	t.Errorf("%s %s: %d", tc.method, tc.path, w.Code)
322	-	}
323	-	if d := time.Since(start); d > 2*time.Second {
324	-	t.Errorf("%s %s: cut after %v", tc.method, tc.path, d)
325	-	}
326	-	}
327	-	if _, err := os.Stat(filepath.Join(p.s.dir, "blobs", sum(body)[7:])); !os.IsNotExist(err) {
328	-	t.Fatal("partial blob stored")
329	-	}
330	-	}
331	-
332	-	// waitFor polls cond (under the store lock) until it holds.
333	-	func waitFor(t testing.T, s store, what string, cond func() bool) {
334	-	t.Helper()
335	-	for end := time.Now().Add(5 * time.Second); ; {
336	-	s.mu.Lock()
337	-	ok := cond()
338	-	s.mu.Unlock()
339	-	if ok {
340	-	return
341	-	}
342	-	if time.Now().After(end) {
343	-	t.Fatalf("timed out waiting for %s", what)
344	-	}
345	-	time.Sleep(time.Millisecond)
346	-	}
347	-	}
348	-
349	-	// waiters is the number of requests waiting for the flight of key (-1: none).
350	-	func waiters(s *store, key string) int {
351	-	if fl, ok := s.flights[key]; ok {
352	-	return fl.waiters
353	-	}
354	-	return -1
355	-	}
356	-
357	-	// The shared download does not belong to the client that started it.
358	-	// That client disconnects mid-download; a second client, joining within the
359	-	// grace period, still gets the blob and the blob is committed, from one
360	-	// upstream fetch.
361	-	func TestBlobFlightSurvivesInitiatorDisconnect(t *testing.T) {
362	-	body := strings.Repeat("layer", 4000)
363	-	half, release := make(chan struct{}), make(chan struct{})
364	-	var releaseOnce sync.Once
365	-	unblock := func() { releaseOnce.Do(func() { close(release) }) }
366	-	var gets atomic.Int32
367	-	srv := upstreamWith(t, func(w http.ResponseWriter, r *http.Request) {
368	-	gets.Add(1)
369	-	w.Header().Set("Content-Length", fmt.Sprint(len(body)))
370	-	_, _ = io.WriteString(w, body[:len(body)/2])
371	-	w.(http.Flusher).Flush()
372	-	close(half)
373	-	<-release
374	-	_, _ = io.WriteString(w, body[len(body)/2:])
375	-	})
376	-	t.Cleanup(unblock) // before srv.Close, which waits for the handler
377	-	p := proxyWith(t, srv, Config{})
378	-	path := "/v2/library/alpine/blobs/" + sum(body)
379	-
380	-	ctx, cancel := context.WithCancel(context.Background())
381	-	first := httptest.NewRecorder()
382	-	done := make(chan struct{})
383	-	go func() {
384	-	defer close(done)
385	-	p.ServeHTTP(first, httptest.NewRequest(http.MethodGet, path, nil).WithContext(ctx))
386	-	}()
387	-	<-half
388	-	cancel()
389	-	<-done
390	-	if strings.Contains(first.Body.String(), "layer") {
391	-	t.Fatal("unverified bytes sent to the initiator")
392	-	}
393	-	key := "blobs/" + sum(body)[7:]
394	-	p.s.mu.Lock()
395	-	left := waiters(p.s, key)
396	-	p.s.mu.Unlock()
397	-	if left != 0 {
398	-	t.Fatalf("waiters after the disconnect: %d", left)
399	-	}
400	-
401	-	second := httptest.NewRecorder()
402	-	done = make(chan struct{})
403	-	go func() {
404	-	defer close(done)
405	-	p.ServeHTTP(second, httptest.NewRequest(http.MethodGet, path, nil))
406	-	}()
407	-	waitFor(t, p.s, "the second client to join", func() bool { return waiters(p.s, key) == 1 })
408	-	unblock()
409	-	<-done
410	-	if second.Code != http.StatusOK \|\| second.Body.String() != body {
411	-	t.Fatalf("second client: %d", second.Code)
412	-	}
413	-	if _, err := os.Stat(filepath.Join(p.s.dir, "blobs", sum(body)[7:])); err != nil {
414	-	t.Fatalf("blob not committed: %v", err)
415	-	}
416	-	if gets.Load() != 1 {
417	-	t.Fatalf("%d upstream fetches", gets.Load())
418	-	}
419	-	}
420	-
421	-	type keepAliveRecorder struct {
422	-	*httptest.ResponseRecorder
423	-	calls atomic.Int32
424	-	}
425	-
426	-	func (k *keepAliveRecorder) KeepAlive() { k.calls.Add(1) }
427	-
428	-	// A client waiting for a progressing download is kept alive, so
429	-	// the cache channel's progress deadline follows the download.
430	-	func TestWaiterKeepAlive(t *testing.T) {
431	-	body := strings.Repeat("layer", 4000)
432	-	srv := upstreamWith(t, slowBlob(body, 10, 30*time.Millisecond))
433	-	p := proxyWith(t, srv, Config{KeepAliveInterval: 10 * time.Millisecond})
434	-	w := &keepAliveRecorder{ResponseRecorder: httptest.NewRecorder()}
435	-	p.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/v2/library/alpine/blobs/"+sum(body), nil))
436	-	if w.Code != http.StatusOK \|\| w.calls.Load() == 0 {
437	-	t.Fatalf("code %d, %d keepalives", w.Code, w.calls.Load())
438	-	}
439	-	}
440	-
441	-	// Auth and manifest requests never follow a redirect.
442	-	func TestMetadataRedirectsNotFollowed(t *testing.T) {
443	-	var elsewhere atomic.Int32
444	-	other := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
445	-	elsewhere.Add(1)
446	-	_, _ = io.WriteString(w, `{"token":"public"}`)
447	-	}))
448	-	t.Cleanup(other.Close)
449	-	for _, redirectAuth := range []bool{true, false} {
450	-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
451	-	if r.URL.Path == "/token" && !redirectAuth {
452	-	_, _ = io.WriteString(w, `{"token":"public"}`)
453	-	return
454	-	}
455	-	http.Redirect(w, r, other.URL+r.URL.Path, http.StatusTemporaryRedirect)
456	-	}))
457	-	p := proxyWith(t, srv, Config{})
458	-	for _, ref := range []string{"3", sum(manifestBody)} {
459	-	if w := request(t, p, http.MethodGet, "/v2/library/alpine/manifests/"+ref); w.Code != http.StatusBadGateway {
460	-	t.Errorf("auth redirect %v, ref %s: %d", redirectAuth, ref, w.Code)
461	-	}
462	-	}
463	-	srv.Close()
464	-	}
465	-	if elsewhere.Load() != 0 {
466	-	t.Fatalf("%d redirects followed", elsewhere.Load())
467	-	}
468	-	}
469	-
470	-	// Blob requests follow the CDN redirect, without the bearer token.
471	-	func TestBlobRedirectStripsBearer(t *testing.T) {
472	-	var cdnHits atomic.Int32
473	-	cdn := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
474	-	cdnHits.Add(1)
475	-	if r.Header.Get("Authorization") != "" {
476	-	t.Error("bearer sent to the CDN")
477	-	}
478	-	_, _ = io.WriteString(w, blobBody)
479	-	}))
480	-	t.Cleanup(cdn.Close)
481	-	srv := upstreamWith(t, func(w http.ResponseWriter, r *http.Request) {
482	-	if r.Header.Get("Authorization") != "Bearer public" {
483	-	t.Error("no bearer on the registry")
484	-	}
485	-	http.Redirect(w, r, cdn.URL+"/layer", http.StatusTemporaryRedirect)
486	-	})
487	-	p := proxyWith(t, srv, Config{})
488	-	if w := request(t, p, http.MethodGet, "/v2/library/alpine/blobs/"+sum(blobBody)); w.Code != http.StatusOK \|\| w.Body.String() != blobBody {
489	-	t.Fatalf("blob: %d", w.Code)
490	-	}
491	-	if cdnHits.Load() != 1 {
492	-	t.Fatalf("CDN hits %d", cdnHits.Load())
493	-	}
494	-	}

D internal/registryproxy/main_test.go +0 -25

@@ -1,25 +0,0 @@
1	-	// SPDX-License-Identifier: MIT
2	-	package registryproxy
3	-
4	-	import (
5	-	"fmt"
6	-	"os"
7	-	"sync/atomic"
8	-	"testing"
9	-	)
10	-
11	-	// TestMain fails the run if any store hit an accounting bug (unpinLocked of
12	-	// an unpinned entry), whichever test caused it.
13	-	func TestMain(m *testing.M) {
14	-	var bugs atomic.Int32
15	-	defaultBugHook = func(msg string) {
16	-	bugs.Add(1)
17	-	fmt.Fprintln(os.Stderr, "registry cache bug:", msg)
18	-	}
19	-	code := m.Run()
20	-	if n := bugs.Load(); n > 0 && code == 0 {
21	-	fmt.Fprintf(os.Stderr, "FAIL: %d registry cache accounting bugs\n", n)
22	-	code = 1
23	-	}
24	-	os.Exit(code)
25	-	}

M internal/registryproxy/proxy.go +63 -44


@@ -28,8 +28,6 @@ type Config struct {
28		Max int64	28		Max int64
29		Upstream string // default https://registry-1.docker.io	29		Upstream string // default https://registry-1.docker.io
30		AuthURL string // default https://auth.docker.io/token	30		AuthURL string // default https://auth.docker.io/token
31	-	Client *http.Client
32	-	Now func() time.Time
33		Log zerowrap.Logger	31		Log zerowrap.Logger
34		// RequestTimeout bounds auth, manifests and HEAD (default 10s).	32		// RequestTimeout bounds auth, manifests and HEAD (default 10s).
35		RequestTimeout time.Duration	33		RequestTimeout time.Duration

@@ -38,7 +36,7 @@ type Config struct {
38		BlobIdleTimeout time.Duration	36		BlobIdleTimeout time.Duration
39		// KeepAliveInterval is how often a client waiting for a cold blob is kept	37		// KeepAliveInterval is how often a client waiting for a cold blob is kept
40		// alive while the download progresses (default 1s), when its	38		// alive while the download progresses (default 1s), when its
41	-	// ResponseWriter has a KeepAlive method.	39	+	// ResponseWriter is a KeepAliver.
42		KeepAliveInterval time.Duration	40		KeepAliveInterval time.Duration
43		// AbandonGrace is how long a download nobody waits for any more keeps	41		// AbandonGrace is how long a download nobody waits for any more keeps
44		// running in case another request joins it; then it is cancelled and its	42		// running in case another request joins it; then it is cancelled and its

@@ -52,6 +50,15 @@ type Config struct {
52		SlotWait time.Duration	50		SlotWait time.Duration
53		}	51		}
54			52
			53	+	// KeepAliver is a ResponseWriter that can move its connection's deadlines
			54	+	// without writing. While a request waits for a cold blob, the Proxy calls
			55	+	// KeepAlive every KeepAliveInterval as long as the download progresses, so
			56	+	// that a slow but moving download does not time the client out before the
			57	+	// first byte of the reply.
			58	+	type KeepAliver interface {
			59	+	KeepAlive()
			60	+	}
			61	+
55		// Proxy implements http.Handler for GET and HEAD under /v2/.	62		// Proxy implements http.Handler for GET and HEAD under /v2/.
56		type Proxy struct {	63		type Proxy struct {
57		s *store	64		s *store

@@ -62,9 +69,6 @@ type Proxy struct {
62			69
63		// New opens the persistent cache. It starts no goroutines or listeners.	70		// New opens the persistent cache. It starts no goroutines or listeners.
64		func New(c Config) (*Proxy, error) {	71		func New(c Config) (*Proxy, error) {
65	-	if c.Now == nil {
66	-	c.Now = time.Now
67	-	}
68		if c.Upstream == "" {	72		if c.Upstream == "" {
69		c.Upstream = "https://registry-1.docker.io"	73		c.Upstream = "https://registry-1.docker.io"
70		}	74		}

@@ -95,14 +99,14 @@ func New(c Config) (*Proxy, error) {
95		if c.SlotWait <= 0 {	99		if c.SlotWait <= 0 {
96		c.SlotWait = 30 * time.Second	100		c.SlotWait = 30 * time.Second
97		}	101		}
98	-	client, blobs := upstreamClients(c)	102	+	st, err := openStore(storeConfig{dir: c.Dir, max: c.Max, now: time.Now, grace: c.AbandonGrace,
99	-		103	+	maxDownloads: c.MaxDownloads, slotWait: c.SlotWait, log: c.Log})
100	-	st, err := openStore(c.Dir, c.Max, c.Now)
101		if err != nil {	104		if err != nil {
102		return nil, err	105		return nil, err
103		}	106		}
104	-	st.grace, st.maxDownloads, st.slotWait, st.log = c.AbandonGrace, c.MaxDownloads, c.SlotWait, c.Log	107	+	client, blobs := upstreamClients(c.RequestTimeout)
105	-	return &Proxy{s: st, u: &upstream{base: strings.TrimRight(c.Upstream, "/"), auth: c.AuthURL, client: client, blobClient: blobs, requestTimeout: c.RequestTimeout, blobIdle: c.BlobIdleTimeout, now: c.Now, tokens: map[string]bearer{}}, log: c.Log, keepEvery: c.KeepAliveInterval}, nil	108	+	return &Proxy{s: st, u: &upstream{base: strings.TrimRight(c.Upstream, "/"), auth: c.AuthURL, client: client, blobClient: blobs,
			109	+	requestTimeout: c.RequestTimeout, blobIdle: c.BlobIdleTimeout, tokens: map[string]bearer{}}, log: c.Log, keepEvery: c.KeepAliveInterval}, nil
106		}	110		}
107			111
108		var namePattern = regexp.MustCompile(`^[a-z0-9]+(?:[._-][a-z0-9]+)(?:/[a-z0-9]+(?:[._-][a-z0-9]+))*$`)	112		var namePattern = regexp.MustCompile(`^[a-z0-9]+(?:[._-][a-z0-9]+)(?:/[a-z0-9]+(?:[._-][a-z0-9]+))*$`)

@@ -147,23 +151,32 @@ func (p Proxy) ServeHTTP(w http.ResponseWriter, r http.Request) {
147		http.NotFound(w, r)	151		http.NotFound(w, r)
148		return	152		return
149		}	153		}
150	-	name, kind := path[:i], path[i+1:]	154	+	name, route := path[:i], path[i+1:]
151		// dockerd's legacy client sends one Accept line per media type.	155		// dockerd's legacy client sends one Accept line per media type.
152		accept := strings.Join(r.Header.Values("Accept"), ",")	156		accept := strings.Join(r.Header.Values("Accept"), ",")
153		if len(name) > 255 \|\| !namePattern.MatchString(name) \|\| len(accept) > 8192 {	157		if len(name) > 255 \|\| !namePattern.MatchString(name) \|\| len(accept) > 8192 {
154		http.NotFound(w, r)	158		http.NotFound(w, r)
155		return	159		return
156		}	160		}
157	-	if kind == "manifests" && (validDigest(ref) \|\| tagPattern.MatchString(ref)) {	161	+	if route == kindManifest.dir() && (validDigest(ref) \|\| tagPattern.MatchString(ref)) {
158		p.manifest(w, r, name, ref, normalizeAccept(accept))	162		p.manifest(w, r, name, ref, normalizeAccept(accept))
159		return	163		return
160		}	164		}
161	-	if kind == "blobs" && validDigest(ref) {	165	+	if route == kindBlob.dir() && validDigest(ref) {
162		p.blob(w, r, name, ref)	166		p.blob(w, r, name, ref)
163		return	167		return
164		}	168		}
165		http.NotFound(w, r)	169		http.NotFound(w, r)
166		}	170		}
			171	+
			172	+	// logFor is the Proxy's logger, with the job_id of the request (WithJob).
			173	+	func (p *Proxy) logFor(ctx context.Context) zerowrap.Logger {
			174	+	if job := jobFrom(ctx); job != "" {
			175	+	return zerowrap.Logger{Logger: p.log.With().Str("job_id", job).Logger()}
			176	+	}
			177	+	return p.log
			178	+	}
			179	+
167		func serveFile(w http.ResponseWriter, r http.Request, f storedFile, media, digest string) {	180		func serveFile(w http.ResponseWriter, r http.Request, f storedFile, media, digest string) {
168		defer func() { _ = f.Close() }()	181		defer func() { _ = f.Close() }()
169		fi, err := f.Stat()	182		fi, err := f.Stat()

@@ -176,29 +189,39 @@ func serveFile(w http.ResponseWriter, r http.Request, f storedFile, media, dig
176		http.ServeContent(w, r, "", fi.ModTime(), f)	189		http.ServeContent(w, r, "", fi.ModTime(), f)
177		}	190		}
178			191
			192	+	// serveStaleTag serves the manifest a tag pointed to when upstream could not
			193	+	// answer, if that manifest is still stored. It reports whether it served one.
			194	+	func (p Proxy) serveStaleTag(w http.ResponseWriter, r http.Request, name, tag, accept string) bool {
			195	+	digest := p.s.tag(name, tag, accept)
			196	+	if !validDigest(digest) {
			197	+	return false
			198	+	}
			199	+	f, media, err := p.s.open(kindManifest, digest)
			200	+	if err != nil {
			201	+	return false
			202	+	}
			203	+	serveFile(w, r, f, media, digest)
			204	+	return true
			205	+	}
			206	+
179		// manifest serves a manifest by tag or digest. accept is normalized.	207		// manifest serves a manifest by tag or digest. accept is normalized.
180		func (p Proxy) manifest(w http.ResponseWriter, r http.Request, name, ref, accept string) {	208		func (p Proxy) manifest(w http.ResponseWriter, r http.Request, name, ref, accept string) {
181		start := time.Now()	209		start := time.Now()
182		hit := false	210		hit := false
			211	+	log := p.logFor(r.Context())
183		defer func() {	212		defer func() {
184	-	p.log.Info().Str("repo", name).Str("ref", ref).Bool("hit", hit).Int64("ms", time.Since(start).Milliseconds()).Msg("registry manifest")	213	+	log.Info().Str("repo", name).Str("ref", ref).Bool("hit", hit).Int64("ms", time.Since(start).Milliseconds()).Msg("registry manifest")
185		}()	214		}()
186		ctx, cancel := context.WithTimeout(r.Context(), p.u.requestTimeout)	215		ctx, cancel := context.WithTimeout(r.Context(), p.u.requestTimeout)
187		defer cancel()	216		defer cancel()
188		digest := ref	217		digest := ref
189		tag := !validDigest(ref)	218		tag := !validDigest(ref)
190		if tag {	219		if tag {
191	-	resp, err := p.u.request(ctx, http.MethodHead, name, "manifests", ref, accept)	220	+	resp, err := p.u.request(ctx, http.MethodHead, name, kindManifest, ref, accept)
192		if err != nil {	221		if err != nil {
193	-	if unavailable(err) {	222	+	if unavailable(err) && p.serveStaleTag(w, r, name, ref, accept) {
194	-	digest = p.s.tag(name, ref, accept)	223	+	hit = true
195	-	if validDigest(digest) {	224	+	return
196	-	if f, media, e := p.s.open("manifests", digest); e == nil {
197	-	hit = true
198	-	serveFile(w, r, f, media, digest)
199	-	return
200	-	}
201	-	}
202		}	225		}
203		http.Error(w, "registry upstream unavailable", proxyStatus(err))	226		http.Error(w, "registry upstream unavailable", proxyStatus(err))
204		return	227		return

@@ -211,10 +234,10 @@ func (p Proxy) manifest(w http.ResponseWriter, r http.Request, name, ref, acce
211		}	234		}
212		}	235		}
213		// Shared flight, as for blobs: it must not end with the initiator's request.	236		// Shared flight, as for blobs: it must not end with the initiator's request.
214	-	f, media, cached, err := p.s.ensure(ctx, "manifests", digest, func(flightCtx context.Context, dst io.Writer) (string, error) {	237	+	f, media, cached, err := p.s.ensure(ctx, kindManifest, digest, func(flightCtx context.Context, dst io.Writer) (string, error) {
215		fillCtx, fillCancel := context.WithTimeout(flightCtx, p.u.requestTimeout)	238		fillCtx, fillCancel := context.WithTimeout(flightCtx, p.u.requestTimeout)
216		defer fillCancel()	239		defer fillCancel()
217	-	resp, err := p.u.request(fillCtx, http.MethodGet, name, "manifests", digest, accept)	240	+	resp, err := p.u.request(fillCtx, http.MethodGet, name, kindManifest, digest, accept)
218		if err != nil {	241		if err != nil {
219		return "", err	242		return "", err
220		}	243		}

@@ -231,15 +254,9 @@ func (p Proxy) manifest(w http.ResponseWriter, r http.Request, name, ref, acce
231		}, keepalive{})	254		}, keepalive{})
232		hit = cached	255		hit = cached
233		if err != nil {	256		if err != nil {
234	-	if tag && unavailable(err) {	257	+	if tag && unavailable(err) && p.serveStaleTag(w, r, name, ref, accept) {
235	-	previous := p.s.tag(name, ref, accept)	258	+	hit = true
236	-	if validDigest(previous) {	259	+	return
237	-	if old, oldMedia, e := p.s.open("manifests", previous); e == nil {
238	-	hit = true
239	-	serveFile(w, r, old, oldMedia, previous)
240	-	return
241	-	}
242	-	}
243		}	260		}
244		http.Error(w, "registry fetch failed", proxyStatus(err))	261		http.Error(w, "registry fetch failed", proxyStatus(err))
245		return	262		return

@@ -249,8 +266,9 @@ func (p Proxy) manifest(w http.ResponseWriter, r http.Request, name, ref, acce
249		}	266		}
250		serveFile(w, r, f, media, digest)	267		serveFile(w, r, f, media, digest)
251		}	268		}
			269	+
252		func (p Proxy) blob(w http.ResponseWriter, r http.Request, name, digest string) {	270		func (p Proxy) blob(w http.ResponseWriter, r http.Request, name, digest string) {
253	-	if f, media, err := p.s.open("blobs", digest); err == nil {	271	+	if f, media, err := p.s.open(kindBlob, digest); err == nil {
254		serveFile(w, r, f, media, digest)	272		serveFile(w, r, f, media, digest)
255		return	273		return
256		}	274		}

@@ -258,7 +276,7 @@ func (p Proxy) blob(w http.ResponseWriter, r http.Request, name, digest string
258		if r.Method == http.MethodHead {	276		if r.Method == http.MethodHead {
259		ctx, cancel := context.WithTimeout(r.Context(), p.u.requestTimeout)	277		ctx, cancel := context.WithTimeout(r.Context(), p.u.requestTimeout)
260		defer cancel()	278		defer cancel()
261	-	resp, err := p.u.request(ctx, http.MethodHead, name, "blobs", digest, "")	279	+	resp, err := p.u.request(ctx, http.MethodHead, name, kindBlob, digest, "")
262		if err != nil {	280		if err != nil {
263		http.Error(w, "registry fetch failed", proxyStatus(err))	281		http.Error(w, "registry fetch failed", proxyStatus(err))
264		return	282		return

@@ -282,18 +300,19 @@ func (p Proxy) blob(w http.ResponseWriter, r http.Request, name, digest string
282		// when no client has waited for it for AbandonGrace. At most MaxDownloads	300		// when no client has waited for it for AbandonGrace. At most MaxDownloads
283		// run at once, and at most its Quota per client: a request over either	301		// run at once, and at most its Quota per client: a request over either
284		// waits up to SlotWait for a slot, then gets 503, which dockerd retries	302		// waits up to SlotWait for a slot, then gets 503, which dockerd retries
285	-	// (about 50 s) before pulling from Docker Hub. While it waits, a writer	303	+	// (about 50 s) before pulling from Docker Hub. While it waits, a
286	-	// with KeepAlive (the cache channel's progress deadline) is kept alive as	304	+	// KeepAliver writer (the cache channel's progress deadline) is kept alive
287	-	// long as the download moves.	305	+	// as long as the download moves.
288		keep := keepalive{every: p.keepEvery}	306		keep := keepalive{every: p.keepEvery}
289	-	if k, ok := w.(interface{ KeepAlive() }); ok {	307	+	if k, ok := w.(KeepAliver); ok {
290		keep.fn = k.KeepAlive	308		keep.fn = k.KeepAlive
291		}	309		}
292	-	f, media, _, err := p.s.ensure(r.Context(), "blobs", digest, func(downloadCtx context.Context, dst io.Writer) (string, error) {	310	+	log := p.logFor(r.Context())
			311	+	f, media, _, err := p.s.ensure(r.Context(), kindBlob, digest, func(downloadCtx context.Context, dst io.Writer) (string, error) {
293		start := time.Now()	312		start := time.Now()
294		var size int64	313		var size int64
295		defer func() {	314		defer func() {
296	-	p.log.Debug().Str("digest", digest).Int64("bytes", size).Int64("ms", time.Since(start).Milliseconds()).Msg("registry blob")	315	+	log.Debug().Str("digest", digest).Int64("bytes", size).Int64("ms", time.Since(start).Milliseconds()).Msg("registry blob")
297		}()	316		}()
298		resp, err := p.u.blob(downloadCtx, name, digest)	317		resp, err := p.u.blob(downloadCtx, name, digest)
299		if err != nil {	318		if err != nil {

M internal/registryproxy/proxy_test.go +21 -118

@@ -1,4 +1,5 @@
1		// SPDX-License-Identifier: MIT	1		// SPDX-License-Identifier: MIT
			2	+
2		package registryproxy	3		package registryproxy
3			4
4		import (	5		import (

@@ -9,100 +10,12 @@ import (
9		"os"	10		"os"
10		"path/filepath"	11		"path/filepath"
11		"strings"	12		"strings"
12	-	"sync/atomic"
13		"testing"	13		"testing"
14		)	14		)
15			15
16	-	const manifestBody = `{"schemaVersion":2}`	16	+	// A pulled manifest and blob are served from the store the second time: a tag
17	-	const blobBody = "a layer of bytes"	17	+	// is revalidated by HEAD, a digest is not fetched again.
18	-		18	+	func TestPullCached(t *testing.T) {
19	-	type fakeRegistry struct {
20	-	server *httptest.Server
21	-	heads, gets, blobs, tokens atomic.Int32
22	-	status atomic.Int32
23	-	corrupt atomic.Bool
24	-	}
25	-
26	-	func fake(t testing.T) fakeRegistry {
27	-	t.Helper()
28	-	f := &fakeRegistry{}
29	-	cdn := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
30	-	if r.Header.Get("Authorization") != "" {
31	-	t.Error("bearer leaked to CDN")
32	-	}
33	-	w.Header().Set("Content-Length", fmt.Sprint(len(blobBody)))
34	-	if r.Method != http.MethodHead {
35	-	b := blobBody
36	-	if f.corrupt.Load() {
37	-	b = strings.Repeat("x", len(b))
38	-	}
39	-	_, _ = io.WriteString(w, b)
40	-	}
41	-	}))
42	-	t.Cleanup(cdn.Close)
43	-	f.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
44	-	if r.URL.Path == "/token" {
45	-	f.tokens.Add(1)
46	-	if r.Header.Get("Authorization") != "" {
47	-	t.Error("guest credentials forwarded to auth")
48	-	}
49	-	if r.URL.Query().Get("scope") != "repository:library/alpine:pull" {
50	-	t.Error("wrong scope")
51	-	}
52	-	_, _ = io.WriteString(w, `{"token":"public","expires_in":300}`)
53	-	return
54	-	}
55	-	if r.Header.Get("Authorization") != "Bearer public" {
56	-	t.Errorf("wrong bearer %q", r.Header.Get("Authorization"))
57	-	}
58	-	if s := f.status.Load(); s != 0 {
59	-	w.WriteHeader(int(s))
60	-	return
61	-	}
62	-	if strings.Contains(r.URL.Path, "/manifests/") {
63	-	w.Header().Set("Content-Type", "application/vnd.oci.image.index.v1+json")
64	-	w.Header().Set("Docker-Content-Digest", sum(manifestBody))
65	-	w.Header().Set("Content-Length", fmt.Sprint(len(manifestBody)))
66	-	if r.Method == http.MethodHead {
67	-	f.heads.Add(1)
68	-	} else {
69	-	f.gets.Add(1)
70	-	_, _ = io.WriteString(w, manifestBody)
71	-	}
72	-	return
73	-	}
74	-	if strings.Contains(r.URL.Path, "/blobs/") {
75	-	if r.Method == http.MethodGet {
76	-	f.blobs.Add(1)
77	-	}
78	-	http.Redirect(w, r, cdn.URL+"/layer", http.StatusTemporaryRedirect)
79	-	return
80	-	}
81	-	http.NotFound(w, r)
82	-	}))
83	-	t.Cleanup(f.server.Close)
84	-	return f
85	-	}
86	-	func proxyFor(t testing.T, f fakeRegistry) *Proxy {
87	-	t.Helper()
88	-	p, err := New(Config{Dir: t.TempDir(), Max: 8 << 20, Upstream: f.server.URL, AuthURL: f.server.URL + "/token"})
89	-	if err != nil {
90	-	t.Fatal(err)
91	-	}
92	-	return p
93	-	}
94	-	func request(t testing.T, p http.Handler, method, path string, headers ...string) httptest.ResponseRecorder {
95	-	t.Helper()
96	-	r := httptest.NewRequest(method, path, nil)
97	-	r.Header.Set("Authorization", "Bearer guest-secret")
98	-	for i := 0; i < len(headers); i += 2 {
99	-	r.Header.Set(headers[i], headers[i+1])
100	-	}
101	-	w := httptest.NewRecorder()
102	-	p.ServeHTTP(w, r)
103	-	return w
104	-	}
105	-	func TestPullAndOfflineTag(t *testing.T) {
106		f := fake(t)	19		f := fake(t)
107		p := proxyFor(t, f)	20		p := proxyFor(t, f)
108		for range 2 {	21		for range 2 {

@@ -118,19 +31,10 @@ func TestPullAndOfflineTag(t *testing.T) {
118		if f.heads.Load() != 2 \|\| f.gets.Load() != 1 \|\| f.blobs.Load() != 1 \|\| f.tokens.Load() != 1 {	31		if f.heads.Load() != 2 \|\| f.gets.Load() != 1 \|\| f.blobs.Load() != 1 \|\| f.tokens.Load() != 1 {
119		t.Fatalf("counts HEAD=%d GET=%d blobs=%d tokens=%d", f.heads.Load(), f.gets.Load(), f.blobs.Load(), f.tokens.Load())	32		t.Fatalf("counts HEAD=%d GET=%d blobs=%d tokens=%d", f.heads.Load(), f.gets.Load(), f.blobs.Load(), f.tokens.Load())
120		}	33		}
121	-	for _, down := range []int32{http.StatusServiceUnavailable, 0} {
122	-	f.status.Store(down)
123	-	if down == 0 {
124	-	f.server.Close()
125	-	}
126	-	if w := request(t, p, http.MethodGet, "/v2/library/alpine/manifests/3"); w.Code != http.StatusOK \|\| w.Body.String() != manifestBody {
127	-	t.Fatalf("stale: %d %s", w.Code, w.Body)
128	-	}
129	-	if w := request(t, p, http.MethodGet, "/v2/library/alpine/manifests/unknown"); w.Code != http.StatusBadGateway {
130	-	t.Fatalf("unknown: %d", w.Code)
131	-	}
132	-	}
133		}	34		}
			35	+
			36	+	// HEAD of a blob answers its length, cold or stored, with no body; a stored
			37	+	// blob serves ranges.
134		func TestBlobHeadAndRange(t *testing.T) {	38		func TestBlobHeadAndRange(t *testing.T) {
135		f := fake(t)	39		f := fake(t)
136		p := proxyFor(t, f)	40		p := proxyFor(t, f)

@@ -162,25 +66,18 @@ func TestCorruptBlobRejected(t *testing.T) {
162		if w.Code != http.StatusBadGateway \|\| strings.Contains(w.Body.String(), "xxxx") {	66		if w.Code != http.StatusBadGateway \|\| strings.Contains(w.Body.String(), "xxxx") {
163		t.Fatalf("corrupt blob: %d %q", w.Code, w.Body)	67		t.Fatalf("corrupt blob: %d %q", w.Code, w.Body)
164		}	68		}
165	-	if _, err := os.Stat(filepath.Join(p.s.dir, "blobs", sum(blobBody)[7:])); !os.IsNotExist(err) {	69	+	if _, err := os.Stat(filepath.Join(p.s.dir, kindBlob.key(sum(blobBody)[7:]))); !os.IsNotExist(err) {
166		t.Fatal("corrupt blob stored")	70		t.Fatal("corrupt blob stored")
167		}	71		}
168	-	ds, _ := os.ReadDir(filepath.Join(p.s.dir, "tmp"))	72	+	if ds, _ := os.ReadDir(filepath.Join(p.s.dir, "tmp")); len(ds) != 0 {
169	-	if len(ds) != 0 {
170		t.Fatal("partial temp retained")	73		t.Fatal("partial temp retained")
171		}	74		}
172		}	75		}
173	-	func TestDeniedAndValidation(t *testing.T) {	76	+
174	-	f := fake(t)	77	+	// Only GET and HEAD of valid names, tags and digests under /v2/ are served.
175	-	p := proxyFor(t, f)	78	+	func TestRouteValidation(t *testing.T) {
176	-	f.status.Store(http.StatusUnauthorized)	79	+	p := proxyFor(t, fake(t))
177	-	if w := request(t, p, http.MethodGet, "/v2/library/alpine/manifests/3"); w.Code != http.StatusNotFound {	80	+	for _, path := range []string{"/v2/../library/alpine/manifests/3", "/v2/Library/alpine/manifests/3", "/v2/library/alpine/manifests/.bad", "/v2/library/alpine/manifests/sha256:bad", "/v2/library/alpine/blobs/sha256:bad", "/v2/library/alpine/tags/list", "/v3/library/alpine/manifests/3"} {
178	-	t.Fatal(w.Code)
179	-	}
180	-	if f.tokens.Load() != 2 {
181	-	t.Fatal("no fresh token retry")
182	-	}
183	-	for _, path := range []string{"/v2/../library/alpine/manifests/3", "/v2/Library/alpine/manifests/3", "/v2/library/alpine/manifests/.bad", "/v2/library/alpine/manifests/sha256:bad", "/v2/library/alpine/blobs/sha256:bad", "/v2/library/alpine/tags/list"} {
184		if w := request(t, p, http.MethodGet, path); w.Code != http.StatusNotFound {	81		if w := request(t, p, http.MethodGet, path); w.Code != http.StatusNotFound {
185		t.Errorf("%s: %d", path, w.Code)	82		t.Errorf("%s: %d", path, w.Code)
186		}	83		}

@@ -188,7 +85,13 @@ func TestDeniedAndValidation(t *testing.T) {
188		if w := request(t, p, http.MethodPut, "/v2/"); w.Code != http.StatusMethodNotAllowed {	85		if w := request(t, p, http.MethodPut, "/v2/"); w.Code != http.StatusMethodNotAllowed {
189		t.Fatal(w.Code)	86		t.Fatal(w.Code)
190		}	87		}
			88	+	if w := request(t, p, http.MethodGet, "/v2/"); w.Code != http.StatusOK \|\| w.Body.String() != "{}" {
			89	+	t.Fatalf("/v2/: %d %s", w.Code, w.Body)
			90	+	}
191		}	91		}
			92	+
			93	+	// A manifest whose bytes do not match its digest, or over the size limit, is
			94	+	// a 502 and is not stored.
192		func TestManifestLimitsAndDigest(t *testing.T) {	95		func TestManifestLimitsAndDigest(t *testing.T) {
193		for _, body := range []string{"tampered", strings.Repeat("x", manifestLimit+1)} {	96		for _, body := range []string{"tampered", strings.Repeat("x", manifestLimit+1)} {
194		up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {	97		up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

@@ -207,7 +110,7 @@ func TestManifestLimitsAndDigest(t *testing.T) {
207		if w.Code != http.StatusBadGateway {	110		if w.Code != http.StatusBadGateway {
208		t.Fatal(w.Code)	111		t.Fatal(w.Code)
209		}	112		}
210	-	if _, _, err := p.s.open("manifests", sum(manifestBody)); err == nil {	113	+	if _, _, err := p.s.open(kindManifest, sum(manifestBody)); err == nil {
211		t.Fatal("invalid manifest stored")	114		t.Fatal("invalid manifest stored")
212		}	115		}
213		}	116		}

M internal/registryproxy/quota.go +14 -3

 // SPDX-License-Identifier: MIT
 package registryproxy
 import (
 type quotaKey struct{}
-// QuotaOf returns the quota attached by WithQuota, or nil.
-func QuotaOf(ctx context.Context) *Quota { return quotaFrom(ctx) }
 func quotaFrom(ctx context.Context) *Quota {
 	q, _ := ctx.Value(quotaKey{}).(*Quota)
 	return q
 }
+// WithJob attaches the id of the job a request serves to its context: the
+// Proxy logs it as job_id on its registry lines.
+func WithJob(ctx context.Context, job string) context.Context {
+	return context.WithValue(ctx, jobKey{}, job)
+}
+type jobKey struct{}
+func jobFrom(ctx context.Context) string {
+	job, _ := ctx.Value(jobKey{}).(string)
+	return job
+}
 func (q *Quota) tryAcquire() bool {
 	for {
 		n := q.used.Load()

R internal/registryproxy/quota_test.go → internal/registryproxy/slots_test.go +88 -125 renamed, 51% similar

 // SPDX-License-Identifier: MIT
 package registryproxy
 import (
 	"io"
 	"net/http"
 	"net/http/httptest"
-	"os"
-	"path/filepath"
 	"strings"
 	"sync"
-	"sync/atomic"
 	"testing"
 	"time"
 )
-// serveAsync serves path with ctx on its own goroutine, into w (a new
+// At most MaxDownloads cold blobs download at once; one more waits for
-// recorder when nil); the status code comes on the returned channel.
+// SlotWait and then gets 503 (dockerd retries or pulls from Docker Hub), a
-func serveAsync(p *Proxy, ctx context.Context, path string, w *keepAliveRecorder) <-chan int {
+// joiner of a running download takes no slot, and a slot frees once a
-	if w == nil {
+// download ends.
-		w = &keepAliveRecorder{ResponseRecorder: httptest.NewRecorder()}
+func TestColdDownloadCap(t *testing.T) {
-	}
+	u := newStallingUpstream(t)
-	code := make(chan int, 1)
+	const limit = 3
-	go func() {
+	const wait = 50 * time.Millisecond
-		p.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil).WithContext(ctx))
+	p := proxyWith(t, u.srv, Config{AbandonGrace: 10 * time.Millisecond, MaxDownloads: limit, SlotWait: wait})
-		code <- w.Code
+	ctx, cancel := context.WithCancel(context.Background())
-	}()
+	var wg sync.WaitGroup
-	return code
+	for i := range limit {
+		wg.Go(func() {
+			p.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, blobPath(i), nil).WithContext(ctx))
+		})
+	}
+	waitFor(t, p.s, "the cap to fill", func() bool { return p.s.downloads == limit })
+	start := time.Now()
+	w := request(t, p, http.MethodGet, blobPath(limit))
+	if w.Code != http.StatusServiceUnavailable || w.Header().Get("Retry-After") == "" {
+		t.Fatalf("over the cap: %d", w.Code)
+	}
+	if d := time.Since(start); d < wait {
+		t.Fatalf("503 after %v, before the slot wait", d)
+	}
+	// Joining a running download takes no slot: the joiner is a waiter of the
+	// flight, and the number of downloads does not move.
+	key := kindBlob.key(sum("blob-0")[7:])
+	go p.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, blobPath(0), nil).WithContext(ctx))
+	waitFor(t, p.s, "the joiner", func() bool { return waiters(p.s, key) == 2 })
+	p.s.mu.Lock()
+	downloads := p.s.downloads
+	p.s.mu.Unlock()
+	if downloads != limit {
+		t.Fatalf("%d downloads with a joiner, want %d", downloads, limit)
+	}
+	cancel()
+	wg.Wait()
+	waitFor(t, p.s, "the slots to free", func() bool { return p.s.downloads == 0 })
+	if u.started.Load() != limit {
+		t.Fatalf("%d downloads started", u.started.Load())
+	}
 }
 // A sandbox that saturates its quota does not hold back another sandbox: its
 // download starts at once while the global cap still has room.
 func TestQuotaIsolatesChannels(t *testing.T) {
 	u := newStallingUpstream(t)
-	p := proxyWith(t, u.srv, Config{MaxDownloads: 16, SlotWait: time.Hour, AbandonGrace: 10 * time.Millisecond})
+	p := proxyWith(t, u.srv, Config{SlotWait: time.Hour, AbandonGrace: 10 * time.Millisecond})
 	a, b := NewQuota(3), NewQuota(3)
 	ctxA, cancelA := context.WithCancel(WithQuota(context.Background(), a))
 	defer cancelA()
 	if d := time.Since(start); d > time.Second {
 		t.Fatalf("B started after %v", d)
 	}
-	if got := u.started.Load(); got != 4 {
+	// The quotas are taken before the downloads reach upstream: wait for the
-		t.Fatalf("%d downloads started, want A's 3 and B's 1", got)
+	// 4 of A and B there, then check under the store lock that A's fourth
+	// took no slot.
+	waitFor(t, p.s, "4 downloads upstream", func() bool { return u.started.Load() == 4 })
+	p.s.mu.Lock()
+	downloads := p.s.downloads
+	p.s.mu.Unlock()
+	if downloads != 4 {
+		t.Fatalf("%d downloads, want A's 3 and B's 1", downloads)
 	}
 	cancelA()
 	cancelB()
 	waitFor(t, p.s, "the quota to fill", func() bool { return q.used.Load() == 1 })
 	w := &keepAliveRecorder{ResponseRecorder: httptest.NewRecorder()}
 	second := serveAsync(p, ctx, "/v2/library/alpine/blobs/"+sum(body), w)
-	for end := time.Now().Add(5 * time.Second); w.calls.Load() == 0; { // waiting for a slot, kept alive
+	waitKeptAlive(t, w) // waiting for a slot
-		select {
+	select {
-		case code := <-second:
+	case code := <-second:
-			t.Fatalf("did not wait for a slot: %d", code)
+		t.Fatalf("did not wait for a slot: %d", code)
-		default:
+	default:
-		}
-		if time.Now().After(end) {
-			t.Fatal("not kept alive while waiting for a slot")
-		}
-		time.Sleep(time.Millisecond)
 	}
 	unblock() // the first download ends (digest mismatch), freeing the slot
 	if code := <-first; code != http.StatusBadGateway {
 	if d := time.Since(start); d < wait {
 		t.Fatalf("503 after %v", d)
 	}
-	// A request that disconnects while waiting leaves at once, with no slot.
-	cctx, ccancel := context.WithCancel(WithQuota(context.Background(), q))
-	waiting := serveAsync(p, cctx, blobPath(2), nil)
-	ccancel()
-	<-waiting
 	if u.started.Load() != 1 {
 		t.Fatalf("%d downloads started", u.started.Load())
 	}
 }
+// A request that disconnects while it waits for a slot leaves at once, with no
+// slot and no 503.
+func TestQuotaWaiterDisconnects(t *testing.T) {
+	u := newStallingUpstream(t)
+	const wait = time.Hour
+	p := proxyWith(t, u.srv, Config{SlotWait: wait, AbandonGrace: 10 * time.Millisecond, KeepAliveInterval: 5 * time.Millisecond})
+	q := NewQuota(1)
+	ctx, cancel := context.WithCancel(WithQuota(context.Background(), q))
+	defer cancel()
+	serveAsync(p, ctx, blobPath(0), nil)
+	waitFor(t, p.s, "the quota to fill", func() bool { return q.used.Load() == 1 })
+	wctx, wcancel := context.WithCancel(WithQuota(context.Background(), q))
+	w := &keepAliveRecorder{ResponseRecorder: httptest.NewRecorder()}
+	waiting := serveAsync(p, wctx, blobPath(1), w)
+	waitKeptAlive(t, w) // really waiting for a slot
+	start := time.Now()
+	wcancel()
+	code := <-waiting
+	if d := time.Since(start); d > time.Second {
+		t.Fatalf("left after %v", d)
+	}
+	if code == http.StatusServiceUnavailable {
+		t.Fatal("a disconnected waiter got 503")
+	}
+	if u.started.Load() != 1 || q.used.Load() != 1 {
+		t.Fatalf("%d downloads started, quota %d", u.started.Load(), q.used.Load())
+	}
+}
 // Joining a download in progress takes none of the joiner's quota.
 func TestQuotaJoinersExempt(t *testing.T) {
 	u := newStallingUpstream(t)
 	ctxB, cancelB := context.WithCancel(WithQuota(context.Background(), b))
 	defer cancelB()
 	serveAsync(p, ctxB, blobPath(0), nil) // joins A's
-	key := "blobs/" + sum("blob-0")[7:]
+	key := kindBlob.key(sum("blob-0")[7:])
 	waitFor(t, p.s, "B to join", func() bool { return waiters(p.s, key) == 2 })
 	if b.used.Load() != 0 {
 		t.Fatal("a joiner took quota")
 	serveAsync(p, ctxB, blobPath(1), nil)
 	waitFor(t, p.s, "B's own download", func() bool { return b.used.Load() == 1 })
 }
-// A panic in a download fails its flight and frees its slots; it does not
-// leave the waiters hanging.
-func TestPanickingFetchFailsFlight(t *testing.T) {
-	s, err := openStore(t.TempDir(), 1<<20, time.Now)
-	if err != nil {
-		t.Fatal(err)
-	}
-	s.maxDownloads = 1
-	q := NewQuota(1)
-	ctx, cancel := context.WithTimeout(WithQuota(context.Background(), q), 5*time.Second)
-	defer cancel()
-	_, _, _, err = s.ensure(ctx, "blobs", sum("x"), func(context.Context, io.Writer) (string, error) {
-		panic("boom")
-	}, keepalive{})
-	if err == nil || !strings.Contains(err.Error(), "panicked") {
-		t.Fatalf("err %v", err)
-	}
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	if s.downloads != 0 || len(s.flights) != 0 || q.used.Load() != 0 || s.pending != 0 {
-		t.Fatalf("leaked: downloads=%d flights=%d quota=%d pending=%d", s.downloads, len(s.flights), q.used.Load(), s.pending)
-	}
-	if ds, _ := os.ReadDir(filepath.Join(s.dir, "tmp")); len(ds) != 0 {
-		t.Fatal("temp file kept")
-	}
-}
-// Eviction restarts from the front after a manifest removal that fails, and
-// does not loop on it.
-func TestEvictionAfterFailedManifestRemoval(t *testing.T) {
-	now := time.Unix(1_700_000_000, 0)
-	const media = "application/vnd.oci.image.index.v1+json"
-	s, err := openStore(t.TempDir(), 1<<20, func() time.Time { return now })
-	if err != nil {
-		t.Fatal(err)
-	}
-	m := `{"schemaVersion":2}`
-	f, _, _, err := s.ensure(context.Background(), "manifests", sum(m), func(_ context.Context, w io.Writer) (string, error) {
-		_, err := io.WriteString(w, m)
-		return media, err
-	}, keepalive{})
-	if err != nil {
-		t.Fatal(err)
-	}
-	_ = f.Close()
-	now = now.Add(time.Second)
-	if err := s.setTag("library/alpine", "3", "", sum(m)); err != nil {
-		t.Fatal(err)
-	}
-	now = now.Add(time.Second)
-	put(t, s, "blob")
-	// The manifest's .type cannot be removed: replace it with a non-empty
-	// directory. Its tag goes first all the same.
-	typ := filepath.Join(s.dir, "manifests", sum(m)[7:]+".type")
-	if err := os.Remove(typ); err != nil {
-		t.Fatal(err)
-	}
-	if err := os.MkdirAll(filepath.Join(typ, "x"), 0700); err != nil {
-		t.Fatal(err)
-	}
-	s.mu.Lock()
-	s.evictLocked(s.max) // ask for everything
-	_, manifestKept := s.entries["manifests/"+sum(m)[7:]]
-	_, blobKept := s.entries["blobs/"+sum("blob")[7:]]
-	tags := len(s.tagsOf)
-	s.mu.Unlock()
-	if !manifestKept || blobKept || tags != 0 {
-		t.Fatalf("manifest kept=%v blob kept=%v tags=%d", manifestKept, blobKept, tags)
-	}
-	accounted(t, s)
-}
-// unpinLocked of an unpinned entry is reported (and fails the test run through
-// TestMain's hook); here the hook is replaced to observe it.
-func TestUnpinBugReported(t *testing.T) {
-	s, err := openStore(t.TempDir(), 1<<20, time.Now)
-	if err != nil {
-		t.Fatal(err)
-	}
-	var bugs atomic.Int32
-	s.hookBug = func(string) { bugs.Add(1) }
-	put(t, s, "aaaa")
-	s.mu.Lock()
-	s.unpinLocked("blobs/" + sum("aaaa")[7:])
-	pins := s.entries["blobs/"+sum("aaaa")[7:]].pins
-	s.mu.Unlock()
-	if bugs.Load() != 1 || pins != 0 {
-		t.Fatalf("bugs %d pins %d", bugs.Load(), pins)
-	}
-}

M internal/registryproxy/store.go +114 -57

 // 503, and dockerd retries the layer (about 50 s) before pulling from Docker Hub.
 var errBusy = errors.New("registry downloads busy")
-const tagCharge int64 = 4096
+// minCharge is the least an entry is charged, whatever its size: about one
+// disk block and inode. It bounds the number of entries, and so of files, to
+// Max/minCharge. A tag file is charged exactly that.
+const minCharge int64 = 4096
+// charge is what an entry of n bytes counts against Max.
+func charge(n int64) int64 { return max(n, minCharge) }
+// kind is the directory of an entry: content (blobs, manifests) or a tag.
+type kind uint8
+const (
+	kindBlob kind = iota
+	kindManifest
+	kindTag
+)
+var kinds = []kind{kindBlob, kindManifest, kindTag}
+// dir is the directory of the kind in the store, and its route in /v2/.
+func (k kind) dir() string {
+	switch k {
+	case kindBlob:
+		return "blobs"
+	case kindManifest:
+		return "manifests"
+	default:
+		return "tags"
+	}
+}
+// key is the key of an entry of kind k: its path in the store.
+func (k kind) key(name string) string { return k.dir() + "/" + name }
 // lowWater bounds the extra room an eviction frees (with 5% of Max), so that a
 // full store evicts once per that many bytes, not on every write.
 type entry struct {
 	key    string
+	kind   kind
 	size   int64
 	used   time.Time
 	pins   int
 	gen      int // bumped each time waiters drops to zero
 	finished bool
 	pinned   bool   // the committed entry holds a pin for the waiters
-	blob     bool   // takes a download slot
+	kind     kind   // a blob flight takes a download slot
 	quota    *Quota // the starter's quota, held until the flight ends
 	cancel   context.CancelFunc
 }
-type store struct {
+// storeConfig is a store's complete configuration: openStore applies no
-	dir                 string
+// defaults.
-	max, total, pending int64
+type storeConfig struct {
-	now                 func() time.Time
+	dir string
-	// grace is how long a flight with no waiter left keeps downloading in
+	max int64
+	now func() time.Time
+	// grace is how long a blob flight with no waiter left keeps downloading in
 	// case another request joins it; then it is cancelled.
 	grace time.Duration
-	// maxDownloads caps the concurrent blob downloads (0: no cap).
+	// maxDownloads caps the concurrent blob downloads.
-	maxDownloads, downloads int
+	maxDownloads int
 	// slotWait is how long a request waits for a download slot before errBusy.
 	slotWait time.Duration
+	log      zerowrap.Logger
+}
+type store struct {
+	storeConfig
+	mu             sync.Mutex
+	total, pending int64
+	downloads      int
 	// slotFreed is closed and replaced each time a download slot frees.
 	slotFreed chan struct{}
-	log       zerowrap.Logger
-	mu      sync.Mutex
 	entries map[string]*entry
 	lru     *list.List                     // *entry, least recently used first
 	tagsOf  map[string]map[string]struct{} // manifest digest -> tag keys
 	return f.err
 }
-func openStore(dir string, max int64, now func() time.Time) (*store, error) {
+func openStore(c storeConfig) (*store, error) {
-	if max <= 0 {
+	if c.max < minCharge {
-		return nil, errors.New("registry cache cap must be positive")
+		return nil, fmt.Errorf("registry cache cap must be at least %d bytes", minCharge)
 	}
-	s := &store{dir: dir, max: max, now: now, entries: map[string]*entry{}, lru: list.New(), tagsOf: map[string]map[string]struct{}{}, flights: map[string]*flight{}, slotFreed: make(chan struct{}), hookBug: defaultBugHook}
+	if c.now == nil || c.grace <= 0 || c.maxDownloads <= 0 || c.slotWait <= 0 {
-	for _, kind := range []string{"blobs", "manifests", "tags", "tmp"} {
+		return nil, errors.New("registry store configuration incomplete")
-		if err := os.MkdirAll(filepath.Join(dir, kind), 0700); err != nil {
+	}
+	dir := c.dir
+	s := &store{storeConfig: c, entries: map[string]*entry{}, lru: list.New(), tagsOf: map[string]map[string]struct{}{}, flights: map[string]*flight{}, slotFreed: make(chan struct{}), hookBug: defaultBugHook}
+	for _, sub := range []string{kindBlob.dir(), kindManifest.dir(), kindTag.dir(), "tmp"} {
+		if err := os.MkdirAll(filepath.Join(dir, sub), 0700); err != nil {
 			return nil, err
 		}
 	}
 		}
 	}
 	var types []string
-	for _, kind := range []string{"blobs", "manifests", "tags"} {
+	for _, k := range kinds {
-		ds, err := os.ReadDir(filepath.Join(dir, kind))
+		ds, err := os.ReadDir(filepath.Join(dir, k.dir()))
 		if err != nil {
 			return nil, err
 		}
 		for _, de := range ds {
-			if base, ok := strings.CutSuffix(de.Name(), ".type"); ok && kind == "manifests" {
+			if base, ok := strings.CutSuffix(de.Name(), ".type"); ok && k == kindManifest {
 				types = append(types, base)
 				continue
 			}
 			if !fi.Mode().IsRegular() {
 				continue
 			}
-			key := kind + "/" + de.Name()
+			key := k.key(de.Name())
-			e := &entry{key: key, size: fi.Size(), used: fi.ModTime()}
+			size := fi.Size()
-			if kind == "manifests" {
+			e := &entry{key: key, kind: k, used: fi.ModTime()}
+			switch k {
+			case kindManifest:
 				info, err := os.Stat(filepath.Join(dir, key) + ".type")
 				if err != nil {
 					_ = os.Remove(filepath.Join(dir, key))
 					continue
 				}
-				e.size += info.Size()
+				size += info.Size()
-			}
+			case kindTag:
-			if kind == "tags" {
 				digest := s.readTagFile(key)
 				if !validDigest(digest) {
 					_ = os.Remove(filepath.Join(dir, key))
 					continue
 				}
-				if _, ok := s.entries["manifests/"+digest[7:]]; !ok {
+				if _, ok := s.entries[kindManifest.key(digest[7:])]; !ok {
 					_ = os.Remove(filepath.Join(dir, key))
 					continue
 				}
-				e.size = tagCharge
+				size = 0
 				e.digest = digest
 			}
+			e.size = charge(size)
 			s.addLocked(e)
 		}
 	}
 	// A .type whose manifest is gone (a crash between the two writes).
 	for _, base := range types {
-		if _, ok := s.entries["manifests/"+base]; !ok {
+		if _, ok := s.entries[kindManifest.key(base)]; !ok {
-			_ = os.Remove(filepath.Join(dir, "manifests", base+".type"))
+			_ = os.Remove(filepath.Join(dir, kindManifest.key(base)+".type"))
 		}
 	}
 	// Order the LRU by the access times kept in the mtimes.
 	if e.pins > 0 {
 		return false
 	}
-	if hexDigest, ok := strings.CutPrefix(key, "manifests/"); ok {
+	if e.kind == kindManifest {
 		// Prune references first. If a tag cannot be removed, retain its target too.
-		for k := range s.tagsOf["sha256:"+hexDigest] {
+		for k := range s.tagsOf["sha256:"+strings.TrimPrefix(key, kindManifest.key(""))] {
 			if !s.removeLocked(k) {
 				return false
 			}
 		next := el.Next()
 		if e.pins == 0 {
 			removed := s.removeLocked(e.key)
-			if strings.HasPrefix(e.key, "manifests/") {
+			if e.kind == kindManifest {
 				// Removing a manifest removes its tags first, even when it then
 				// fails: next may be gone. Start over; only pinned or
 				// unremovable entries are walked again.
 		return nil, "", err
 	}
 	media := "application/octet-stream"
-	if strings.HasPrefix(key, "manifests/") {
+	if e.kind == kindManifest {
 		b, err := os.ReadFile(filepath.Join(s.dir, key) + ".type")
 		if err != nil {
 			_ = f.Close()
 	e.pins++
 	return &storedFile{File: f, s: s, key: key}, media, nil
 }
-func (s *store) open(kind, digest string) (*storedFile, string, error) {
+func (s *store) open(k kind, digest string) (*storedFile, string, error) {
-	if !validDigest(digest) || (kind != "blobs" && kind != "manifests") {
+	if !validDigest(digest) {
 		return nil, "", errDigest
 	}
 	s.mu.Lock()
 	defer s.mu.Unlock()
-	return s.openLocked(kind + "/" + digest[7:])
+	return s.openLocked(k.key(digest[7:]))
 }
 // fetch writes the upstream content to w. ctx belongs to the flight: it ends
 // carried by ctx (WithQuota), both held until the flight really ends; joining
 // one takes neither. When a slot is missing, the request waits for one, kept
 // alive, for at most the slot wait, and then fails with errBusy.
-func (s *store) ensure(ctx context.Context, kind, digest string, fetch fetch, keep keepalive) (f *storedFile, media string, hit bool, err error) {
+func (s *store) ensure(ctx context.Context, k kind, digest string, fetch fetch, keep keepalive) (f *storedFile, media string, hit bool, err error) {
-	if !validDigest(digest) || (kind != "blobs" && kind != "manifests") {
+	if !validDigest(digest) {
 		return nil, "", false, errDigest
 	}
-	key := kind + "/" + digest[7:]
+	key := k.key(digest[7:])
-	blob := kind == "blobs"
+	blob := k == kindBlob
 	quota := quotaFrom(ctx)
 	var slotDeadline <-chan time.Time
 	var fl *flight
 		}
 		if !blob || s.takeSlotLocked(quota) {
 			dctx, cancel := context.WithCancel(context.Background())
-			fl = &flight{key: key, done: make(chan struct{}), cancel: cancel, blob: blob}
+			fl = &flight{key: key, kind: k, done: make(chan struct{}), cancel: cancel}
 			if blob {
 				fl.quota = quota
 			}
 	// once, so its flights never outnumber the requests in progress. A blob
 	// holds a download slot until it really ends, which bounds the abandoned
 	// ones to MaxDownloads.
-	if s.grace <= 0 || !fl.blob {
+	if fl.kind != kindBlob {
 		s.abandonLocked(fl)
 		return
 	}
 // takeSlotLocked takes a global download slot and one of quota (nil: none),
 // both or neither.
 func (s *store) takeSlotLocked(quota *Quota) bool {
-	if s.maxDownloads > 0 && s.downloads >= s.maxDownloads {
+	if s.downloads >= s.maxDownloads {
 		return false
 	}
 	if quota != nil && !quota.tryAcquire() {
 	if s.flights[fl.key] == fl {
 		delete(s.flights, fl.key)
 	}
-	if fl.blob {
+	if fl.kind == kindBlob {
 		s.downloads--
 		if fl.quota != nil {
 			fl.quota.release()
 	}
 	path := filepath.Join(s.dir, key)
 	metadata := int64(0)
-	if strings.HasPrefix(key, "manifests/") {
+	if fl.kind == kindManifest {
 		if len(media) > 1024 {
 			return errors.New("manifest media type too large")
 		}
 		metadata = int64(len(media))
-		s.evictLocked(metadata)
+	}
-		if metadata > s.max-s.total-s.pending {
+	// The download is charged its bytes in pending; the entry is charged at
+	// least minCharge, and its media type.
+	size := charge(cw.n + metadata)
+	if extra := size - cw.n; extra > 0 {
+		s.evictLocked(extra)
+		if extra > s.max-s.total-s.pending {
 			return errTooLarge
 		}
+	}
+	if fl.kind == kindManifest {
 		if err := os.WriteFile(path+".type", []byte(media), 0600); err != nil {
 			return err
 		}
 	}
 	if err := os.Rename(f.Name(), path); err != nil {
-		if metadata > 0 {
+		if fl.kind == kindManifest {
 			_ = os.Remove(path + ".type")
 		}
 		return err
 	}
-	e := &entry{key: key, size: cw.n + metadata, used: s.now()}
+	e := &entry{key: key, kind: fl.kind, size: size, used: s.now()}
 	s.addLocked(e)
 	s.pending -= cw.n
 	cw.n = 0
 	s.finishLocked(fl, nil, e)
 	return nil
 }
+// tagKey is the key of a tag for a normalized Accept (normalizeAccept).
 func tagKey(name, tag, accept string) string {
-	return fmt.Sprintf("%x", sha256.Sum256([]byte(name+"\x00"+tag+"\x00"+normalizeAccept(accept))))
+	return kindTag.key(fmt.Sprintf("%x", sha256.Sum256([]byte(name+"\x00"+tag+"\x00"+accept))))
 }
 func (s *store) readTagFile(key string) string {
 	f, err := os.Open(filepath.Join(s.dir, key))
 func (s *store) tag(name, tag, accept string) string {
 	s.mu.Lock()
 	defer s.mu.Unlock()
-	e, ok := s.entries["tags/"+tagKey(name, tag, accept)]
+	e, ok := s.entries[tagKey(name, tag, accept)]
 	if !ok {
 		return ""
 	}
 	}
 	s.mu.Lock()
 	defer s.mu.Unlock()
-	manifest, ok := s.entries["manifests/"+digest[7:]]
+	manifest, ok := s.entries[kindManifest.key(digest[7:])]
 	if !ok {
 		return os.ErrNotExist
 	}
 	// Protect the target while making room for its metadata.
 	manifest.pins++
 	defer func() { manifest.pins-- }()
-	key := "tags/" + tagKey(name, tag, accept)
+	key := tagKey(name, tag, accept)
 	old, exists := s.entries[key]
 	if exists {
 		// Already charged: rewriting it needs no room.
 		s.touchLocked(old)
 	} else {
-		s.evictLocked(tagCharge)
+		s.evictLocked(charge(0))
-		if tagCharge > s.max-s.total-s.pending {
+		if charge(0) > s.max-s.total-s.pending {
 			return errTooLarge
 		}
 	}
 		s.indexTagLocked(key, digest)
 		return nil
 	}
-	s.addLocked(&entry{key: key, size: tagCharge, used: s.now(), digest: digest})
+	s.addLocked(&entry{key: key, kind: kindTag, size: charge(0), used: s.now(), digest: digest})
 	return nil
 }

M internal/registryproxy/store_test.go +280 -78

@@ -1,125 +1,327 @@
1		// SPDX-License-Identifier: MIT	1		// SPDX-License-Identifier: MIT
			2	+
2		package registryproxy	3		package registryproxy
3			4
4		import (	5		import (
5		"context"	6		"context"
6	-	"crypto/sha256"
7		"fmt"	7		"fmt"
8		"io"	8		"io"
			9	+	"net/http"
			10	+	"net/http/httptest"
9		"os"	11		"os"
10		"path/filepath"	12		"path/filepath"
11	-	"sync"	13	+	"strings"
12	-	"sync/atomic"
13		"testing"	14		"testing"
14		"time"	15		"time"
15		)	16		)
16			17
17	-	func sum(b string) string { return fmt.Sprintf("sha256:%x", sha256.Sum256([]byte(b))) }	18	+	// Reopening a store keeps its content, charges it again and clears the
18	-	func put(t testing.T, s store, b string) {	19	+	// incomplete downloads in tmp/.
19	-	t.Helper()	20	+	func TestStoreReopen(t *testing.T) {
20	-	f, _, _, err := s.ensure(context.Background(), "blobs", sum(b), func(_ context.Context, w io.Writer) (string, error) { _, err := io.WriteString(w, b); return "", err }, keepalive{})	21	+	dir := t.TempDir()
			22	+	s := mustOpen(t, storeCfg(dir, 8*minCharge))
			23	+	put(t, s, "aaaa")
			24	+	putManifest(t, s, manifestBody, "application/vnd.oci.image.index.v1+json")
			25	+	if err := s.setTag("library/alpine", "3", normalizeAccept(""), sum(manifestBody)); err != nil {
			26	+	t.Fatal(err)
			27	+	}
			28	+	if err := os.WriteFile(filepath.Join(dir, "tmp", "leftover"), []byte("partial"), 0600); err != nil {
			29	+	t.Fatal(err)
			30	+	}
			31	+	total := s.total
			32	+	s = mustOpen(t, storeCfg(dir, 8*minCharge))
			33	+	f, _, err := s.open(kindBlob, sum("aaaa"))
21		if err != nil {	34		if err != nil {
22		t.Fatal(err)	35		t.Fatal(err)
23		}	36		}
24		_ = f.Close()	37		_ = f.Close()
			38	+	if got := s.tag("library/alpine", "3", normalizeAccept("")); got != sum(manifestBody) {
			39	+	t.Fatalf("tag after reopen: %q", got)
			40	+	}
			41	+	if ds, _ := os.ReadDir(filepath.Join(dir, "tmp")); len(ds) != 0 {
			42	+	t.Fatal("tmp not cleared")
			43	+	}
			44	+	if accounted(t, s); s.total != total {
			45	+	t.Fatalf("reopened total %d, was %d", s.total, total)
			46	+	}
25		}	47		}
26	-	func TestStoreIntegrityReopenEviction(t *testing.T) {	48	+
27	-	now := time.Now()	49	+	// Every entry is charged at least minCharge, on commit and on reopen, so tiny
			50	+	// blobs cannot make more than Max/minCharge files.
			51	+	func TestEntriesChargedAtLeastMinCharge(t *testing.T) {
28		dir := t.TempDir()	52		dir := t.TempDir()
29	-	s, err := openStore(dir, 8, func() time.Time { return now })	53	+	s := mustOpen(t, storeCfg(dir, 3*minCharge))
30	-	if err != nil {	54	+	for i := range 10 {
			55	+	put(t, s, fmt.Sprint("tiny-", i))
			56	+	}
			57	+	accounted(t, s)
			58	+	if n := len(s.entries); n > 3 {
			59	+	t.Fatalf("%d entries under a cap of 3 charges", n)
			60	+	}
			61	+	s = mustOpen(t, storeCfg(dir, 3*minCharge))
			62	+	accounted(t, s)
			63	+	if s.total != int64(len(s.entries))*minCharge {
			64	+	t.Fatalf("reopened total %d for %d entries", s.total, len(s.entries))
			65	+	}
			66	+	}
			67	+
			68	+	// A cap below one entry's minimum charge is refused; a blob larger than Max is
			69	+	// refused and leaves nothing charged.
			70	+	func TestStoreOversize(t *testing.T) {
			71	+	if _, err := openStore(storeCfg(t.TempDir(), minCharge-1)); err == nil {
			72	+	t.Fatal("cap below minCharge accepted")
			73	+	}
			74	+	s := mustOpen(t, storeCfg(t.TempDir(), minCharge))
			75	+	large := strings.Repeat("x", int(minCharge)+1)
			76	+	_, _, _, err := s.ensure(context.Background(), kindBlob, sum(large), func(_ context.Context, w io.Writer) (string, error) {
			77	+	_, err := io.WriteString(w, large)
			78	+	return "", err
			79	+	}, keepalive{})
			80	+	if err != errTooLarge {
31		t.Fatal(err)	81		t.Fatal(err)
32		}	82		}
33	-	put(t, s, "aaaa")	83	+	if s.pending != 0 \|\| s.total != 0 {
34	-	now = now.Add(time.Second)	84	+	t.Fatalf("accounting: %d %d", s.pending, s.total)
35	-	put(t, s, "bbbb")	85	+	}
36	-	now = now.Add(time.Second)	86	+	}
37	-	f, _, err := s.open("blobs", sum("aaaa"))	87	+
			88	+	// Eviction walks the LRU, evicts least recently used first, frees down to the
			89	+	// low-water mark rather than the bare need, and skips pinned entries.
			90	+	func TestEvictionLRUAndLowWater(t *testing.T) {
			91	+	now := time.Unix(1_700_000_000, 0)
			92	+	const n = 40
			93	+	c := storeCfg(t.TempDir(), n*minCharge) // low water: 5%, two entries
			94	+	c.now = func() time.Time { return now }
			95	+	s := mustOpen(t, c)
			96	+	// 40 blobs fill the store. Freeing only the need would evict one blob; the
			97	+	// low-water mark evicts two.
			98	+	blob := func(i int) string { return fmt.Sprintf("%03d", i) + strings.Repeat("x", 47) }
			99	+	for i := range n {
			100	+	now = now.Add(time.Second)
			101	+	put(t, s, blob(i))
			102	+	}
			103	+	// Touch 0 and 1, and keep 2 open: the least recently used are then 3, 4...
			104	+	for _, i := range []int{0, 1} {
			105	+	now = now.Add(time.Second)
			106	+	f, _, err := s.open(kindBlob, sum(blob(i)))
			107	+	if err != nil {
			108	+	t.Fatal(err)
			109	+	}
			110	+	_ = f.Close()
			111	+	}
			112	+	pinned, _, err := s.open(kindBlob, sum(blob(2)))
38		if err != nil {	113		if err != nil {
39		t.Fatal(err)	114		t.Fatal(err)
40		}	115		}
41	-	_ = f.Close()	116	+	defer func() { _ = pinned.Close() }()
			117	+	s.mu.Lock()
			118	+	s.lru.MoveToFront(s.entries[kindBlob.key(sum(blob(2))[7:])].elem) // oldest of all, but open
			119	+	s.mu.Unlock()
42		now = now.Add(time.Second)	120		now = now.Add(time.Second)
43	-	put(t, s, "cccc")	121	+	put(t, s, "a") // over the cap: frees the low-water mark
44	-	if f, _, err := s.open("blobs", sum("bbbb")); err == nil {	122	+	present := func(i int) bool {
45	-	_ = f.Close()	123	+	_, err := os.Stat(filepath.Join(s.dir, kindBlob.key(sum(blob(i))[7:])))
46	-	t.Fatal("LRU survived eviction")	124	+	return err == nil
47		}	125		}
48	-	_, _, _, err = s.ensure(context.Background(), "blobs", sum("dddd"), func(_ context.Context, w io.Writer) (string, error) {	126	+	for i := range n {
49	-	_, err := io.WriteString(w, "evil")	127	+	want := i != 3 && i != 4 // 3 and 4 are the least recently used unpinned
50	-	return "", err	128	+	if present(i) != want {
51	-	}, keepalive{})	129	+	t.Errorf("blob %d present=%v", i, present(i))
52	-	if err != errDigest {	130	+	}
53	-	t.Fatalf("mismatch: %v", err)
54		}	131		}
55	-	if _, err := os.Stat(filepath.Join(dir, "blobs", sum("dddd")[7:])); !os.IsNotExist(err) {	132	+	accounted(t, s)
56	-	t.Fatalf("bad commit: %v", err)	133	+	s.mu.Lock()
			134	+	free := s.max - s.total
			135	+	s.mu.Unlock()
			136	+	if free < min(lowWater, s.max/20)-minCharge {
			137	+	t.Fatalf("freed only to %d", free)
57		}	138		}
58	-	put(t, s, "eeee")	139	+	}
59	-	if err := os.WriteFile(filepath.Join(dir, "tmp", "leftover"), []byte("partial"), 0600); err != nil {	140	+
			141	+	// Eviction restarts from the front after a manifest removal that fails, and
			142	+	// does not loop on it.
			143	+	func TestEvictionAfterFailedManifestRemoval(t *testing.T) {
			144	+	now := time.Unix(1_700_000_000, 0)
			145	+	c := storeCfg(t.TempDir(), 1<<20)
			146	+	c.now = func() time.Time { return now }
			147	+	s := mustOpen(t, c)
			148	+	m := `{"schemaVersion":2}`
			149	+	putManifest(t, s, m, "application/vnd.oci.image.index.v1+json")
			150	+	now = now.Add(time.Second)
			151	+	if err := s.setTag("library/alpine", "3", "", sum(m)); err != nil {
60		t.Fatal(err)	152		t.Fatal(err)
61		}	153		}
62	-	s, err = openStore(dir, 8, time.Now)	154	+	now = now.Add(time.Second)
63	-	if err != nil {	155	+	put(t, s, "blob")
			156	+	// The manifest's .type cannot be removed: replace it with a non-empty
			157	+	// directory. Its tag goes first all the same.
			158	+	typ := filepath.Join(s.dir, kindManifest.key(sum(m)[7:])+".type")
			159	+	if err := os.Remove(typ); err != nil {
64		t.Fatal(err)	160		t.Fatal(err)
65		}	161		}
66	-	if f, _, err := s.open("blobs", sum("eeee")); err != nil {	162	+	if err := os.MkdirAll(filepath.Join(typ, "x"), 0700); err != nil {
67		t.Fatal(err)	163		t.Fatal(err)
68	-	} else {
69	-	_ = f.Close()
70	-	}
71	-	ds, _ := os.ReadDir(filepath.Join(dir, "tmp"))
72	-	if len(ds) != 0 {
73	-	t.Fatal("tmp not cleared")
74		}	164		}
75		s.mu.Lock()	165		s.mu.Lock()
76	-	defer s.mu.Unlock()	166	+	s.evictLocked(s.max) // ask for everything
77	-	if s.total+s.pending > 8 {	167	+	_, manifestKept := s.entries[kindManifest.key(sum(m)[7:])]
78	-	t.Fatal("cap exceeded")	168	+	_, blobKept := s.entries[kindBlob.key(sum("blob")[7:])]
			169	+	tags := len(s.tagsOf)
			170	+	s.mu.Unlock()
			171	+	if !manifestKept \|\| blobKept \|\| tags != 0 {
			172	+	t.Fatalf("manifest kept=%v blob kept=%v tags=%d", manifestKept, blobKept, tags)
79		}	173		}
			174	+	accounted(t, s)
80		}	175		}
81	-	func TestStoreConcurrentEnsure(t *testing.T) {	176	+
82	-	s, err := openStore(t.TempDir(), 1024, time.Now)	177	+	// The committed entry is pinned for its waiters, so an eviction between the
			178	+	// commit and their open cannot remove it.
			179	+	func TestCommittedEntryPinnedForWaiters(t *testing.T) {
			180	+	s := mustOpen(t, storeCfg(t.TempDir(), minCharge))
			181	+	key := kindBlob.key(sum("aaaa")[7:])
			182	+	s.hookCommitted = func(string) {
			183	+	// Ask for the whole store before the waiters are woken.
			184	+	s.mu.Lock()
			185	+	s.evictLocked(s.max)
			186	+	s.mu.Unlock()
			187	+	}
			188	+	f, _, _, err := s.ensure(context.Background(), kindBlob, sum("aaaa"), func(_ context.Context, w io.Writer) (string, error) {
			189	+	_, err := io.WriteString(w, "aaaa")
			190	+	return "", err
			191	+	}, keepalive{})
83		if err != nil {	192		if err != nil {
84	-	t.Fatal(err)	193	+	t.Fatalf("waiter lost its entry: %v", err)
			194	+	}
			195	+	s.mu.Lock()
			196	+	pins := s.entries[key].pins
			197	+	s.mu.Unlock()
			198	+	if pins != 1 {
			199	+	t.Fatalf("pins %d: the commit pin was not released to the reader's", pins)
85		}	200		}
86	-	var calls atomic.Int32	201	+	_ = f.Close()
87	-	var wg sync.WaitGroup	202	+	s.hookCommitted = nil
88	-	start := make(chan struct{})	203	+	s.mu.Lock()
89	-	for range 20 {	204	+	s.evictLocked(s.max)
90	-	wg.Go(func() {	205	+	_, kept := s.entries[key]
91	-	<-start	206	+	s.mu.Unlock()
92	-	f, _, _, err := s.ensure(context.Background(), "blobs", sum("content"), func(_ context.Context, w io.Writer) (string, error) {	207	+	if kept {
93	-	calls.Add(1)	208	+	t.Fatal("unpinned entry not evictable")
94	-	_, err := io.WriteString(w, "content")
95	-	return "", err
96	-	}, keepalive{})
97	-	if err != nil {
98	-	t.Error(err)
99	-	return
100	-	}
101	-	_ = f.Close()
102	-	})
103	-	}
104	-	close(start)
105	-	wg.Wait()
106	-	if calls.Load() != 1 {
107	-	t.Fatalf("fetches %d", calls.Load())
108		}	209		}
109		}	210		}
110	-	func TestStoreOversize(t *testing.T) {	211	+
111	-	s, err := openStore(t.TempDir(), 3, time.Now)	212	+	// A blob that a response is still sending is pinned: a fill that needs its
112	-	if err != nil {	213	+	// room fails, and succeeds once the response has closed the file.
			214	+	func TestEvictionWhileServing(t *testing.T) {
			215	+	a, b := strings.Repeat("a", int(minCharge)), strings.Repeat("b", int(minCharge))
			216	+	srv := upstreamWith(t, func(w http.ResponseWriter, r *http.Request) {
			217	+	switch {
			218	+	case strings.HasSuffix(r.URL.Path, sum(a)):
			219	+	_, _ = io.WriteString(w, a)
			220	+	case strings.HasSuffix(r.URL.Path, sum(b)):
			221	+	_, _ = io.WriteString(w, b)
			222	+	default:
			223	+	http.NotFound(w, r)
			224	+	}
			225	+	})
			226	+	p := proxyWith(t, srv, Config{Max: minCharge * 3 / 2})
			227	+	if w := request(t, p, http.MethodGet, "/v2/library/alpine/blobs/"+sum(a)); w.Code != http.StatusOK {
			228	+	t.Fatal(w.Code)
			229	+	}
			230	+	bw := &blockingWriter{ResponseRecorder: httptest.NewRecorder(), started: make(chan struct{}), release: make(chan struct{})}
			231	+	done := make(chan struct{})
			232	+	go func() {
			233	+	defer close(done)
			234	+	p.ServeHTTP(bw, httptest.NewRequest(http.MethodGet, "/v2/library/alpine/blobs/"+sum(a), nil))
			235	+	}()
			236	+	<-bw.started
			237	+	if w := request(t, p, http.MethodGet, "/v2/library/alpine/blobs/"+sum(b)); w.Code != http.StatusBadGateway {
			238	+	t.Fatalf("fill evicted a blob being served: %d", w.Code)
			239	+	}
			240	+	if _, err := os.Stat(filepath.Join(p.s.dir, kindBlob.key(sum(a)[7:]))); err != nil {
			241	+	t.Fatalf("served blob removed: %v", err)
			242	+	}
			243	+	accounted(t, p.s)
			244	+	close(bw.release)
			245	+	<-done
			246	+	if bw.Code != http.StatusOK \|\| bw.Body.String() != a {
			247	+	t.Fatalf("served: %d", bw.Code)
			248	+	}
			249	+	if w := request(t, p, http.MethodGet, "/v2/library/alpine/blobs/"+sum(b)); w.Code != http.StatusOK \|\| w.Body.String() != b {
			250	+	t.Fatalf("after release: %d", w.Code)
			251	+	}
			252	+	if _, err := os.Stat(filepath.Join(p.s.dir, kindBlob.key(sum(a)[7:]))); !os.IsNotExist(err) {
			253	+	t.Fatal("released blob not evicted")
			254	+	}
			255	+	accounted(t, p.s)
			256	+	}
			257	+
			258	+	// Tag files share the LRU of the store, so they cannot grow the store past
			259	+	// Max; reopening charges them again.
			260	+	func TestTagsChargedAndEvicted(t *testing.T) {
			261	+	now := time.Unix(1_700_000_000, 0)
			262	+	dir := t.TempDir()
			263	+	c := storeCfg(dir, 4*minCharge) // the manifest and 3 tags
			264	+	c.now = func() time.Time { return now }
			265	+	s := mustOpen(t, c)
			266	+	putManifest(t, s, manifestBody, "application/vnd.oci.image.index.v1+json")
			267	+	for i := range 20 {
			268	+	now = now.Add(time.Second)
			269	+	if err := s.setTag("library/alpine", fmt.Sprint("t", i), "", sum(manifestBody)); err != nil {
			270	+	t.Fatalf("tag %d: %v", i, err)
			271	+	}
			272	+	if n := accounted(t, s); n > 3 {
			273	+	t.Fatalf("%d tags over a cap of 3", n)
			274	+	}
			275	+	}
			276	+	if s.tag("library/alpine", "t19", "") != sum(manifestBody) \|\| s.tag("library/alpine", "t0", "") != "" {
			277	+	t.Fatal("tags not evicted least recently used first")
			278	+	}
			279	+	s = mustOpen(t, storeCfg(dir, 4*minCharge))
			280	+	if n := accounted(t, s); n != 3 {
			281	+	t.Fatalf("reopened with %d tags", n)
			282	+	}
			283	+	}
			284	+
			285	+	// The tag index follows rewrites, and removing a manifest takes exactly its
			286	+	// own tags along.
			287	+	func TestTagIndex(t *testing.T) {
			288	+	s := mustOpen(t, storeCfg(t.TempDir(), 1<<20))
			289	+	manifests := []string{`{"schemaVersion":2,"a":1}`, `{"schemaVersion":2,"b":1}`}
			290	+	for _, m := range manifests {
			291	+	putManifest(t, s, m, "application/vnd.oci.image.index.v1+json")
			292	+	}
			293	+	for _, tag := range []string{"x", "y"} {
			294	+	if err := s.setTag("library/alpine", tag, "", sum(manifests[0])); err != nil {
			295	+	t.Fatal(err)
			296	+	}
			297	+	}
			298	+	// y moves to the second manifest.
			299	+	if err := s.setTag("library/alpine", "y", "", sum(manifests[1])); err != nil {
113		t.Fatal(err)	300		t.Fatal(err)
114		}	301		}
115	-	_, _, _, err = s.ensure(context.Background(), "blobs", sum("large"), func(_ context.Context, w io.Writer) (string, error) {	302	+	s.mu.Lock()
116	-	_, err := io.WriteString(w, "large")	303	+	if !s.removeLocked(kindManifest.key(sum(manifests[0])[7:])) {
117	-	return "", err	304	+	t.Fatal("remove failed")
118	-	}, keepalive{})	305	+	}
119	-	if err != errTooLarge {	306	+	n0, n1 := len(s.tagsOf[sum(manifests[0])]), len(s.tagsOf[sum(manifests[1])])
			307	+	s.mu.Unlock()
			308	+	if s.tag("library/alpine", "x", "") != "" \|\| s.tag("library/alpine", "y", "") != sum(manifests[1]) \|\| n0 != 0 \|\| n1 != 1 {
			309	+	t.Fatalf("index: %d %d", n0, n1)
			310	+	}
			311	+	accounted(t, s)
			312	+	}
			313	+
			314	+	// A .type file without its manifest is removed at startup.
			315	+	func TestOrphanTypeRemoved(t *testing.T) {
			316	+	dir := t.TempDir()
			317	+	mustOpen(t, storeCfg(dir, 1<<20))
			318	+	orphan := filepath.Join(dir, kindManifest.key(sum("gone")[7:])+".type")
			319	+	if err := os.WriteFile(orphan, []byte("application/json"), 0600); err != nil {
120		t.Fatal(err)	320		t.Fatal(err)
121		}	321		}
122	-	if s.pending != 0 \|\| s.total != 0 {	322	+	s := mustOpen(t, storeCfg(dir, 1<<20))
123	-	t.Fatalf("accounting: %d %d", s.pending, s.total)	323	+	if _, err := os.Stat(orphan); !os.IsNotExist(err) {
			324	+	t.Fatal("orphan .type kept")
124		}	325		}
			326	+	accounted(t, s)
125		}	327		}

M internal/registryproxy/upstream.go +35 -22

 	var network net.Error
 	return errors.As(err, &network) || errors.Is(err, context.DeadlineExceeded)
 }
 func proxyStatus(err error) int {
 	var e upstreamError
 	if errors.As(err, &e) && (e.status == http.StatusUnauthorized || e.status == http.StatusForbidden || e.status == http.StatusNotFound) {
 	value string
 	until time.Time
 }
 type upstream struct {
 	base, auth               string
 	client                   *http.Client
 	blobClient               *http.Client
 	requestTimeout, blobIdle time.Duration
-	now                      func() time.Time
 	mu                       sync.Mutex
 	tokens                   map[string]bearer
 }
 	u.mu.Lock()
 	t, ok := u.tokens[name]
 	u.mu.Unlock()
-	if ok && !refresh && u.now().Before(t.until) {
+	if ok && !refresh && time.Now().Before(t.until) {
 		return t.value, nil
 	}
 	a, err := url.Parse(u.auth)
 	if len(u.tokens) >= 1024 {
 		clear(u.tokens)
 	}
-	u.tokens[name] = bearer{data.Token, u.now().Add(time.Duration(data.Expires) * time.Second)}
+	u.tokens[name] = bearer{data.Token, time.Now().Add(time.Duration(data.Expires) * time.Second)}
 	return data.Token, nil
 }
-// request is a metadata request (manifests, blob HEAD) bounded by ctx. Manifests
+// authorized calls send with a bearer token for name, and once more with a
-// use the client that never follows redirects; a blob HEAD uses the blob client,
+// fresh token after a 401 or 403. Each token request is bounded by
-// because Docker Hub answers it with a redirect to its CDN.
+// RequestTimeout (and ctx). It returns a 200 response; any other closes the
-func (u *upstream) request(ctx context.Context, method, name, kind, ref, accept string) (*http.Response, error) {
+// body and is an upstreamError.
-	client := u.client
+func (u *upstream) authorized(ctx context.Context, name string, send func(tok string) (*http.Response, error)) (*http.Response, error) {
-	if kind == "blobs" {
-		client = u.blobClient
-	}
 	for attempt := 0; attempt < 2; attempt++ {
-		tok, err := u.token(ctx, name, attempt > 0)
+		authCtx, cancel := context.WithTimeout(ctx, u.requestTimeout)
+		tok, err := u.token(authCtx, name, attempt > 0)
+		cancel()
 		if err != nil {
 			return nil, err
 		}
-		req, err := http.NewRequestWithContext(ctx, method, u.base+"/v2/"+name+"/"+kind+"/"+ref, nil)
+		resp, err := send(tok)
-		if err != nil {
-			return nil, err
-		}
-		req.Header.Set("Authorization", "Bearer "+tok)
-		if accept != "" {
-			req.Header.Set("Accept", accept)
-		}
-		// Do not forward guest Authorization, cookies, query parameters or Range.
-		resp, err := client.Do(req)
 		if err != nil {
 			return nil, err
 		}
 	}
 	return nil, upstreamError{http.StatusUnauthorized}
 }
+// request is a metadata request (manifests, blob HEAD) bounded by ctx. Manifests
+// use the client that never follows redirects; a blob HEAD uses the blob client,
+// because Docker Hub answers it with a redirect to its CDN.
+func (u *upstream) request(ctx context.Context, method, name string, k kind, ref, accept string) (*http.Response, error) {
+	client := u.client
+	if k == kindBlob {
+		client = u.blobClient
+	}
+	return u.authorized(ctx, name, func(tok string) (*http.Response, error) {
+		req, err := http.NewRequestWithContext(ctx, method, u.base+"/v2/"+name+"/"+k.dir()+"/"+ref, nil)
+		if err != nil {
+			return nil, err
+		}
+		req.Header.Set("Authorization", "Bearer "+tok)
+		if accept != "" {
+			req.Header.Set("Accept", accept)
+		}
+		// Do not forward guest Authorization, cookies, query parameters or Range.
+		return client.Do(req)
+	})
+}

A internal/registryproxy/upstream_test.go +172 -0

@@ -0,0 +1,172 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package registryproxy
	4	+
	5	+	import (
	6	+	"fmt"
	7	+	"io"
	8	+	"net/http"
	9	+	"net/http/httptest"
	10	+	"os"
	11	+	"path/filepath"
	12	+	"strings"
	13	+	"sync/atomic"
	14	+	"testing"
	15	+	"time"
	16	+	)
	17	+
	18	+	// A blob body has no total deadline: a transfer that keeps moving may last
	19	+	// far longer than RequestTimeout and BlobIdleTimeout.
	20	+	func TestBlobTransferOutlivesRequestTimeout(t *testing.T) {
	21	+	body := strings.Repeat("layer", 4000)
	22	+	srv := upstreamWith(t, slowBlob(body, 20, 25*time.Millisecond))
	23	+	p := proxyWith(t, srv, Config{RequestTimeout: 100 * time.Millisecond, BlobIdleTimeout: 150 * time.Millisecond})
	24	+	start := time.Now()
	25	+	w := request(t, p, http.MethodGet, "/v2/library/alpine/blobs/"+sum(body))
	26	+	if w.Code != http.StatusOK \|\| w.Body.String() != body {
	27	+	t.Fatalf("blob: %d", w.Code)
	28	+	}
	29	+	if d := time.Since(start); d < 300*time.Millisecond {
	30	+	t.Fatalf("transfer took %v, not longer than the timeouts", d)
	31	+	}
	32	+	}
	33	+
	34	+	// A blob body that stops moving is cut after BlobIdleTimeout, and auth,
	35	+	// manifests and HEAD keep their total RequestTimeout.
	36	+	func TestStalledUpstreamTimesOut(t *testing.T) {
	37	+	body := strings.Repeat("layer", 4000)
	38	+	srv := upstreamWith(t, func(w http.ResponseWriter, r *http.Request) {
	39	+	if strings.Contains(r.URL.Path, "/blobs/") && r.Method == http.MethodGet {
	40	+	w.Header().Set("Content-Length", fmt.Sprint(len(body)))
	41	+	_, _ = io.WriteString(w, body[:100])
	42	+	w.(http.Flusher).Flush()
	43	+	}
	44	+	select { // a stalled body, or headers that never come
	45	+	case <-r.Context().Done():
	46	+	case <-time.After(5 * time.Second):
	47	+	}
	48	+	})
	49	+	p := proxyWith(t, srv, Config{RequestTimeout: 100 * time.Millisecond, BlobIdleTimeout: 150 * time.Millisecond})
	50	+	for _, tc := range []struct{ method, path string }{
	51	+	{http.MethodGet, "/v2/library/alpine/blobs/" + sum(body)},
	52	+	{http.MethodHead, "/v2/library/alpine/blobs/" + sum(body)},
	53	+	{http.MethodGet, "/v2/library/alpine/manifests/3"},
	54	+	} {
	55	+	start := time.Now()
	56	+	if w := request(t, p, tc.method, tc.path); w.Code != http.StatusBadGateway {
	57	+	t.Errorf("%s %s: %d", tc.method, tc.path, w.Code)
	58	+	}
	59	+	if d := time.Since(start); d > 2*time.Second {
	60	+	t.Errorf("%s %s: cut after %v", tc.method, tc.path, d)
	61	+	}
	62	+	}
	63	+	if _, err := os.Stat(filepath.Join(p.s.dir, kindBlob.key(sum(body)[7:]))); !os.IsNotExist(err) {
	64	+	t.Fatal("partial blob stored")
	65	+	}
	66	+	}
	67	+
	68	+	// Auth and manifest requests never follow a redirect.
	69	+	func TestMetadataRedirectsNotFollowed(t *testing.T) {
	70	+	var elsewhere atomic.Int32
	71	+	other := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	72	+	elsewhere.Add(1)
	73	+	_, _ = io.WriteString(w, `{"token":"public"}`)
	74	+	}))
	75	+	t.Cleanup(other.Close)
	76	+	for _, redirectAuth := range []bool{true, false} {
	77	+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	78	+	if r.URL.Path == "/token" && !redirectAuth {
	79	+	_, _ = io.WriteString(w, `{"token":"public"}`)
	80	+	return
	81	+	}
	82	+	http.Redirect(w, r, other.URL+r.URL.Path, http.StatusTemporaryRedirect)
	83	+	}))
	84	+	p := proxyWith(t, srv, Config{})
	85	+	for _, ref := range []string{"3", sum(manifestBody)} {
	86	+	if w := request(t, p, http.MethodGet, "/v2/library/alpine/manifests/"+ref); w.Code != http.StatusBadGateway {
	87	+	t.Errorf("auth redirect %v, ref %s: %d", redirectAuth, ref, w.Code)
	88	+	}
	89	+	}
	90	+	srv.Close()
	91	+	}
	92	+	if elsewhere.Load() != 0 {
	93	+	t.Fatalf("%d redirects followed", elsewhere.Load())
	94	+	}
	95	+	}
	96	+
	97	+	// Blob requests, HEAD included, follow the CDN redirect without the bearer
	98	+	// token.
	99	+	func TestBlobRedirectStripsBearer(t *testing.T) {
	100	+	var cdnHits atomic.Int32
	101	+	cdn := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	102	+	cdnHits.Add(1)
	103	+	if r.Header.Get("Authorization") != "" {
	104	+	t.Error("bearer sent to the CDN")
	105	+	}
	106	+	w.Header().Set("Content-Length", fmt.Sprint(len(blobBody)))
	107	+	if r.Method == http.MethodGet {
	108	+	_, _ = io.WriteString(w, blobBody)
	109	+	}
	110	+	}))
	111	+	t.Cleanup(cdn.Close)
	112	+	srv := upstreamWith(t, func(w http.ResponseWriter, r *http.Request) {
	113	+	if r.Header.Get("Authorization") != "Bearer public" {
	114	+	t.Error("no bearer on the registry")
	115	+	}
	116	+	http.Redirect(w, r, cdn.URL+"/layer", http.StatusTemporaryRedirect)
	117	+	})
	118	+	p := proxyWith(t, srv, Config{})
	119	+	path := "/v2/library/alpine/blobs/" + sum(blobBody)
	120	+	if w := request(t, p, http.MethodHead, path); w.Code != http.StatusOK \|\| w.Header().Get("Content-Length") != fmt.Sprint(len(blobBody)) {
	121	+	t.Fatalf("HEAD: %d %v", w.Code, w.Header())
	122	+	}
	123	+	if w := request(t, p, http.MethodGet, path); w.Code != http.StatusOK \|\| w.Body.String() != blobBody {
	124	+	t.Fatalf("blob: %d", w.Code)
	125	+	}
	126	+	if cdnHits.Load() != 2 {
	127	+	t.Fatalf("CDN hits %d", cdnHits.Load())
	128	+	}
	129	+	}
	130	+
	131	+	// A 401 or 403 is retried once with a fresh token; a second one is a 404, so
	132	+	// that dockerd falls back to Docker Hub with the job's credentials.
	133	+	func TestUnauthorizedRetriedOnce(t *testing.T) {
	134	+	for _, path := range []string{"/v2/library/alpine/manifests/3", "/v2/library/alpine/blobs/" + sum(blobBody)} {
	135	+	f := fake(t)
	136	+	p := proxyFor(t, f)
	137	+	f.status.Store(http.StatusUnauthorized)
	138	+	if w := request(t, p, http.MethodGet, path); w.Code != http.StatusNotFound {
	139	+	t.Fatalf("%s: %d", path, w.Code)
	140	+	}
	141	+	if f.tokens.Load() != 2 {
	142	+	t.Fatalf("%s: %d token requests, want a fresh one", path, f.tokens.Load())
	143	+	}
	144	+	}
	145	+	}
	146	+
	147	+	// When upstream is down, fails or rate-limits (429), a known tag is served
	148	+	// stale; an unknown one is an error.
	149	+	func TestStaleTagWhenUpstreamUnavailable(t *testing.T) {
	150	+	for _, tc := range []struct {
	151	+	name string
	152	+	status int32 // 0: the server is closed
	153	+	}{{"rate limited", http.StatusTooManyRequests}, {"failing", http.StatusServiceUnavailable}, {"down", 0}} {
	154	+	t.Run(tc.name, func(t *testing.T) {
	155	+	f := fake(t)
	156	+	p := proxyFor(t, f)
	157	+	if w := request(t, p, http.MethodGet, "/v2/library/alpine/manifests/3"); w.Code != http.StatusOK {
	158	+	t.Fatal(w.Code)
	159	+	}
	160	+	if tc.status == 0 {
	161	+	f.server.Close()
	162	+	}
	163	+	f.status.Store(tc.status)
	164	+	if w := request(t, p, http.MethodGet, "/v2/library/alpine/manifests/3"); w.Code != http.StatusOK \|\| w.Body.String() != manifestBody {
	165	+	t.Fatalf("stale: %d %s", w.Code, w.Body)
	166	+	}
	167	+	if w := request(t, p, http.MethodGet, "/v2/library/alpine/manifests/unknown"); w.Code != http.StatusBadGateway {
	168	+	t.Fatalf("unknown: %d", w.Code)
	169	+	}
	170	+	})
	171	+	}
	172	+	}

M internal/sandbox/container/driver.go +4 -10

 	"context"
 	"errors"
 	"fmt"
+	"maps"
 	"net"
 	"net/netip"
 	"os"
 	attached  bool
 }
-// StartTimings returns a copy of the successful start's phase durations.
+// StartTimings returns a copy of the phase durations of a successful Start.
-func (s *containerSandbox) StartTimings() map[string]int64 {
+// It is valid once Start has returned, which writes them.
-	s.mu.Lock()
+func (s *containerSandbox) StartTimings() map[string]int64 { return maps.Clone(s.timings) }
-	defer s.mu.Unlock()
-	out := make(map[string]int64, len(s.timings))
-	for k, v := range s.timings {
-		out[k] = v
-	}
-	return out
-}
 // ID implements sandbox.Sandbox.
 func (s *containerSandbox) ID() string { return s.id }

M internal/sandbox/sandbox.go +2 -0


@@ -91,6 +91,8 @@ type Sandbox interface {
91		91
92	// Timed is implemented by sandboxes that measure their start phases.	92		// Timed is implemented by sandboxes that measure their start phases.
93	type Timed interface {	93		type Timed interface {
		94	+	// StartTimings returns a copy of the phase durations (ms) of a successful
		95	+	// Start, nil after a failed one. Call it only once Start has returned.
94	StartTimings() map[string]int64	96		StartTimings() map[string]int64
95	}	97		}
96		98

M internal/sandbox/vm/driver.go +4 -10

 	"errors"
 	"fmt"
 	"io/fs"
+	"maps"
 	"net"
 	"net/netip"
 	"os"
 	attached bool
 }
-// StartTimings returns a copy of the successful start's phase durations.
+// StartTimings returns a copy of the phase durations of a successful Start.
-func (s *vmSandbox) StartTimings() map[string]int64 {
+// It is valid once Start has returned, which writes them.
-	s.mu.Lock()
+func (s *vmSandbox) StartTimings() map[string]int64 { return maps.Clone(s.timings) }
-	defer s.mu.Unlock()
-	out := make(map[string]int64, len(s.timings))
-	for k, v := range s.timings {
-		out[k] = v
-	}
-	return out
-}
 // ID implements sandbox.Sandbox.
 func (s *vmSandbox) ID() string { return s.id }

M registry_setup_test.go +49 -17

 // SPDX-License-Identifier: MIT
 package main
 import (
-	"bnema.dev/ermite-runner/internal/sandbox"
 	"context"
+	"strings"
 	"testing"
+	"bnema.dev/ermite-runner/internal/actionscache"
+	"bnema.dev/ermite-runner/internal/cachesvc"
+	"bnema.dev/ermite-runner/internal/sandbox"
 )
-func TestRegistryConfiguration(t *testing.T) {
+// The three size caps share one parser: a zero size in any unit turns the
+// registry proxy and the action archive cache off, and is refused for the
+// cache API, which ERMITE_RUNNER_CACHE=0 turns off. Each error names its
+// variable.
+func TestCacheSizeConfiguration(t *testing.T) {
 	t.Setenv(envCache, "1")
-	t.Setenv("ERMITE_RUNNER_ACTIONS_CACHE_MAX", "0")
 	for _, tc := range []struct {
-		registry, cache string
+		registry, actions, cache string
-		on              bool
+		registryOn, archivesOn   bool
-	}{{"", "1M", true}, {"1M", "2M", true}, {"0", "1M", false}, {"0G", "1M", false}, {" 0M ", "1M", false}, {"00", "1M", false}, {"0GiB", "1M", false}, {"10M", "1M", true}} {
+	}{
+		{"", "", "1M", true, true},
+		{"1M", "0", "2M", true, false},
+		{"0", "1M", "1M", false, true},
+		{"0G", "0GiB", "1M", false, false},
+		{" 0M ", "", "1M", false, true},
+		{"00", "", "1M", false, true},
+		{"0GiB", "", "1M", false, true},
+		{"10M", "", "1M", true, true},
+	} {
 		t.Setenv(envRegistryMax, tc.registry)
-		t.Setenv("ERMITE_RUNNER_CACHE_MAX", tc.cache)
+		t.Setenv(actionscache.EnvMax, tc.actions)
+		t.Setenv(cachesvc.EnvMax, tc.cache)
 		cfg := sandbox.Config{StateDir: t.TempDir()}
-		svc, _, err := setupCache(context.Background(), &cfg)
+		svc, archives, err := setupCache(context.Background(), &cfg)
 		if err != nil {
-			t.Fatal(err)
+			t.Fatalf("%+v: %v", tc, err)
 		}
-		if svc.RegistryEnabled() != tc.on {
+		if svc.RegistryEnabled() != tc.registryOn || (archives != nil) != tc.archivesOn {
-			t.Fatalf("%+v: registry=%v", tc, svc.RegistryEnabled())
+			t.Fatalf("%+v: registry=%v archives=%v", tc, svc.RegistryEnabled(), archives != nil)
 		}
 		_ = svc.CloseAll()
 	}
-	for _, bad := range []string{"invalid", "G", "-1G"} {
+	t.Setenv(envRegistryMax, "")
-		t.Setenv(envRegistryMax, bad)
+	t.Setenv(actionscache.EnvMax, "")
-		cfg := sandbox.Config{StateDir: t.TempDir()}
+	t.Setenv(cachesvc.EnvMax, "")
-		if _, _, err := setupCache(context.Background(), &cfg); err == nil {
+	for _, tc := range []struct{ env, value string }{
-			t.Fatalf("%q accepted", bad)
+		{envRegistryMax, "invalid"}, {envRegistryMax, "G"}, {envRegistryMax, "-1G"},
-		}
+		{actionscache.EnvMax, "x"},
+		{cachesvc.EnvMax, "0"}, {cachesvc.EnvMax, "0G"}, {cachesvc.EnvMax, "-1"},
+	} {
+		t.Run(tc.env+"="+tc.value, func(t *testing.T) {
+			t.Setenv(tc.env, tc.value)
+			cfg := sandbox.Config{StateDir: t.TempDir()}
+			svc, _, err := setupCache(context.Background(), &cfg)
+			if err == nil {
+				_ = svc.CloseAll()
+				t.Fatal("accepted")
+			}
+			if !strings.HasPrefix(err.Error(), tc.env+": ") {
+				t.Fatalf("error does not name %s: %v", tc.env, err)
+			}
+		})
 	}
 	t.Setenv(envCache, "0")
 	t.Setenv(envRegistryMax, "invalid")