feat(registryproxy): cache Docker Hub pulls of the sandboxes on the runner host

commit: 57617df7e2e0aba70ddd201c05642bd937d7a47d
author: brice · 28h ago · 2026-10-21 08:41 UTC
committer: brice · 28h ago · 2026-10-21 08:41 UTC
parent: 09df1ab
signature: GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email

29 files · +3727 -17

M docs/cache-and-artifacts.md +53 -0


@@ -79,3 +79,56 @@ Works with `actions/cache`, `setup-go` and `setup-node` with `cache:`, and anyth
79	79		- Artifacts v3.
80	80		- The `artifact-url` output. It points at the fake host. The run page in ermite lists the artifacts.
81	81		- Downloads from another run (`run-id` with `github-token`).
	82	+
	83	+	## Docker Hub images
	84	+
	85	+	The runner provides an anonymous, pull-only Docker Hub mirror on the same private
	86	+	channel. Each job's dockerd uses `http://127.0.0.1:9089` as a registry mirror.
	87	+	Other registries are not cached; there is no HTTPS interception or guest network
	88	+	policy change. Guest credentials are never forwarded upstream. Private pulls
	89	+	fall back to Docker Hub directly, where dockerd can use the job's credentials.
	90	+
	91	+	Verified sha256 blobs and manifests live under `<state>/registry/`. Tags are
	92	+	revalidated by an upstream HEAD on each request, with the client's Accept headers
	93	+	reduced to the known manifest media types. If Docker Hub is unreachable, times
	94	+	out, returns a server error or rate-limits (429), a previously cached tag can be
	95	+	served stale while its manifest remains stored. Unknown tags return an error so
	96	+	dockerd can fall back.
	97	+
	98	+	A cold blob is downloaded once, whoever asks for it, and is not tied to any one
	99	+	client: every client waits for the digest check and then reads the stored file.
	100	+	A corrupt download returns 502 and leaves no file. A client that disconnects
	101	+	does not stop the download while another client waits for it; once no client
	102	+	has waited for 5 s, the download is cancelled and its temporary file removed.
	103	+	Each job may have started at most 3 cold blob downloads at once (dockerd's
	104	+	default concurrency), and at most 16 run at once across all jobs. A download
	105	+	counts against the job that started it until it ends; a job that asks for a
	106	+	layer another job is already downloading waits for it and uses none of its
	107	+	quota. A request over either limit waits up to 30 s for a slot, its connection
	108	+	kept alive, and then gets 503. A refused layer makes dockerd retry for about
	109	+	50 s before it pulls that layer from Docker Hub directly. Each sandbox may also
	110	+	have at most 8 registry requests in progress; one more gets 503 at once.
	111	+
	112	+	Downloads have no total time limit, but they fail after 30 s without progress.
	113	+	Auth, manifest and HEAD requests are limited to 10 s. Auth and manifest requests
	114	+	never follow redirects; blob requests follow the CDN redirect without the bearer
	115	+	token. Stored blobs support range requests. Cold blob HEAD requests fetch only
	116	+	upstream metadata, not the layer body.
	117	+
	118	+	`ERMITE_RUNNER_REGISTRY_CACHE_MAX` defaults to `10G`. Blobs, manifests and tag
	119	+	files (charged 4 KiB each) share one least-recently-used eviction; an entry that
	120	+	is being served is never evicted. This cap is independent of
	121	+	`ERMITE_RUNNER_CACHE_MAX`. A zero size in any unit (`0`, `0M`, `0G`) disables only
	122	+	the proxy. `ERMITE_RUNNER_CACHE=0` disables the whole channel and therefore also
	123	+	the proxy (logged at startup). Dockerd retains its mirror setting and falls back
	124	+	to Docker Hub if the mirror is unavailable. No deployment is needed to clear
	125	+	entries: stop the runner before removing `<state>/registry/`.
	126	+
	127	+	Jobs with neither cache permissions nor artifacts still get the loopback channel
	128	+	for dockerd, but no `ACTIONS_CACHE_URL`, `ACTIONS_RESULTS_URL`, or
	129	+	`ACTIONS_RUNTIME_TOKEN` is exposed to their steps. Registry routes need no token:
	130	+	only public content is served over a channel exclusive to that sandbox.
	131	+
	132	+	The host logs one `registry manifest` line per request with `repo`, `ref`, `hit`
	133	+	and `ms`. Blob misses log `registry blob` at debug level with `digest`, `bytes`
	134	+	and `ms`.


@@ -79,3 +79,56 @@ Works with `actions/cache`, `setup-go` and `setup-node` with `cache:`, and anyth
79	- Artifacts v3.	79		- Artifacts v3.
80	- The `artifact-url` output. It points at the fake host. The run page in ermite lists the artifacts.	80		- The `artifact-url` output. It points at the fake host. The run page in ermite lists the artifacts.
81	- Downloads from another run (`run-id` with `github-token`).	81		- Downloads from another run (`run-id` with `github-token`).
		82	+
		83	+	## Docker Hub images
		84	+
		85	+	The runner provides an anonymous, pull-only Docker Hub mirror on the same private
		86	+	channel. Each job's dockerd uses `http://127.0.0.1:9089` as a registry mirror.
		87	+	Other registries are not cached; there is no HTTPS interception or guest network
		88	+	policy change. Guest credentials are never forwarded upstream. Private pulls
		89	+	fall back to Docker Hub directly, where dockerd can use the job's credentials.
		90	+
		91	+	Verified sha256 blobs and manifests live under `<state>/registry/`. Tags are
		92	+	revalidated by an upstream HEAD on each request, with the client's Accept headers
		93	+	reduced to the known manifest media types. If Docker Hub is unreachable, times
		94	+	out, returns a server error or rate-limits (429), a previously cached tag can be
		95	+	served stale while its manifest remains stored. Unknown tags return an error so
		96	+	dockerd can fall back.
		97	+
		98	+	A cold blob is downloaded once, whoever asks for it, and is not tied to any one
		99	+	client: every client waits for the digest check and then reads the stored file.
		100	+	A corrupt download returns 502 and leaves no file. A client that disconnects
		101	+	does not stop the download while another client waits for it; once no client
		102	+	has waited for 5 s, the download is cancelled and its temporary file removed.
		103	+	Each job may have started at most 3 cold blob downloads at once (dockerd's
		104	+	default concurrency), and at most 16 run at once across all jobs. A download
		105	+	counts against the job that started it until it ends; a job that asks for a
		106	+	layer another job is already downloading waits for it and uses none of its
		107	+	quota. A request over either limit waits up to 30 s for a slot, its connection
		108	+	kept alive, and then gets 503. A refused layer makes dockerd retry for about
		109	+	50 s before it pulls that layer from Docker Hub directly. Each sandbox may also
		110	+	have at most 8 registry requests in progress; one more gets 503 at once.
		111	+
		112	+	Downloads have no total time limit, but they fail after 30 s without progress.
		113	+	Auth, manifest and HEAD requests are limited to 10 s. Auth and manifest requests
		114	+	never follow redirects; blob requests follow the CDN redirect without the bearer
		115	+	token. Stored blobs support range requests. Cold blob HEAD requests fetch only
		116	+	upstream metadata, not the layer body.
		117	+
		118	+	`ERMITE_RUNNER_REGISTRY_CACHE_MAX` defaults to `10G`. Blobs, manifests and tag
		119	+	files (charged 4 KiB each) share one least-recently-used eviction; an entry that
		120	+	is being served is never evicted. This cap is independent of
		121	+	`ERMITE_RUNNER_CACHE_MAX`. A zero size in any unit (`0`, `0M`, `0G`) disables only
		122	+	the proxy. `ERMITE_RUNNER_CACHE=0` disables the whole channel and therefore also
		123	+	the proxy (logged at startup). Dockerd retains its mirror setting and falls back
		124	+	to Docker Hub if the mirror is unavailable. No deployment is needed to clear
		125	+	entries: stop the runner before removing `<state>/registry/`.
		126	+
		127	+	Jobs with neither cache permissions nor artifacts still get the loopback channel
		128	+	for dockerd, but no `ACTIONS_CACHE_URL`, `ACTIONS_RESULTS_URL`, or
		129	+	`ACTIONS_RUNTIME_TOKEN` is exposed to their steps. Registry routes need no token:
		130	+	only public content is served over a channel exclusive to that sandbox.
		131	+
		132	+	The host logs one `registry manifest` line per request with `repo`, `ref`, `hit`
		133	+	and `ms`. Blob misses log `registry blob` at debug level with `digest`, `bytes`
		134	+	and `ms`.

M docs/configuration.md +2 -1


@@ -74,9 +74,10 @@ The token has the scope `runner:register`. It can read the code and the secrets
74	74
75	75		\| Variable \| Default \| Meaning \|
76	76		\| --- \| --- \| --- \|
77		-	\| `ERMITE_RUNNER_CACHE` \| `1` \| `0` turns the Actions cache off. See [cache and artifacts](cache-and-artifacts.md) \|
	77	+	\| `ERMITE_RUNNER_CACHE` \| `1` \| `0` turns the cache channel off, including the Actions cache and Docker Hub proxy. See [cache and artifacts](cache-and-artifacts.md) \|
78	78		\| `ERMITE_RUNNER_CACHE_MAX` \| `10G` \| Size cap of the cache. Least recently used entries are evicted \|
79	79		\| `ERMITE_RUNNER_ACTIONS_CACHE_MAX` \| `2G` \| Size cap of the action archive cache. `0` turns it off \|
	80	+	\| `ERMITE_RUNNER_REGISTRY_CACHE_MAX` \| `10G` \| Docker Hub blob and manifest cache cap, independent of `ERMITE_RUNNER_CACHE_MAX`. A zero size (`0`, `0M`, `0G`) turns only the proxy off; `ERMITE_RUNNER_CACHE=0` also disables it \|
80	81		\| `ERMITE_RUNNER_ARTIFACTS` \| `1` \| `0` turns artifacts v4 off. Artifacts need the cache service \|
81	82		\| `ERMITE_RUNNER_NODE` \| `1` \| `0` turns the Node.js externals off. See [externals](externals.md) \|
82	83		\| `ERMITE_RUNNER_GUEST_FORGE` \| `0` \| `1` lets sandboxes reach the server host and port. Read the security notes first. See [sandboxes](sandboxes.md#guest-forge-access) \|

 | Variable | Default | Meaning |
 | --- | --- | --- |
-| `ERMITE_RUNNER_CACHE` | `1` | `0` turns the Actions cache off. See [cache and artifacts](cache-and-artifacts.md) |
+| `ERMITE_RUNNER_CACHE` | `1` | `0` turns the cache channel off, including the Actions cache and Docker Hub proxy. See [cache and artifacts](cache-and-artifacts.md) |
 | `ERMITE_RUNNER_CACHE_MAX` | `10G` | Size cap of the cache. Least recently used entries are evicted |
 | `ERMITE_RUNNER_ACTIONS_CACHE_MAX` | `2G` | Size cap of the action archive cache. `0` turns it off |
+| `ERMITE_RUNNER_REGISTRY_CACHE_MAX` | `10G` | Docker Hub blob and manifest cache cap, independent of `ERMITE_RUNNER_CACHE_MAX`. A zero size (`0`, `0M`, `0G`) turns only the proxy off; `ERMITE_RUNNER_CACHE=0` also disables it |
 | `ERMITE_RUNNER_ARTIFACTS` | `1` | `0` turns artifacts v4 off. Artifacts need the cache service |
 | `ERMITE_RUNNER_NODE` | `1` | `0` turns the Node.js externals off. See [externals](externals.md) |
 | `ERMITE_RUNNER_GUEST_FORGE` | `0` | `1` lets sandboxes reach the server host and port. Read the security notes first. See [sandboxes](sandboxes.md#guest-forge-access) |

M host_cmd.go +41 -2


@@ -8,6 +8,7 @@ import (
8	8		"flag"
9	9		"fmt"
10	10		"net"
	11	+	"net/http"
11	12		"os"
12	13		"path/filepath"
13	14		"strings"

@@ -21,6 +22,7 @@ import (
21	22		"bnema.dev/ermite-runner/internal/cachesvc"
22	23		"bnema.dev/ermite-runner/internal/externals"
23	24		"bnema.dev/ermite-runner/internal/host"
	25	+	"bnema.dev/ermite-runner/internal/registryproxy"
24	26		"bnema.dev/ermite-runner/internal/resultssvc"
25	27		"bnema.dev/ermite-runner/internal/sandbox"
26	28		"bnema.dev/ermite-runner/internal/sandbox/drivers"

@@ -46,7 +48,7 @@ func setupCache(ctx context.Context, scfg sandbox.Config) (cachesvc.Service, *
46	48		switch strings.ToLower(strings.TrimSpace(os.Getenv(envCache))) {
47	49		case "", "1", "true", "yes":
48	50		case "0", "false", "no":
49		-	log.Info().Msg("the cache service is off (" + envCache + "=0)")
	51	+	log.Info().Msg("the cache service and registry proxy are off (" + envCache + "=0)")
50	52		return nil, nil, nil
51	53		default:
52	54		return nil, nil, fmt.Errorf("%s: %q is not 0 or 1", envCache, os.Getenv(envCache))

@@ -59,7 +61,11 @@ func setupCache(ctx context.Context, scfg sandbox.Config) (cachesvc.Service, *
59	61		if err != nil {
60	62		return nil, nil, err
61	63		}
62		-	svc, err := cachesvc.New(cachesvc.Config{Dir: filepath.Join(scfg.StateDir, "cache"), Max: max, Log: log, Archives: archives})
	64	+	registry, err := setupRegistry(ctx, scfg.StateDir)
	65	+	if err != nil {
	66	+	return nil, nil, err
	67	+	}
	68	+	svc, err := cachesvc.New(cachesvc.Config{Dir: filepath.Join(scfg.StateDir, "cache"), Max: max, Log: log, Archives: archives, Registry: registry})
63	69		if err != nil {
64	70		return nil, nil, err
65	71		}

@@ -89,6 +95,38 @@ func setupArchives(ctx context.Context, stateDir string) (*actionscache.Cache, e
89	95		return c, nil
90	96		}
91	97
	98	+	const envRegistryMax = "ERMITE_RUNNER_REGISTRY_CACHE_MAX"
	99	+
	100	+	// setupRegistry opens the Docker Hub mirror. The caller only calls this when
	101	+	// the cache channel is enabled; its capacity is independent of the cache API.
	102	+	func setupRegistry(ctx context.Context, stateDir string) (http.Handler, error) {
	103	+	log := zerowrap.FromCtx(ctx)
	104	+	value := strings.TrimSpace(os.Getenv(envRegistryMax))
	105	+	if zeroSize(value) {
	106	+	log.Info().Msg("registry proxy off (" + envRegistryMax + "=" + value + ")")
	107	+	return nil, nil
	108	+	}
	109	+	if value == "" {
	110	+	value = "10G"
	111	+	}
	112	+	max, err := cachesvc.ParseMax(value)
	113	+	if err != nil {
	114	+	return nil, fmt.Errorf("%s: %w", envRegistryMax, err)
	115	+	}
	116	+	p, err := registryproxy.New(registryproxy.Config{Dir: filepath.Join(stateDir, "registry"), Max: max, Log: log})
	117	+	if err != nil {
	118	+	return nil, err
	119	+	}
	120	+	log.Info().Int64("max_bytes", max).Msg("registry proxy on (Docker Hub pull-through)")
	121	+	return p, nil
	122	+	}
	123	+
	124	+	// zeroSize reports a size of zero in any unit: 0, 00, 0M, 0G, 0GiB...
	125	+	func zeroSize(v string) bool {
	126	+	digits := strings.TrimRight(v, "kKmMgGtTiIbB")
	127	+	return digits != "" && strings.Trim(digits, "0") == ""
	128	+	}
	129	+
92	130		// envCache=0 turns the cache service off.
93	131		const envCache = "ERMITE_RUNNER_CACHE"
94	132

@@ -307,6 +345,7 @@ func runCmd(ctx context.Context) int {
307	345		},
308	346		Features: hcfg.ServerFeaturesFunc(url),
309	347		Cache: cache,
	348	+	Registry: cache != nil && cache.RegistryEnabled(),
310	349		Archives: archives,
311	350		Results: results,
312	351		Artifacts: func(tok string) ermiteconnect.JobArtifactServiceClient {


@@ -8,6 +8,7 @@ import (
8		"flag"	8		"flag"
9		"fmt"	9		"fmt"
10		"net"	10		"net"
			11	+	"net/http"
11		"os"	12		"os"
12		"path/filepath"	13		"path/filepath"
13		"strings"	14		"strings"

@@ -21,6 +22,7 @@ import (
21		"bnema.dev/ermite-runner/internal/cachesvc"	22		"bnema.dev/ermite-runner/internal/cachesvc"
22		"bnema.dev/ermite-runner/internal/externals"	23		"bnema.dev/ermite-runner/internal/externals"
23		"bnema.dev/ermite-runner/internal/host"	24		"bnema.dev/ermite-runner/internal/host"
			25	+	"bnema.dev/ermite-runner/internal/registryproxy"
24		"bnema.dev/ermite-runner/internal/resultssvc"	26		"bnema.dev/ermite-runner/internal/resultssvc"
25		"bnema.dev/ermite-runner/internal/sandbox"	27		"bnema.dev/ermite-runner/internal/sandbox"
26		"bnema.dev/ermite-runner/internal/sandbox/drivers"	28		"bnema.dev/ermite-runner/internal/sandbox/drivers"

@@ -46,7 +48,7 @@ func setupCache(ctx context.Context, scfg sandbox.Config) (cachesvc.Service, *
46		switch strings.ToLower(strings.TrimSpace(os.Getenv(envCache))) {	48		switch strings.ToLower(strings.TrimSpace(os.Getenv(envCache))) {
47		case "", "1", "true", "yes":	49		case "", "1", "true", "yes":
48		case "0", "false", "no":	50		case "0", "false", "no":
49	-	log.Info().Msg("the cache service is off (" + envCache + "=0)")	51	+	log.Info().Msg("the cache service and registry proxy are off (" + envCache + "=0)")
50		return nil, nil, nil	52		return nil, nil, nil
51		default:	53		default:
52		return nil, nil, fmt.Errorf("%s: %q is not 0 or 1", envCache, os.Getenv(envCache))	54		return nil, nil, fmt.Errorf("%s: %q is not 0 or 1", envCache, os.Getenv(envCache))

@@ -59,7 +61,11 @@ func setupCache(ctx context.Context, scfg sandbox.Config) (cachesvc.Service, *
59		if err != nil {	61		if err != nil {
60		return nil, nil, err	62		return nil, nil, err
61		}	63		}
62	-	svc, err := cachesvc.New(cachesvc.Config{Dir: filepath.Join(scfg.StateDir, "cache"), Max: max, Log: log, Archives: archives})	64	+	registry, err := setupRegistry(ctx, scfg.StateDir)
			65	+	if err != nil {
			66	+	return nil, nil, err
			67	+	}
			68	+	svc, err := cachesvc.New(cachesvc.Config{Dir: filepath.Join(scfg.StateDir, "cache"), Max: max, Log: log, Archives: archives, Registry: registry})
63		if err != nil {	69		if err != nil {
64		return nil, nil, err	70		return nil, nil, err
65		}	71		}

@@ -89,6 +95,38 @@ func setupArchives(ctx context.Context, stateDir string) (*actionscache.Cache, e
89		return c, nil	95		return c, nil
90		}	96		}
91			97
			98	+	const envRegistryMax = "ERMITE_RUNNER_REGISTRY_CACHE_MAX"
			99	+
			100	+	// setupRegistry opens the Docker Hub mirror. The caller only calls this when
			101	+	// the cache channel is enabled; its capacity is independent of the cache API.
			102	+	func setupRegistry(ctx context.Context, stateDir string) (http.Handler, error) {
			103	+	log := zerowrap.FromCtx(ctx)
			104	+	value := strings.TrimSpace(os.Getenv(envRegistryMax))
			105	+	if zeroSize(value) {
			106	+	log.Info().Msg("registry proxy off (" + envRegistryMax + "=" + value + ")")
			107	+	return nil, nil
			108	+	}
			109	+	if value == "" {
			110	+	value = "10G"
			111	+	}
			112	+	max, err := cachesvc.ParseMax(value)
			113	+	if err != nil {
			114	+	return nil, fmt.Errorf("%s: %w", envRegistryMax, err)
			115	+	}
			116	+	p, err := registryproxy.New(registryproxy.Config{Dir: filepath.Join(stateDir, "registry"), Max: max, Log: log})
			117	+	if err != nil {
			118	+	return nil, err
			119	+	}
			120	+	log.Info().Int64("max_bytes", max).Msg("registry proxy on (Docker Hub pull-through)")
			121	+	return p, nil
			122	+	}
			123	+
			124	+	// zeroSize reports a size of zero in any unit: 0, 00, 0M, 0G, 0GiB...
			125	+	func zeroSize(v string) bool {
			126	+	digits := strings.TrimRight(v, "kKmMgGtTiIbB")
			127	+	return digits != "" && strings.Trim(digits, "0") == ""
			128	+	}
			129	+
92		// envCache=0 turns the cache service off.	130		// envCache=0 turns the cache service off.
93		const envCache = "ERMITE_RUNNER_CACHE"	131		const envCache = "ERMITE_RUNNER_CACHE"
94			132

@@ -307,6 +345,7 @@ func runCmd(ctx context.Context) int {
307		},	345		},
308		Features: hcfg.ServerFeaturesFunc(url),	346		Features: hcfg.ServerFeaturesFunc(url),
309		Cache: cache,	347		Cache: cache,
			348	+	Registry: cache != nil && cache.RegistryEnabled(),
310		Archives: archives,	349		Archives: archives,
311		Results: results,	350		Results: results,
312		Artifacts: func(tok string) ermiteconnect.JobArtifactServiceClient {	351		Artifacts: func(tok string) ermiteconnect.JobArtifactServiceClient {

M internal/cachesvc/handler.go +42 -1


@@ -13,6 +13,8 @@ import (
13	13		"strconv"
14	14		"strings"
15	15		"time"
	16	+
	17	+	"bnema.dev/ermite-runner/internal/registryproxy"
16	18		)
17	19
18	20		// APIPath is the prefix of the cache API: the toolkit appends "_apis/

@@ -45,6 +47,11 @@ func (s *Service) Handler() http.Handler { return s.handler("") }
45	47		// any other sandbox's channel.
46	48		func (s *Service) handler(job string) http.Handler {
47	49		mux := http.NewServeMux()
	50	+	if s.registry != nil {
	51	+	registry := s.serveRegistry(make(chan struct{}, s.registryPerChannel), registryproxy.NewQuota(s.registryDownloads))
	52	+	mux.HandleFunc("GET /v2/", registry)
	53	+	mux.HandleFunc("HEAD /v2/", registry)
	54	+	}
48	55		// The routes are those of the toolkit's v1 client (cacheHttpClient.ts):
49	56		// GET cache?keys=k1,k2&version=v :89 lookup
50	57		// GET caches?key=k :132 diagnostics (core.isDebug())

@@ -83,7 +90,7 @@ func (s *Service) channelHandler(job string, results http.Handler) http.Handler
83	90		return cache
84	91		}
85	92		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
86		-	if strings.HasPrefix(r.URL.Path, APIPath) \|\| strings.HasPrefix(r.URL.Path, ArchivePath) {
	93	+	if strings.HasPrefix(r.URL.Path, APIPath) \|\| strings.HasPrefix(r.URL.Path, ArchivePath) \|\| strings.HasPrefix(r.URL.Path, "/v2/") {
87	94		cache.ServeHTTP(w, r)
88	95		return
89	96		}

@@ -388,6 +395,40 @@ func (p *progressWriter) Write(b []byte) (int, error) {
388	395		// Unwrap lets http.ResponseController reach the real writer.
389	396		func (p *progressWriter) Unwrap() http.ResponseWriter { return p.ResponseWriter }
390	397
	398	+	// KeepAlive moves the deadlines without writing. The registry proxy calls it
	399	+	// while the client waits for a shared upstream download that is progressing,
	400	+	// before the first byte of the reply. The read deadline moves too: once it
	401	+	// passes, net/http cancels the request's context and the wait would end.
	402	+	func (p *progressWriter) KeepAlive() {
	403	+	d := deadline(p.idle, p.remaining)
	404	+	_ = p.rc.SetReadDeadline(d)
	405	+	_ = p.rc.SetWriteDeadline(d)
	406	+	}
	407	+
	408	+	// serveRegistry serves the Docker Hub mirror with the same sliding progress
	409	+	// deadline as the other downloads: a layer of GiB keeps going as long as it
	410	+	// moves, and a client that stops reading is cut after one progress period.
	411	+	// slots caps the requests of this channel in progress at once; one over the
	412	+	// cap gets 503 at once, which dockerd retries or answers by pulling from
	413	+	// Docker Hub, so the channel's connections stay free for the cache. quota is
	414	+	// the channel's share of the proxy's cold downloads.
	415	+	func (s Service) serveRegistry(slots chan struct{}, quota registryproxy.Quota) http.HandlerFunc {
	416	+	return func(w http.ResponseWriter, r *http.Request) {
	417	+	select {
	418	+	case slots <- struct{}{}:
	419	+	defer func() { <-slots }()
	420	+	default:
	421	+	w.Header().Set("Retry-After", "1")
	422	+	reply(w, http.StatusServiceUnavailable, "registry busy")
	423	+	return
	424	+	}
	425	+	rc := http.NewResponseController(w)
	426	+	defer func() { _ = rc.SetWriteDeadline(deadline(s.progress, nil)) }()
	427	+	r = r.WithContext(registryproxy.WithQuota(r.Context(), quota))
	428	+	s.registry.ServeHTTP(&progressWriter{ResponseWriter: w, rc: rc, idle: s.progress}, r)
	429	+	}
	430	+	}
	431	+
391	432		func (s Service) commit(w http.ResponseWriter, r http.Request, c Claims) {
392	433		id, ok := parseID(r)
393	434		var req struct {


@@ -13,6 +13,8 @@ import (
13		"strconv"	13		"strconv"
14		"strings"	14		"strings"
15		"time"	15		"time"
			16	+
			17	+	"bnema.dev/ermite-runner/internal/registryproxy"
16		)	18		)
17			19
18		// APIPath is the prefix of the cache API: the toolkit appends "_apis/	20		// APIPath is the prefix of the cache API: the toolkit appends "_apis/

@@ -45,6 +47,11 @@ func (s *Service) Handler() http.Handler { return s.handler("") }
45		// any other sandbox's channel.	47		// any other sandbox's channel.
46		func (s *Service) handler(job string) http.Handler {	48		func (s *Service) handler(job string) http.Handler {
47		mux := http.NewServeMux()	49		mux := http.NewServeMux()
			50	+	if s.registry != nil {
			51	+	registry := s.serveRegistry(make(chan struct{}, s.registryPerChannel), registryproxy.NewQuota(s.registryDownloads))
			52	+	mux.HandleFunc("GET /v2/", registry)
			53	+	mux.HandleFunc("HEAD /v2/", registry)
			54	+	}
48		// The routes are those of the toolkit's v1 client (cacheHttpClient.ts):	55		// The routes are those of the toolkit's v1 client (cacheHttpClient.ts):
49		// GET cache?keys=k1,k2&version=v :89 lookup	56		// GET cache?keys=k1,k2&version=v :89 lookup
50		// GET caches?key=k :132 diagnostics (core.isDebug())	57		// GET caches?key=k :132 diagnostics (core.isDebug())

@@ -83,7 +90,7 @@ func (s *Service) channelHandler(job string, results http.Handler) http.Handler
83		return cache	90		return cache
84		}	91		}
85		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {	92		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
86	-	if strings.HasPrefix(r.URL.Path, APIPath) \|\| strings.HasPrefix(r.URL.Path, ArchivePath) {	93	+	if strings.HasPrefix(r.URL.Path, APIPath) \|\| strings.HasPrefix(r.URL.Path, ArchivePath) \|\| strings.HasPrefix(r.URL.Path, "/v2/") {
87		cache.ServeHTTP(w, r)	94		cache.ServeHTTP(w, r)
88		return	95		return
89		}	96		}

@@ -388,6 +395,40 @@ func (p *progressWriter) Write(b []byte) (int, error) {
388		// Unwrap lets http.ResponseController reach the real writer.	395		// Unwrap lets http.ResponseController reach the real writer.
389		func (p *progressWriter) Unwrap() http.ResponseWriter { return p.ResponseWriter }	396		func (p *progressWriter) Unwrap() http.ResponseWriter { return p.ResponseWriter }
390			397
			398	+	// KeepAlive moves the deadlines without writing. The registry proxy calls it
			399	+	// while the client waits for a shared upstream download that is progressing,
			400	+	// before the first byte of the reply. The read deadline moves too: once it
			401	+	// passes, net/http cancels the request's context and the wait would end.
			402	+	func (p *progressWriter) KeepAlive() {
			403	+	d := deadline(p.idle, p.remaining)
			404	+	_ = p.rc.SetReadDeadline(d)
			405	+	_ = p.rc.SetWriteDeadline(d)
			406	+	}
			407	+
			408	+	// serveRegistry serves the Docker Hub mirror with the same sliding progress
			409	+	// deadline as the other downloads: a layer of GiB keeps going as long as it
			410	+	// moves, and a client that stops reading is cut after one progress period.
			411	+	// slots caps the requests of this channel in progress at once; one over the
			412	+	// cap gets 503 at once, which dockerd retries or answers by pulling from
			413	+	// Docker Hub, so the channel's connections stay free for the cache. quota is
			414	+	// the channel's share of the proxy's cold downloads.
			415	+	func (s Service) serveRegistry(slots chan struct{}, quota registryproxy.Quota) http.HandlerFunc {
			416	+	return func(w http.ResponseWriter, r *http.Request) {
			417	+	select {
			418	+	case slots <- struct{}{}:
			419	+	defer func() { <-slots }()
			420	+	default:
			421	+	w.Header().Set("Retry-After", "1")
			422	+	reply(w, http.StatusServiceUnavailable, "registry busy")
			423	+	return
			424	+	}
			425	+	rc := http.NewResponseController(w)
			426	+	defer func() { _ = rc.SetWriteDeadline(deadline(s.progress, nil)) }()
			427	+	r = r.WithContext(registryproxy.WithQuota(r.Context(), quota))
			428	+	s.registry.ServeHTTP(&progressWriter{ResponseWriter: w, rc: rc, idle: s.progress}, r)
			429	+	}
			430	+	}
			431	+
391		func (s Service) commit(w http.ResponseWriter, r http.Request, c Claims) {	432		func (s Service) commit(w http.ResponseWriter, r http.Request, c Claims) {
392		id, ok := parseID(r)	433		id, ok := parseID(r)
393		var req struct {	434		var req struct {

A internal/cachesvc/registry_progress_test.go +235 -0

@@ -0,0 +1,235 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+	package cachesvc
	3	+
	4	+	import (
	5	+	"bufio"
	6	+	"fmt"
	7	+	"io"
	8	+	"net"
	9	+	"net/http"
	10	+	"strings"
	11	+	"sync"
	12	+	"sync/atomic"
	13	+	"testing"
	14	+	"time"
	15	+
	16	+	"bnema.dev/ermite-runner/internal/registryproxy"
	17	+	)
	18	+
	19	+	const registryProgress = 150 * time.Millisecond
	20	+
	21	+	// registryRig serves a fake registry behind the service's real HTTP server, so
	22	+	// that the connection deadlines apply.
	23	+	func registryRig(t testing.T, h http.HandlerFunc) rig {
	24	+	t.Helper()
	25	+	return newRigWith(t, Config{Max: 1 << 20, MinCharge: 1, ProgressTimeout: registryProgress, Registry: h})
	26	+	}
	27	+
	28	+	// A registry download that keeps moving may last longer than the
	29	+	// progress period set when the request started.
	30	+	func TestRegistryProgressingTransferSucceeds(t *testing.T) {
	31	+	const chunks = 8
	32	+	r := registryRig(t, func(w http.ResponseWriter, _ *http.Request) {
	33	+	w.Header().Set("Content-Length", fmt.Sprint(chunks*1024))
	34	+	for range chunks {
	35	+	if _, err := w.Write([]byte(strings.Repeat("x", 1024))); err != nil {
	36	+	return
	37	+	}
	38	+	_ = http.NewResponseController(w).Flush()
	39	+	time.Sleep(registryProgress / 3)
	40	+	}
	41	+	})
	42	+	start := time.Now()
	43	+	resp, err := http.Get(r.srv.URL + "/v2/library/alpine/blobs/x")
	44	+	if err != nil {
	45	+	t.Fatal(err)
	46	+	}
	47	+	defer func() { _ = resp.Body.Close() }()
	48	+	b, err := io.ReadAll(resp.Body)
	49	+	if err != nil \|\| len(b) != chunks*1024 {
	50	+	t.Fatalf("read %d bytes: %v", len(b), err)
	51	+	}
	52	+	if d := time.Since(start); d < 2*registryProgress {
	53	+	t.Fatalf("transfer took %v, not longer than the progress period", d)
	54	+	}
	55	+	}
	56	+
	57	+	// The proxy keeps a client alive while it waits for a shared upstream
	58	+	// download, before the first byte; the read deadline moves too, so the request
	59	+	// context stays alive.
	60	+	func TestRegistryKeepAliveWhileWaiting(t *testing.T) {
	61	+	r := registryRig(t, func(w http.ResponseWriter, req *http.Request) {
	62	+	k, ok := w.(interface{ KeepAlive() })
	63	+	if !ok {
	64	+	t.Error("no KeepAlive on the registry writer")
	65	+	return
	66	+	}
	67	+	for range 6 {
	68	+	time.Sleep(registryProgress / 3)
	69	+	k.KeepAlive()
	70	+	if req.Context().Err() != nil {
	71	+	t.Error("request context ended while kept alive")
	72	+	return
	73	+	}
	74	+	}
	75	+	_, _ = io.WriteString(w, "verified")
	76	+	})
	77	+	resp, err := http.Get(r.srv.URL + "/v2/library/alpine/blobs/x")
	78	+	if err != nil {
	79	+	t.Fatal(err)
	80	+	}
	81	+	defer func() { _ = resp.Body.Close() }()
	82	+	if b, err := io.ReadAll(resp.Body); err != nil \|\| string(b) != "verified" {
	83	+	t.Fatalf("%q %v", b, err)
	84	+	}
	85	+	}
	86	+
	87	+	// The registry requests of one channel in progress at once are capped: one
	88	+	// over the cap gets 503 at once, the cache API of that channel still answers,
	89	+	// and another channel has its own slots.
	90	+	func TestRegistryPerChannelCap(t *testing.T) {
	91	+	const limit = 2
	92	+	var inside atomic.Int32
	93	+	release := make(chan struct{})
	94	+	svc, err := New(Config{Dir: t.TempDir(), RegistryPerChannel: limit, Registry: http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
	95	+	inside.Add(1)
	96	+	<-release
	97	+	_, _ = io.WriteString(w, "ok")
	98	+	})})
	99	+	if err != nil {
	100	+	t.Fatal(err)
	101	+	}
	102	+	t.Cleanup(func() { _ = svc.CloseAll() })
	103	+	var once sync.Once
	104	+	unblock := func() { once.Do(func() { close(release) }) }
	105	+	t.Cleanup(unblock)
	106	+	a, b := channel(t, svc, "sb-a", "job-a"), channel(t, svc, "sb-b", "job-b")
	107	+	get := func(c *http.Client) int {
	108	+	resp, err := c.Get("http://cache/v2/library/alpine/blobs/x")
	109	+	if err != nil {
	110	+	t.Error(err)
	111	+	return 0
	112	+	}
	113	+	_, _ = io.Copy(io.Discard, resp.Body)
	114	+	_ = resp.Body.Close()
	115	+	return resp.StatusCode
	116	+	}
	117	+	var wg sync.WaitGroup
	118	+	codes := make(chan int, limit)
	119	+	for range limit {
	120	+	wg.Go(func() { codes <- get(a) })
	121	+	}
	122	+	waitUntil(t, func() bool { return inside.Load() == limit })
	123	+	if code := get(a); code != http.StatusServiceUnavailable {
	124	+	t.Fatalf("over the cap: %d", code)
	125	+	}
	126	+	if code := lookupStatus(t, a, ""); code != http.StatusForbidden {
	127	+	t.Fatalf("cache API behind the registry cap: %d", code)
	128	+	}
	129	+	other := make(chan int, 1)
	130	+	go func() { other <- get(b) }()
	131	+	waitUntil(t, func() bool { return inside.Load() == limit+1 })
	132	+	unblock()
	133	+	wg.Wait()
	134	+	close(codes)
	135	+	for code := range codes {
	136	+	if code != http.StatusOK {
	137	+	t.Fatalf("within the cap: %d", code)
	138	+	}
	139	+	}
	140	+	if code := <-other; code != http.StatusOK {
	141	+	t.Fatalf("other channel: %d", code)
	142	+	}
	143	+	if code := get(a); code != http.StatusOK {
	144	+	t.Fatalf("slots not released: %d", code)
	145	+	}
	146	+	}
	147	+
	148	+	// Each channel hands the registry its own download quota, the same one for
	149	+	// all of its requests.
	150	+	func TestRegistryQuotaPerChannel(t *testing.T) {
	151	+	var mu sync.Mutex
	152	+	seen := map[string]map[*registryproxy.Quota]bool{}
	153	+	svc, err := New(Config{Dir: t.TempDir(), Registry: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	154	+	q := registryproxy.QuotaOf(r.Context())
	155	+	if q == nil {
	156	+	t.Error("no quota on the registry request")
	157	+	}
	158	+	mu.Lock()
	159	+	ch := r.Header.Get("X-Channel")
	160	+	if seen[ch] == nil {
	161	+	seen[ch] = map[*registryproxy.Quota]bool{}
	162	+	}
	163	+	seen[ch][q] = true
	164	+	mu.Unlock()
	165	+	})})
	166	+	if err != nil {
	167	+	t.Fatal(err)
	168	+	}
	169	+	t.Cleanup(func() { _ = svc.CloseAll() })
	170	+	for _, ch := range []string{"a", "b"} {
	171	+	c := channel(t, svc, "sb-"+ch, "job-"+ch)
	172	+	for range 3 {
	173	+	req, _ := http.NewRequest(http.MethodGet, "http://cache/v2/library/alpine/blobs/x", nil)
	174	+	req.Header.Set("X-Channel", ch)
	175	+	resp, err := c.Do(req)
	176	+	if err != nil {
	177	+	t.Fatal(err)
	178	+	}
	179	+	_ = resp.Body.Close()
	180	+	}
	181	+	}
	182	+	mu.Lock()
	183	+	defer mu.Unlock()
	184	+	if len(seen["a"]) != 1 \|\| len(seen["b"]) != 1 {
	185	+	t.Fatalf("quotas per channel: a=%d b=%d", len(seen["a"]), len(seen["b"]))
	186	+	}
	187	+	for qa := range seen["a"] {
	188	+	if seen["b"][qa] {
	189	+	t.Fatal("two channels share a quota")
	190	+	}
	191	+	}
	192	+	}
	193	+
	194	+	// A client that stops reading is cut after one progress period.
	195	+	func TestRegistryStalledReaderIsCut(t *testing.T) {
	196	+	result := make(chan error, 1)
	197	+	r := registryRig(t, func(w http.ResponseWriter, _ *http.Request) {
	198	+	chunk := []byte(strings.Repeat("x", 64<<10))
	199	+	deadline := time.Now().Add(5 * time.Second)
	200	+	for time.Now().Before(deadline) {
	201	+	if _, err := w.Write(chunk); err != nil {
	202	+	result <- err
	203	+	return
	204	+	}
	205	+	}
	206	+	result <- nil
	207	+	})
	208	+	conn, err := net.Dial("tcp", strings.TrimPrefix(r.srv.URL, "http://"))
	209	+	if err != nil {
	210	+	t.Fatal(err)
	211	+	}
	212	+	defer func() { _ = conn.Close() }()
	213	+	if tc, ok := conn.(*net.TCPConn); ok {
	214	+	_ = tc.SetReadBuffer(4096)
	215	+	}
	216	+	if _, err := fmt.Fprint(conn, "GET /v2/library/alpine/blobs/x HTTP/1.1\r\nHost: x\r\n\r\n"); err != nil {
	217	+	t.Fatal(err)
	218	+	}
	219	+	resp, err := http.ReadResponse(bufio.NewReader(conn), nil)
	220	+	if err != nil {
	221	+	t.Fatal(err)
	222	+	}
	223	+	if resp.StatusCode != http.StatusOK {
	224	+	t.Fatal(resp.StatusCode)
	225	+	}
	226	+	// Read nothing more.
	227	+	select {
	228	+	case err := <-result:
	229	+	if err == nil {
	230	+	t.Fatal("a stalled reader was not cut")
	231	+	}
	232	+	case <-time.After(10 * time.Second):
	233	+	t.Fatal("handler still blocked")
	234	+	}
	235	+	}

@@ -0,0 +1,235 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+	package cachesvc
	3	+
	4	+	import (
	5	+	"bufio"
	6	+	"fmt"
	7	+	"io"
	8	+	"net"
	9	+	"net/http"
	10	+	"strings"
	11	+	"sync"
	12	+	"sync/atomic"
	13	+	"testing"
	14	+	"time"
	15	+
	16	+	"bnema.dev/ermite-runner/internal/registryproxy"
	17	+	)
	18	+
	19	+	const registryProgress = 150 * time.Millisecond
	20	+
	21	+	// registryRig serves a fake registry behind the service's real HTTP server, so
	22	+	// that the connection deadlines apply.
	23	+	func registryRig(t testing.T, h http.HandlerFunc) rig {
	24	+	t.Helper()
	25	+	return newRigWith(t, Config{Max: 1 << 20, MinCharge: 1, ProgressTimeout: registryProgress, Registry: h})
	26	+	}
	27	+
	28	+	// A registry download that keeps moving may last longer than the
	29	+	// progress period set when the request started.
	30	+	func TestRegistryProgressingTransferSucceeds(t *testing.T) {
	31	+	const chunks = 8
	32	+	r := registryRig(t, func(w http.ResponseWriter, _ *http.Request) {
	33	+	w.Header().Set("Content-Length", fmt.Sprint(chunks*1024))
	34	+	for range chunks {
	35	+	if _, err := w.Write([]byte(strings.Repeat("x", 1024))); err != nil {
	36	+	return
	37	+	}
	38	+	_ = http.NewResponseController(w).Flush()
	39	+	time.Sleep(registryProgress / 3)
	40	+	}
	41	+	})
	42	+	start := time.Now()
	43	+	resp, err := http.Get(r.srv.URL + "/v2/library/alpine/blobs/x")
	44	+	if err != nil {
	45	+	t.Fatal(err)
	46	+	}
	47	+	defer func() { _ = resp.Body.Close() }()
	48	+	b, err := io.ReadAll(resp.Body)
	49	+	if err != nil \|\| len(b) != chunks*1024 {
	50	+	t.Fatalf("read %d bytes: %v", len(b), err)
	51	+	}
	52	+	if d := time.Since(start); d < 2*registryProgress {
	53	+	t.Fatalf("transfer took %v, not longer than the progress period", d)
	54	+	}
	55	+	}
	56	+
	57	+	// The proxy keeps a client alive while it waits for a shared upstream
	58	+	// download, before the first byte; the read deadline moves too, so the request
	59	+	// context stays alive.
	60	+	func TestRegistryKeepAliveWhileWaiting(t *testing.T) {
	61	+	r := registryRig(t, func(w http.ResponseWriter, req *http.Request) {
	62	+	k, ok := w.(interface{ KeepAlive() })
	63	+	if !ok {
	64	+	t.Error("no KeepAlive on the registry writer")
	65	+	return
	66	+	}
	67	+	for range 6 {
	68	+	time.Sleep(registryProgress / 3)
	69	+	k.KeepAlive()
	70	+	if req.Context().Err() != nil {
	71	+	t.Error("request context ended while kept alive")
	72	+	return
	73	+	}
	74	+	}
	75	+	_, _ = io.WriteString(w, "verified")
	76	+	})
	77	+	resp, err := http.Get(r.srv.URL + "/v2/library/alpine/blobs/x")
	78	+	if err != nil {
	79	+	t.Fatal(err)
	80	+	}
	81	+	defer func() { _ = resp.Body.Close() }()
	82	+	if b, err := io.ReadAll(resp.Body); err != nil \|\| string(b) != "verified" {
	83	+	t.Fatalf("%q %v", b, err)
	84	+	}
	85	+	}
	86	+
	87	+	// The registry requests of one channel in progress at once are capped: one
	88	+	// over the cap gets 503 at once, the cache API of that channel still answers,
	89	+	// and another channel has its own slots.
	90	+	func TestRegistryPerChannelCap(t *testing.T) {
	91	+	const limit = 2
	92	+	var inside atomic.Int32
	93	+	release := make(chan struct{})
	94	+	svc, err := New(Config{Dir: t.TempDir(), RegistryPerChannel: limit, Registry: http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
	95	+	inside.Add(1)
	96	+	<-release
	97	+	_, _ = io.WriteString(w, "ok")
	98	+	})})
	99	+	if err != nil {
	100	+	t.Fatal(err)
	101	+	}
	102	+	t.Cleanup(func() { _ = svc.CloseAll() })
	103	+	var once sync.Once
	104	+	unblock := func() { once.Do(func() { close(release) }) }
	105	+	t.Cleanup(unblock)
	106	+	a, b := channel(t, svc, "sb-a", "job-a"), channel(t, svc, "sb-b", "job-b")
	107	+	get := func(c *http.Client) int {
	108	+	resp, err := c.Get("http://cache/v2/library/alpine/blobs/x")
	109	+	if err != nil {
	110	+	t.Error(err)
	111	+	return 0
	112	+	}
	113	+	_, _ = io.Copy(io.Discard, resp.Body)
	114	+	_ = resp.Body.Close()
	115	+	return resp.StatusCode
	116	+	}
	117	+	var wg sync.WaitGroup
	118	+	codes := make(chan int, limit)
	119	+	for range limit {
	120	+	wg.Go(func() { codes <- get(a) })
	121	+	}
	122	+	waitUntil(t, func() bool { return inside.Load() == limit })
	123	+	if code := get(a); code != http.StatusServiceUnavailable {
	124	+	t.Fatalf("over the cap: %d", code)
	125	+	}
	126	+	if code := lookupStatus(t, a, ""); code != http.StatusForbidden {
	127	+	t.Fatalf("cache API behind the registry cap: %d", code)
	128	+	}
	129	+	other := make(chan int, 1)
	130	+	go func() { other <- get(b) }()
	131	+	waitUntil(t, func() bool { return inside.Load() == limit+1 })
	132	+	unblock()
	133	+	wg.Wait()
	134	+	close(codes)
	135	+	for code := range codes {
	136	+	if code != http.StatusOK {
	137	+	t.Fatalf("within the cap: %d", code)
	138	+	}
	139	+	}
	140	+	if code := <-other; code != http.StatusOK {
	141	+	t.Fatalf("other channel: %d", code)
	142	+	}
	143	+	if code := get(a); code != http.StatusOK {
	144	+	t.Fatalf("slots not released: %d", code)
	145	+	}
	146	+	}
	147	+
	148	+	// Each channel hands the registry its own download quota, the same one for
	149	+	// all of its requests.
	150	+	func TestRegistryQuotaPerChannel(t *testing.T) {
	151	+	var mu sync.Mutex
	152	+	seen := map[string]map[*registryproxy.Quota]bool{}
	153	+	svc, err := New(Config{Dir: t.TempDir(), Registry: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	154	+	q := registryproxy.QuotaOf(r.Context())
	155	+	if q == nil {
	156	+	t.Error("no quota on the registry request")
	157	+	}
	158	+	mu.Lock()
	159	+	ch := r.Header.Get("X-Channel")
	160	+	if seen[ch] == nil {
	161	+	seen[ch] = map[*registryproxy.Quota]bool{}
	162	+	}
	163	+	seen[ch][q] = true
	164	+	mu.Unlock()
	165	+	})})
	166	+	if err != nil {
	167	+	t.Fatal(err)
	168	+	}
	169	+	t.Cleanup(func() { _ = svc.CloseAll() })
	170	+	for _, ch := range []string{"a", "b"} {
	171	+	c := channel(t, svc, "sb-"+ch, "job-"+ch)
	172	+	for range 3 {
	173	+	req, _ := http.NewRequest(http.MethodGet, "http://cache/v2/library/alpine/blobs/x", nil)
	174	+	req.Header.Set("X-Channel", ch)
	175	+	resp, err := c.Do(req)
	176	+	if err != nil {
	177	+	t.Fatal(err)
	178	+	}
	179	+	_ = resp.Body.Close()
	180	+	}
	181	+	}
	182	+	mu.Lock()
	183	+	defer mu.Unlock()
	184	+	if len(seen["a"]) != 1 \|\| len(seen["b"]) != 1 {
	185	+	t.Fatalf("quotas per channel: a=%d b=%d", len(seen["a"]), len(seen["b"]))
	186	+	}
	187	+	for qa := range seen["a"] {
	188	+	if seen["b"][qa] {
	189	+	t.Fatal("two channels share a quota")
	190	+	}
	191	+	}
	192	+	}
	193	+
	194	+	// A client that stops reading is cut after one progress period.
	195	+	func TestRegistryStalledReaderIsCut(t *testing.T) {
	196	+	result := make(chan error, 1)
	197	+	r := registryRig(t, func(w http.ResponseWriter, _ *http.Request) {
	198	+	chunk := []byte(strings.Repeat("x", 64<<10))
	199	+	deadline := time.Now().Add(5 * time.Second)
	200	+	for time.Now().Before(deadline) {
	201	+	if _, err := w.Write(chunk); err != nil {
	202	+	result <- err
	203	+	return
	204	+	}
	205	+	}
	206	+	result <- nil
	207	+	})
	208	+	conn, err := net.Dial("tcp", strings.TrimPrefix(r.srv.URL, "http://"))
	209	+	if err != nil {
	210	+	t.Fatal(err)
	211	+	}
	212	+	defer func() { _ = conn.Close() }()
	213	+	if tc, ok := conn.(*net.TCPConn); ok {
	214	+	_ = tc.SetReadBuffer(4096)
	215	+	}
	216	+	if _, err := fmt.Fprint(conn, "GET /v2/library/alpine/blobs/x HTTP/1.1\r\nHost: x\r\n\r\n"); err != nil {
	217	+	t.Fatal(err)
	218	+	}
	219	+	resp, err := http.ReadResponse(bufio.NewReader(conn), nil)
	220	+	if err != nil {
	221	+	t.Fatal(err)
	222	+	}
	223	+	if resp.StatusCode != http.StatusOK {
	224	+	t.Fatal(resp.StatusCode)
	225	+	}
	226	+	// Read nothing more.
	227	+	select {
	228	+	case err := <-result:
	229	+	if err == nil {
	230	+	t.Fatal("a stalled reader was not cut")
	231	+	}
	232	+	case <-time.After(10 * time.Second):
	233	+	t.Fatal("handler still blocked")
	234	+	}
	235	+	}

A internal/cachesvc/registry_test.go +34 -0

@@ -0,0 +1,34 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+	package cachesvc
	3	+
	4	+	import (
	5	+	"io"
	6	+	"net/http"
	7	+	"net/http/httptest"
	8	+	"testing"
	9	+	)
	10	+
	11	+	func TestRegistryRoute(t *testing.T) {
	12	+	for _, on := range []bool{false, true} {
	13	+	cfg := Config{}
	14	+	if on {
	15	+	cfg.Registry = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = io.WriteString(w, "registry") })
	16	+	}
	17	+	r := newRigWith(t, cfg)
	18	+	results := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	19	+	t.Error("registry fell through to results")
	20	+	w.WriteHeader(http.StatusInternalServerError)
	21	+	})
	22	+	for _, h := range []http.Handler{r.svc.Handler(), r.svc.channelHandler("job", results)} {
	23	+	w := httptest.NewRecorder()
	24	+	h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/v2/", nil))
	25	+	if on {
	26	+	if w.Code != http.StatusOK \|\| w.Body.String() != "registry" {
	27	+	t.Fatalf("%d %s", w.Code, w.Body)
	28	+	}
	29	+	} else if w.Code != http.StatusNotFound {
	30	+	t.Fatal(w.Code)
	31	+	}
	32	+	}
	33	+	}
	34	+	}

@@ -0,0 +1,34 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+	package cachesvc
	3	+
	4	+	import (
	5	+	"io"
	6	+	"net/http"
	7	+	"net/http/httptest"
	8	+	"testing"
	9	+	)
	10	+
	11	+	func TestRegistryRoute(t *testing.T) {
	12	+	for _, on := range []bool{false, true} {
	13	+	cfg := Config{}
	14	+	if on {
	15	+	cfg.Registry = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = io.WriteString(w, "registry") })
	16	+	}
	17	+	r := newRigWith(t, cfg)
	18	+	results := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	19	+	t.Error("registry fell through to results")
	20	+	w.WriteHeader(http.StatusInternalServerError)
	21	+	})
	22	+	for _, h := range []http.Handler{r.svc.Handler(), r.svc.channelHandler("job", results)} {
	23	+	w := httptest.NewRecorder()
	24	+	h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/v2/", nil))
	25	+	if on {
	26	+	if w.Code != http.StatusOK \|\| w.Body.String() != "registry" {
	27	+	t.Fatalf("%d %s", w.Code, w.Body)
	28	+	}
	29	+	} else if w.Code != http.StatusNotFound {
	30	+	t.Fatal(w.Code)
	31	+	}
	32	+	}
	33	+	}
	34	+	}

M internal/cachesvc/service.go +33 -6


@@ -58,6 +58,15 @@ const (
58	58		// blocks at once by default, besides downloads and the cache.
59	59		DefaultConnsPerListener = 32
60	60		DefaultConns = 128
	61	+	// DefaultRegistryPerChannel bounds the registry requests of one channel in
	62	+	// progress at once, so that one sandbox's pulls leave room for its cache and
	63	+	// artifact traffic. dockerd pulls 3 layers at once by default.
	64	+	DefaultRegistryPerChannel = 8
	65	+	// DefaultRegistryDownloadsPerChannel bounds the cold Docker Hub downloads
	66	+	// one channel may have started at once (dockerd's default
	67	+	// max-concurrent-downloads), so that one sandbox cannot take every slot of
	68	+	// the registry proxy.
	69	+	DefaultRegistryDownloadsPerChannel = 3
61	70		)
62	71
63	72		// Config configures a Service.

@@ -78,10 +87,14 @@ type Config struct {
78	87		// Archives, when not nil, is the host's cache of action archives, served on
79	88		// the channels (see ArchiveURL).
80	89		Archives *actionscache.Cache
	90	+	// Registry is an anonymous pull-only registry on the private channel.
	91	+	Registry http.Handler
81	92		// The limits below take the Default… values (or those of the store) when
82	93		// zero; tests set them.
83	94		ProgressTimeout time.Duration
84	95		ConnsPerListener, Conns int
	96	+	RegistryPerChannel int
	97	+	RegistryDownloadsPerChannel int
85	98		MinCharge int64
86	99		MaxPending, MaxPendingPerJob int
87	100		// hookChunk, when set, is called by an upload once its range is admitted and

@@ -102,11 +115,16 @@ type Service struct {
102	115		wg sync.WaitGroup
103	116		closed bool
104	117
105		-	archives *actionscache.Cache
106		-	hookChunk func(id int64)
107		-	progress time.Duration
108		-	perListener int
109		-	connsGlobal chan struct{}
	118	+	archives *actionscache.Cache
	119	+	registry http.Handler
	120	+	// registryPerChannel caps the registry requests in progress per handler.
	121	+	registryPerChannel int
	122	+	// registryDownloads is the cold download quota of each channel.
	123	+	registryDownloads int
	124	+	hookChunk func(id int64)
	125	+	progress time.Duration
	126	+	perListener int
	127	+	connsGlobal chan struct{}
110	128		}
111	129
112	130		type listener struct {

@@ -144,8 +162,14 @@ func New(cfg Config) (*Service, error) {
144	162		if cfg.Conns <= 0 {
145	163		cfg.Conns = DefaultConns
146	164		}
	165	+	if cfg.RegistryPerChannel <= 0 {
	166	+	cfg.RegistryPerChannel = DefaultRegistryPerChannel
	167	+	}
	168	+	if cfg.RegistryDownloadsPerChannel <= 0 {
	169	+	cfg.RegistryDownloadsPerChannel = DefaultRegistryDownloadsPerChannel
	170	+	}
147	171		s := &Service{store: st, sign: sg, now: cfg.Now, log: cfg.Log, channels: map[string]*listener{}, stop: make(chan struct{}),
148		-	archives: cfg.Archives, hookChunk: cfg.hookChunk, progress: cfg.ProgressTimeout, perListener: cfg.ConnsPerListener, connsGlobal: make(chan struct{}, cfg.Conns)}
	172	+	archives: cfg.Archives, registry: cfg.Registry, hookChunk: cfg.hookChunk, progress: cfg.ProgressTimeout, perListener: cfg.ConnsPerListener, connsGlobal: make(chan struct{}, cfg.Conns), registryPerChannel: cfg.RegistryPerChannel, registryDownloads: cfg.RegistryDownloadsPerChannel}
149	173		s.wg.Add(1)
150	174		go s.housekeeping()
151	175		return s, nil

@@ -169,6 +193,9 @@ func (s *Service) housekeeping() {
169	193		}
170	194		}
171	195
	196	+	// RegistryEnabled reports whether the channel serves an anonymous registry.
	197	+	func (s *Service) RegistryEnabled() bool { return s.registry != nil }
	198	+
172	199		// Token returns the ACTIONS_RUNTIME_TOKEN of a job. It is valid for ttl, which
173	200		// should be the job timeout plus a margin. sc is the job's cache scope (the
174	201		// zero Scope for a job without a cache: the cache then refuses the token);

 	// blocks at once by default, besides downloads and the cache.
 	DefaultConnsPerListener = 32
 	DefaultConns            = 128
+	// DefaultRegistryPerChannel bounds the registry requests of one channel in
+	// progress at once, so that one sandbox's pulls leave room for its cache and
+	// artifact traffic. dockerd pulls 3 layers at once by default.
+	DefaultRegistryPerChannel = 8
+	// DefaultRegistryDownloadsPerChannel bounds the cold Docker Hub downloads
+	// one channel may have started at once (dockerd's default
+	// max-concurrent-downloads), so that one sandbox cannot take every slot of
+	// the registry proxy.
+	DefaultRegistryDownloadsPerChannel = 3
 )
 // Config configures a Service.
 	// Archives, when not nil, is the host's cache of action archives, served on
 	// the channels (see ArchiveURL).
 	Archives *actionscache.Cache
+	// Registry is an anonymous pull-only registry on the private channel.
+	Registry http.Handler
 	// The limits below take the Default… values (or those of the store) when
 	// zero; tests set them.
 	ProgressTimeout              time.Duration
 	ConnsPerListener, Conns      int
+	RegistryPerChannel           int
+	RegistryDownloadsPerChannel  int
 	MinCharge                    int64
 	MaxPending, MaxPendingPerJob int
 	// hookChunk, when set, is called by an upload once its range is admitted and
 	wg       sync.WaitGroup
 	closed   bool
-	archives    *actionscache.Cache
+	archives *actionscache.Cache
-	hookChunk   func(id int64)
+	registry http.Handler
-	progress    time.Duration
+	// registryPerChannel caps the registry requests in progress per handler.
-	perListener int
+	registryPerChannel int
-	connsGlobal chan struct{}
+	// registryDownloads is the cold download quota of each channel.
+	registryDownloads int
+	hookChunk         func(id int64)
+	progress          time.Duration
+	perListener       int
+	connsGlobal       chan struct{}
 }
 type listener struct {
 	if cfg.Conns <= 0 {
 		cfg.Conns = DefaultConns
 	}
+	if cfg.RegistryPerChannel <= 0 {
+		cfg.RegistryPerChannel = DefaultRegistryPerChannel
+	}
+	if cfg.RegistryDownloadsPerChannel <= 0 {
+		cfg.RegistryDownloadsPerChannel = DefaultRegistryDownloadsPerChannel
+	}
 	s := &Service{store: st, sign: sg, now: cfg.Now, log: cfg.Log, channels: map[string]*listener{}, stop: make(chan struct{}),
-		archives: cfg.Archives, hookChunk: cfg.hookChunk, progress: cfg.ProgressTimeout, perListener: cfg.ConnsPerListener, connsGlobal: make(chan struct{}, cfg.Conns)}
+		archives: cfg.Archives, registry: cfg.Registry, hookChunk: cfg.hookChunk, progress: cfg.ProgressTimeout, perListener: cfg.ConnsPerListener, connsGlobal: make(chan struct{}, cfg.Conns), registryPerChannel: cfg.RegistryPerChannel, registryDownloads: cfg.RegistryDownloadsPerChannel}
 	s.wg.Add(1)
 	go s.housekeeping()
 	return s, nil
 	}
 }
+// RegistryEnabled reports whether the channel serves an anonymous registry.
+func (s *Service) RegistryEnabled() bool { return s.registry != nil }
 // Token returns the ACTIONS_RUNTIME_TOKEN of a job. It is valid for ttl, which
 // should be the job timeout plus a margin. sc is the job's cache scope (the
 // zero Scope for a job without a cache: the cache then refuses the token);

A internal/e2e/registry_test.go +112 -0

@@ -0,0 +1,112 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+	package e2e
	3	+
	4	+	import (
	5	+	"bytes"
	6	+	"encoding/json"
	7	+	"net/http"
	8	+	"strings"
	9	+	"sync"
	10	+	"testing"
	11	+	"time"
	12	+
	13	+	"bnema.dev/ermite-runner/internal/cachesvc"
	14	+	"bnema.dev/ermite-runner/internal/execstream/execpb"
	15	+	"bnema.dev/ermite-runner/internal/registryproxy"
	16	+	"github.com/bnema/zerowrap"
	17	+	)
	18	+
	19	+	type registryLogs struct {
	20	+	mu sync.Mutex
	21	+	b bytes.Buffer
	22	+	}
	23	+
	24	+	func (b *registryLogs) Write(p []byte) (int, error) {
	25	+	b.mu.Lock()
	26	+	defer b.mu.Unlock()
	27	+	return b.b.Write(p)
	28	+	}
	29	+	func (b *registryLogs) text() string { b.mu.Lock(); defer b.mu.Unlock(); return b.b.String() }
	30	+	func (b *registryLogs) reset() { b.mu.Lock(); defer b.mu.Unlock(); b.b.Reset() }
	31	+
	32	+	func TestDockerHubRegistryAcrossJobs(t *testing.T) {
	33	+	image := needE2E(t)
	34	+	cl := &http.Client{Timeout: 10 * time.Second}
	35	+	resp, err := cl.Get("https://registry-1.docker.io/v2/")
	36	+	if err != nil {
	37	+	t.Skipf("Docker Hub unreachable: %v", err)
	38	+	}
	39	+	_ = resp.Body.Close()
	40	+	if resp.StatusCode != http.StatusUnauthorized && resp.StatusCode != http.StatusOK {
	41	+	t.Skipf("Docker Hub unavailable: HTTP %d", resp.StatusCode)
	42	+	}
	43	+	var logs registryLogs
	44	+	p, err := registryproxy.New(registryproxy.Config{Dir: t.TempDir(), Max: 256 << 20, Log: zerowrap.New(zerowrap.Config{Level: "debug", Format: "json", Output: &logs})})
	45	+	if err != nil {
	46	+	t.Fatal(err)
	47	+	}
	48	+	svc, err := cachesvc.New(cachesvc.Config{Dir: t.TempDir(), Registry: p})
	49	+	if err != nil {
	50	+	t.Fatal(err)
	51	+	}
	52	+	defer func() { _ = svc.CloseAll() }()
	53	+	d := newCacheDriver(t, image, svc)
	54	+	probe := runJob(t, d, jobSpec(`on: push
	55	+	jobs:
	56	+	j:
	57	+	runs-on: x
	58	+	steps:
	59	+	- run: if docker info >/dev/null 2>&1; then echo nested-dockerd-ready; else echo nested-dockerd-unavailable; fi
	60	+	`), nil)
	61	+	if !strings.Contains(strings.Join(probe.Logs, "\n"), "nested-dockerd-ready") {
	62	+	t.Skip("nested dockerd unavailable in sandbox; rootless Docker harness cannot run job containers")
	63	+	}
	64	+	for i := 0; i < 2; i++ {
	65	+	logs.reset()
	66	+	spec := jobSpec(`on: push
	67	+	jobs:
	68	+	j:
	69	+	runs-on: x
	70	+	container: alpine:3
	71	+	steps:
	72	+	- run: echo alpine-job
	73	+	`)
	74	+	spec.CachePort = cachesvc.DefaultPort
	75	+	spec.CacheToken = svc.Token("e2e", cachesvc.Scope{}, time.Hour)
	76	+	s := runJob(t, d, spec, nil)
	77	+	if s.Result == nil \|\| s.Result.Conclusion != execpb.StepConclusion_STEP_CONCLUSION_SUCCESS {
	78	+	t.Fatalf("pull job %d: %v\n%s", i, s.Result, strings.Join(s.Logs, "\n"))
	79	+	}
	80	+	all := logs.text()
	81	+	manifestHit := false
	82	+	for _, line := range strings.Split(all, "\n") {
	83	+	var event struct {
	84	+	Message string `json:"message"`
	85	+	Hit bool `json:"hit"`
	86	+	}
	87	+	if json.Unmarshal([]byte(line), &event) == nil && event.Message == "registry manifest" && event.Hit {
	88	+	manifestHit = true
	89	+	}
	90	+	}
	91	+	if i == 1 && !manifestHit {
	92	+	t.Fatalf("second job missing manifest hit:\n%s", all)
	93	+	}
	94	+	t.Logf("pull job %d proxy logs:\n%s", i+1, all)
	95	+	if i == 1 && strings.Contains(all, `"message":"registry blob"`) {
	96	+	t.Fatalf("second job fetched upstream blobs:\n%s", all)
	97	+	}
	98	+	}
	99	+	// The mirror argv is unconditional: a job without the runtime channel
	100	+	// must still pull directly from Docker Hub.
	101	+	off := runJob(t, d, jobSpec(`on: push
	102	+	jobs:
	103	+	j:
	104	+	runs-on: x
	105	+	container: alpine:3
	106	+	steps:
	107	+	- run: echo direct-pull
	108	+	`), nil)
	109	+	if off.Result == nil \|\| off.Result.Conclusion != execpb.StepConclusion_STEP_CONCLUSION_SUCCESS {
	110	+	t.Fatalf("proxy-off pull: %v\n%s", off.Result, strings.Join(off.Logs, "\n"))
	111	+	}
	112	+	}

@@ -0,0 +1,112 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+	package e2e
	3	+
	4	+	import (
	5	+	"bytes"
	6	+	"encoding/json"
	7	+	"net/http"
	8	+	"strings"
	9	+	"sync"
	10	+	"testing"
	11	+	"time"
	12	+
	13	+	"bnema.dev/ermite-runner/internal/cachesvc"
	14	+	"bnema.dev/ermite-runner/internal/execstream/execpb"
	15	+	"bnema.dev/ermite-runner/internal/registryproxy"
	16	+	"github.com/bnema/zerowrap"
	17	+	)
	18	+
	19	+	type registryLogs struct {
	20	+	mu sync.Mutex
	21	+	b bytes.Buffer
	22	+	}
	23	+
	24	+	func (b *registryLogs) Write(p []byte) (int, error) {
	25	+	b.mu.Lock()
	26	+	defer b.mu.Unlock()
	27	+	return b.b.Write(p)
	28	+	}
	29	+	func (b *registryLogs) text() string { b.mu.Lock(); defer b.mu.Unlock(); return b.b.String() }
	30	+	func (b *registryLogs) reset() { b.mu.Lock(); defer b.mu.Unlock(); b.b.Reset() }
	31	+
	32	+	func TestDockerHubRegistryAcrossJobs(t *testing.T) {
	33	+	image := needE2E(t)
	34	+	cl := &http.Client{Timeout: 10 * time.Second}
	35	+	resp, err := cl.Get("https://registry-1.docker.io/v2/")
	36	+	if err != nil {
	37	+	t.Skipf("Docker Hub unreachable: %v", err)
	38	+	}
	39	+	_ = resp.Body.Close()
	40	+	if resp.StatusCode != http.StatusUnauthorized && resp.StatusCode != http.StatusOK {
	41	+	t.Skipf("Docker Hub unavailable: HTTP %d", resp.StatusCode)
	42	+	}
	43	+	var logs registryLogs
	44	+	p, err := registryproxy.New(registryproxy.Config{Dir: t.TempDir(), Max: 256 << 20, Log: zerowrap.New(zerowrap.Config{Level: "debug", Format: "json", Output: &logs})})
	45	+	if err != nil {
	46	+	t.Fatal(err)
	47	+	}
	48	+	svc, err := cachesvc.New(cachesvc.Config{Dir: t.TempDir(), Registry: p})
	49	+	if err != nil {
	50	+	t.Fatal(err)
	51	+	}
	52	+	defer func() { _ = svc.CloseAll() }()
	53	+	d := newCacheDriver(t, image, svc)
	54	+	probe := runJob(t, d, jobSpec(`on: push
	55	+	jobs:
	56	+	j:
	57	+	runs-on: x
	58	+	steps:
	59	+	- run: if docker info >/dev/null 2>&1; then echo nested-dockerd-ready; else echo nested-dockerd-unavailable; fi
	60	+	`), nil)
	61	+	if !strings.Contains(strings.Join(probe.Logs, "\n"), "nested-dockerd-ready") {
	62	+	t.Skip("nested dockerd unavailable in sandbox; rootless Docker harness cannot run job containers")
	63	+	}
	64	+	for i := 0; i < 2; i++ {
	65	+	logs.reset()
	66	+	spec := jobSpec(`on: push
	67	+	jobs:
	68	+	j:
	69	+	runs-on: x
	70	+	container: alpine:3
	71	+	steps:
	72	+	- run: echo alpine-job
	73	+	`)
	74	+	spec.CachePort = cachesvc.DefaultPort
	75	+	spec.CacheToken = svc.Token("e2e", cachesvc.Scope{}, time.Hour)
	76	+	s := runJob(t, d, spec, nil)
	77	+	if s.Result == nil \|\| s.Result.Conclusion != execpb.StepConclusion_STEP_CONCLUSION_SUCCESS {
	78	+	t.Fatalf("pull job %d: %v\n%s", i, s.Result, strings.Join(s.Logs, "\n"))
	79	+	}
	80	+	all := logs.text()
	81	+	manifestHit := false
	82	+	for _, line := range strings.Split(all, "\n") {
	83	+	var event struct {
	84	+	Message string `json:"message"`
	85	+	Hit bool `json:"hit"`
	86	+	}
	87	+	if json.Unmarshal([]byte(line), &event) == nil && event.Message == "registry manifest" && event.Hit {
	88	+	manifestHit = true
	89	+	}
	90	+	}
	91	+	if i == 1 && !manifestHit {
	92	+	t.Fatalf("second job missing manifest hit:\n%s", all)
	93	+	}
	94	+	t.Logf("pull job %d proxy logs:\n%s", i+1, all)
	95	+	if i == 1 && strings.Contains(all, `"message":"registry blob"`) {
	96	+	t.Fatalf("second job fetched upstream blobs:\n%s", all)
	97	+	}
	98	+	}
	99	+	// The mirror argv is unconditional: a job without the runtime channel
	100	+	// must still pull directly from Docker Hub.
	101	+	off := runJob(t, d, jobSpec(`on: push
	102	+	jobs:
	103	+	j:
	104	+	runs-on: x
	105	+	container: alpine:3
	106	+	steps:
	107	+	- run: echo direct-pull
	108	+	`), nil)
	109	+	if off.Result == nil \|\| off.Result.Conclusion != execpb.StepConclusion_STEP_CONCLUSION_SUCCESS {
	110	+	t.Fatalf("proxy-off pull: %v\n%s", off.Result, strings.Join(off.Logs, "\n"))
	111	+	}
	112	+	}

A internal/guest/dockerd.go +13 -0

@@ -0,0 +1,13 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+	package guest
	3	+
	4	+	import (
	5	+	"bnema.dev/ermite-runner/internal/sandbox"
	6	+	"fmt"
	7	+	)
	8	+
	9	+	func defaultDockerdArgs() []string {
	10	+	return []string{"/usr/bin/dockerd", "--data-root", sandbox.GuestScratch + "/docker",
	11	+	"--storage-driver", "overlay2", "--host", "unix:///var/run/docker.sock",
	12	+	"--registry-mirror", fmt.Sprintf("http://127.0.0.1:%d", sandbox.CacheVsockPort)}
	13	+	}

@@ -0,0 +1,13 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+	package guest
	3	+
	4	+	import (
	5	+	"bnema.dev/ermite-runner/internal/sandbox"
	6	+	"fmt"
	7	+	)
	8	+
	9	+	func defaultDockerdArgs() []string {
	10	+	return []string{"/usr/bin/dockerd", "--data-root", sandbox.GuestScratch + "/docker",
	11	+	"--storage-driver", "overlay2", "--host", "unix:///var/run/docker.sock",
	12	+	"--registry-mirror", fmt.Sprintf("http://127.0.0.1:%d", sandbox.CacheVsockPort)}
	13	+	}

M internal/guest/init.go +1 -2


@@ -159,8 +159,7 @@ func runPayload(ctx context.Context, cfg InitConfig, mode string, args map[strin
159	159		_ = os.MkdirAll(sandbox.GuestScratch+"/docker", 0o711)
160	160		argv := cfg.DockerdArgs
161	161		if argv == nil {
162		-	argv = []string{"/usr/bin/dockerd", "--data-root", sandbox.GuestScratch + "/docker",
163		-	"--storage-driver", "overlay2", "--host", "unix:///var/run/docker.sock"}
	162	+	argv = defaultDockerdArgs()
164	163		// dockerd sets its umask to 0, and a containerd it starts itself
165	164		// inherits it, then every `docker exec` (the steps of a job
166	165		// container): their files would be world-writable. Start containerd

 		_ = os.MkdirAll(sandbox.GuestScratch+"/docker", 0o711)
 		argv := cfg.DockerdArgs
 		if argv == nil {
-			argv = []string{"/usr/bin/dockerd", "--data-root", sandbox.GuestScratch + "/docker",
+			argv = defaultDockerdArgs()
-				"--storage-driver", "overlay2", "--host", "unix:///var/run/docker.sock"}
 			// dockerd sets its umask to 0, and a containerd it starts itself
 			// inherits it, then every `docker exec` (the steps of a job
 			// container): their files would be world-writable. Start containerd

M internal/guest/jobexec.go +6 -0


@@ -365,6 +365,12 @@ func startCacheRelay(ctx context.Context, target Target, spec *execpb.JobSpec, b
365	365		return cacheRelay{}
366	366		}
367	367		results := spec.GetResults()
	368	+	// Registry-only jobs need dockerd's loopback relay, but no action runtime
	369	+	// URLs or token. Leave those fields empty so neither node nor docker
	370	+	// actions receive ACTIONS_* runtime variables.
	371	+	if !results && spec.GetCacheMode() == "" {
	372	+	return cacheRelay{relay: r}
	373	+	}
368	374		c := cacheRelay{relay: r, url: url, token: spec.GetCacheToken(), mode: spec.GetCacheMode(),
369	375		results: results, cacheOff: results && spec.GetCacheMode() == ""}
370	376		if bridge == nil {


@@ -365,6 +365,12 @@ func startCacheRelay(ctx context.Context, target Target, spec *execpb.JobSpec, b
365	return cacheRelay{}	365		return cacheRelay{}
366	}	366		}
367	results := spec.GetResults()	367		results := spec.GetResults()
		368	+	// Registry-only jobs need dockerd's loopback relay, but no action runtime
		369	+	// URLs or token. Leave those fields empty so neither node nor docker
		370	+	// actions receive ACTIONS_* runtime variables.
		371	+	if !results && spec.GetCacheMode() == "" {
		372	+	return cacheRelay{relay: r}
		373	+	}
368	c := cacheRelay{relay: r, url: url, token: spec.GetCacheToken(), mode: spec.GetCacheMode(),	374		c := cacheRelay{relay: r, url: url, token: spec.GetCacheToken(), mode: spec.GetCacheMode(),
369	results: results, cacheOff: results && spec.GetCacheMode() == ""}	375		results: results, cacheOff: results && spec.GetCacheMode() == ""}
370	if bridge == nil {	376		if bridge == nil {

A internal/guest/registry_test.go +27 -0

@@ -0,0 +1,27 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+	package guest
	3	+
	4	+	import (
	5	+	"bnema.dev/ermite-runner/internal/execstream/execpb"
	6	+	"context"
	7	+	"strings"
	8	+	"testing"
	9	+	)
	10	+
	11	+	func TestDefaultDockerdMirror(t *testing.T) {
	12	+	args := strings.Join(defaultDockerdArgs(), " ")
	13	+	if !strings.Contains(args, "--registry-mirror http://127.0.0.1:9089") {
	14	+	t.Fatal(args)
	15	+	}
	16	+	}
	17	+	func TestRegistryOnlyRelayHasNoRuntimeEnvironment(t *testing.T) {
	18	+	target := serveCache(t, t.TempDir())
	19	+	got := startCacheRelay(context.Background(), target, &execpb.JobSpec{CachePort: freePort(t), CacheToken: "unscoped"}, noBridge)
	20	+	defer got.close()
	21	+	if got.relay == nil {
	22	+	t.Fatal("missing loopback relay")
	23	+	}
	24	+	if got.url != "" \|\| got.containerURL != "" \|\| got.token != "" \|\| got.mode != "" \|\| got.results \|\| got.bridge != nil {
	25	+	t.Fatalf("runtime environment leaked: %+v", got)
	26	+	}
	27	+	}

@@ -0,0 +1,27 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+	package guest
	3	+
	4	+	import (
	5	+	"bnema.dev/ermite-runner/internal/execstream/execpb"
	6	+	"context"
	7	+	"strings"
	8	+	"testing"
	9	+	)
	10	+
	11	+	func TestDefaultDockerdMirror(t *testing.T) {
	12	+	args := strings.Join(defaultDockerdArgs(), " ")
	13	+	if !strings.Contains(args, "--registry-mirror http://127.0.0.1:9089") {
	14	+	t.Fatal(args)
	15	+	}
	16	+	}
	17	+	func TestRegistryOnlyRelayHasNoRuntimeEnvironment(t *testing.T) {
	18	+	target := serveCache(t, t.TempDir())
	19	+	got := startCacheRelay(context.Background(), target, &execpb.JobSpec{CachePort: freePort(t), CacheToken: "unscoped"}, noBridge)
	20	+	defer got.close()
	21	+	if got.relay == nil {
	22	+	t.Fatal("missing loopback relay")
	23	+	}
	24	+	if got.url != "" \|\| got.containerURL != "" \|\| got.token != "" \|\| got.mode != "" \|\| got.results \|\| got.bridge != nil {
	25	+	t.Fatalf("runtime environment leaked: %+v", got)
	26	+	}
	27	+	}

M internal/host/cache.go +5 -5


@@ -111,10 +111,10 @@ func githubRepository(contexts string) string {
111	111		// runtimeFor returns what the executor needs of the job's runtime channel: the
112	112		// port it has in the guest, the job's ACTIONS_RUNTIME_TOKEN, the cache mode and
113	113		// whether the channel serves the results service. The channel exists when the
114		-	// runner serves a cache and the job has a cache scope or the results service;
115		-	// without it, 0, "", "" and false. A job with the results service and no cache
116		-	// scope gets an empty mode (the executor then sets no ACTIONS_CACHE_URL) and a
117		-	// token that the cache refuses.
	114	+	// runner serves a cache and the job has a cache scope, the results service,
	115	+	// or the runner has a registry proxy. Without it, 0, "", "" and false.
	116	+	// A job without a cache scope gets an empty mode and a token that the cache
	117	+	// refuses. Registry-only jobs expose no runtime environment variables to steps.
118	118		func (j jobRun) runtimeFor(rt Runner) (port uint32, token, mode string, results bool) {
119	119		if rt.Cache == nil {
120	120		return 0, "", "", false

@@ -134,7 +134,7 @@ func (j jobRun) runtimeFor(rt Runner) (port uint32, token, mode string, result
134	134		mode = "write"
135	135		}
136	136		}
137		-	if !ok && !j.results {
	137	+	if !ok && !j.results && !rt.Registry {
138	138		return 0, "", "", false
139	139		}
140	140		var scopes []string


@@ -111,10 +111,10 @@ func githubRepository(contexts string) string {
111		// runtimeFor returns what the executor needs of the job's runtime channel: the	111		// runtimeFor returns what the executor needs of the job's runtime channel: the
112		// port it has in the guest, the job's ACTIONS_RUNTIME_TOKEN, the cache mode and	112		// port it has in the guest, the job's ACTIONS_RUNTIME_TOKEN, the cache mode and
113		// whether the channel serves the results service. The channel exists when the	113		// whether the channel serves the results service. The channel exists when the
114	-	// runner serves a cache and the job has a cache scope or the results service;	114	+	// runner serves a cache and the job has a cache scope, the results service,
115	-	// without it, 0, "", "" and false. A job with the results service and no cache	115	+	// or the runner has a registry proxy. Without it, 0, "", "" and false.
116	-	// scope gets an empty mode (the executor then sets no ACTIONS_CACHE_URL) and a	116	+	// A job without a cache scope gets an empty mode and a token that the cache
117	-	// token that the cache refuses.	117	+	// refuses. Registry-only jobs expose no runtime environment variables to steps.
118		func (j jobRun) runtimeFor(rt Runner) (port uint32, token, mode string, results bool) {	118		func (j jobRun) runtimeFor(rt Runner) (port uint32, token, mode string, results bool) {
119		if rt.Cache == nil {	119		if rt.Cache == nil {
120		return 0, "", "", false	120		return 0, "", "", false

@@ -134,7 +134,7 @@ func (j jobRun) runtimeFor(rt Runner) (port uint32, token, mode string, result
134		mode = "write"	134		mode = "write"
135		}	135		}
136		}	136		}
137	-	if !ok && !j.results {	137	+	if !ok && !j.results && !rt.Registry {
138		return 0, "", "", false	138		return 0, "", "", false
139		}	139		}
140		var scopes []string	140		var scopes []string

A internal/host/registry_test.go +41 -0

@@ -0,0 +1,41 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+	package host
	3	+
	4	+	import (
	5	+	"bnema.dev/ermite-api/gen/ermite"
	6	+	"bnema.dev/ermite-runner/internal/cachesvc"
	7	+	"net/http"
	8	+	"net/http/httptest"
	9	+	"strings"
	10	+	"testing"
	11	+	)
	12	+
	13	+	func TestRuntimeForRegistryOnly(t *testing.T) {
	14	+	svc, err := cachesvc.New(cachesvc.Config{Dir: t.TempDir()})
	15	+	if err != nil {
	16	+	t.Fatal(err)
	17	+	}
	18	+	defer func() { _ = svc.CloseAll() }()
	19	+	j := &jobRun{job: &ermite.Job{JobId: "j1", TimeoutMinutes: 5}}
	20	+	port, tok, mode, results := j.runtimeFor(&Runner{Cache: svc, Registry: true})
	21	+	if port != cachesvc.DefaultPort \|\| tok == "" \|\| mode != "" \|\| results {
	22	+	t.Fatalf("%d %q %q %v", port, tok, mode, results)
	23	+	}
	24	+	job, scopes, _, err := svc.RuntimeToken(tok)
	25	+	if err != nil \|\| job != "j1" \|\| strings.Contains(scopes, "Actions.Results") {
	26	+	t.Fatalf("%q %q %v", job, scopes, err)
	27	+	}
	28	+	req := httptest.NewRequest(http.MethodPost, cachesvc.APIPath+"caches", strings.NewReader(`{"key":"k","version":"v","cacheSize":1}`))
	29	+	req.Header.Set("Authorization", "Bearer "+tok)
	30	+	w := httptest.NewRecorder()
	31	+	svc.Handler().ServeHTTP(w, req)
	32	+	if w.Code != http.StatusForbidden {
	33	+	t.Fatalf("unscoped token allowed cache: %d", w.Code)
	34	+	}
	35	+	if p, _, _, _ := j.runtimeFor(&Runner{Cache: svc}); p != 0 {
	36	+	t.Fatal("proxy off still supplies channel")
	37	+	}
	38	+	if p, _, _, _ := j.runtimeFor(&Runner{Registry: true}); p != 0 {
	39	+	t.Fatal("cache off still supplies channel")
	40	+	}
	41	+	}

@@ -0,0 +1,41 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+	package host
	3	+
	4	+	import (
	5	+	"bnema.dev/ermite-api/gen/ermite"
	6	+	"bnema.dev/ermite-runner/internal/cachesvc"
	7	+	"net/http"
	8	+	"net/http/httptest"
	9	+	"strings"
	10	+	"testing"
	11	+	)
	12	+
	13	+	func TestRuntimeForRegistryOnly(t *testing.T) {
	14	+	svc, err := cachesvc.New(cachesvc.Config{Dir: t.TempDir()})
	15	+	if err != nil {
	16	+	t.Fatal(err)
	17	+	}
	18	+	defer func() { _ = svc.CloseAll() }()
	19	+	j := &jobRun{job: &ermite.Job{JobId: "j1", TimeoutMinutes: 5}}
	20	+	port, tok, mode, results := j.runtimeFor(&Runner{Cache: svc, Registry: true})
	21	+	if port != cachesvc.DefaultPort \|\| tok == "" \|\| mode != "" \|\| results {
	22	+	t.Fatalf("%d %q %q %v", port, tok, mode, results)
	23	+	}
	24	+	job, scopes, _, err := svc.RuntimeToken(tok)
	25	+	if err != nil \|\| job != "j1" \|\| strings.Contains(scopes, "Actions.Results") {
	26	+	t.Fatalf("%q %q %v", job, scopes, err)
	27	+	}
	28	+	req := httptest.NewRequest(http.MethodPost, cachesvc.APIPath+"caches", strings.NewReader(`{"key":"k","version":"v","cacheSize":1}`))
	29	+	req.Header.Set("Authorization", "Bearer "+tok)
	30	+	w := httptest.NewRecorder()
	31	+	svc.Handler().ServeHTTP(w, req)
	32	+	if w.Code != http.StatusForbidden {
	33	+	t.Fatalf("unscoped token allowed cache: %d", w.Code)
	34	+	}
	35	+	if p, _, _, _ := j.runtimeFor(&Runner{Cache: svc}); p != 0 {
	36	+	t.Fatal("proxy off still supplies channel")
	37	+	}
	38	+	if p, _, _, _ := j.runtimeFor(&Runner{Registry: true}); p != 0 {
	39	+	t.Fatal("cache off still supplies channel")
	40	+	}
	41	+	}

M internal/host/runner.go +2 -0


@@ -45,6 +45,8 @@ type Runner struct {
45	45		// which the guest reaches on its loopback (cachesvc.DefaultPort), and each
46	46		// job gets a token scoped to its repository and refs.
47	47		Cache *cachesvc.Service
	48	+	// Registry enables the runtime channel for anonymous registry-only jobs.
	49	+	Registry bool
48	50		// Archives, when not nil (and Cache is), is the host's cache of action
49	51		// archives: the host downloads them and the sandbox reads them from the cache
50	52		// channel (see cacheArchives). ArchiveHTTP is its HTTP client (tests); nil


@@ -45,6 +45,8 @@ type Runner struct {
45	// which the guest reaches on its loopback (cachesvc.DefaultPort), and each	45		// which the guest reaches on its loopback (cachesvc.DefaultPort), and each
46	// job gets a token scoped to its repository and refs.	46		// job gets a token scoped to its repository and refs.
47	Cache *cachesvc.Service	47		Cache *cachesvc.Service
		48	+	// Registry enables the runtime channel for anonymous registry-only jobs.
		49	+	Registry bool
48	// Archives, when not nil (and Cache is), is the host's cache of action	50		// Archives, when not nil (and Cache is), is the host's cache of action
49	// archives: the host downloads them and the sandbox reads them from the cache	51		// archives: the host downloads them and the sandbox reads them from the cache
50	// channel (see cacheArchives). ArchiveHTTP is its HTTP client (tests); nil	52		// channel (see cacheArchives). ArchiveHTTP is its HTTP client (tests); nil

A internal/registryproxy/accept.go +40 -0

@@ -0,0 +1,40 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+	package registryproxy
	3	+
	4	+	import (
	5	+	"sort"
	6	+	"strings"
	7	+	)
	8	+
	9	+	var manifestTypes = []string{
	10	+	"application/vnd.docker.distribution.manifest.v2+json",
	11	+	"application/vnd.docker.distribution.manifest.list.v2+json",
	12	+	"application/vnd.oci.image.manifest.v1+json",
	13	+	"application/vnd.oci.image.index.v1+json",
	14	+	}
	15	+
	16	+	// normalizeAccept bounds both negotiation and metadata cardinality to the
	17	+	// fifteen nonempty subsets of the supported manifest types.
	18	+	func normalizeAccept(raw string) string {
	19	+	seen := map[string]bool{}
	20	+	for _, part := range strings.Split(raw, ",") {
	21	+	typ, _, _ := strings.Cut(part, ";")
	22	+	typ = strings.ToLower(strings.TrimSpace(typ))
	23	+	for _, known := range manifestTypes {
	24	+	if typ == known {
	25	+	seen[typ] = true
	26	+	}
	27	+	}
	28	+	}
	29	+	if len(seen) == 0 {
	30	+	for _, typ := range manifestTypes {
	31	+	seen[typ] = true
	32	+	}
	33	+	}
	34	+	types := make([]string, 0, len(seen))
	35	+	for typ := range seen {
	36	+	types = append(types, typ)
	37	+	}
	38	+	sort.Strings(types)
	39	+	return strings.Join(types, ",")
	40	+	}

@@ -0,0 +1,40 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+	package registryproxy
	3	+
	4	+	import (
	5	+	"sort"
	6	+	"strings"
	7	+	)
	8	+
	9	+	var manifestTypes = []string{
	10	+	"application/vnd.docker.distribution.manifest.v2+json",
	11	+	"application/vnd.docker.distribution.manifest.list.v2+json",
	12	+	"application/vnd.oci.image.manifest.v1+json",
	13	+	"application/vnd.oci.image.index.v1+json",
	14	+	}
	15	+
	16	+	// normalizeAccept bounds both negotiation and metadata cardinality to the
	17	+	// fifteen nonempty subsets of the supported manifest types.
	18	+	func normalizeAccept(raw string) string {
	19	+	seen := map[string]bool{}
	20	+	for _, part := range strings.Split(raw, ",") {
	21	+	typ, _, _ := strings.Cut(part, ";")
	22	+	typ = strings.ToLower(strings.TrimSpace(typ))
	23	+	for _, known := range manifestTypes {
	24	+	if typ == known {
	25	+	seen[typ] = true
	26	+	}
	27	+	}
	28	+	}
	29	+	if len(seen) == 0 {
	30	+	for _, typ := range manifestTypes {
	31	+	seen[typ] = true
	32	+	}
	33	+	}
	34	+	types := make([]string, 0, len(seen))
	35	+	for typ := range seen {
	36	+	types = append(types, typ)
	37	+	}
	38	+	sort.Strings(types)
	39	+	return strings.Join(types, ",")
	40	+	}

A internal/registryproxy/clients.go +126 -0

@@ -0,0 +1,126 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+	package registryproxy
	3	+
	4	+	import (
	5	+	"context"
	6	+	"errors"
	7	+	"io"
	8	+	"net/http"
	9	+	"sync"
	10	+	"time"
	11	+	)
	12	+
	13	+	// upstreamClients returns the metadata client (auth, manifests: a total
	14	+	// timeout and no redirects) and the blob client (no total timeout, a response
	15	+	// header timeout, and redirects that drop the bearer when leaving the host).
	16	+	func upstreamClients(c Config) (http.Client, http.Client) {
	17	+	base := http.Client{}
	18	+	if c.Client != nil {
	19	+	base = *c.Client
	20	+	}
	21	+	metadata := base
	22	+	metadata.Timeout = c.RequestTimeout
	23	+	metadata.CheckRedirect = func(http.Request, []http.Request) error { return http.ErrUseLastResponse }
	24	+	blobs := base
	25	+	blobs.Timeout = 0
	26	+	transport := base.Transport
	27	+	if transport == nil {
	28	+	transport = http.DefaultTransport
	29	+	}
	30	+	if tr, ok := transport.(*http.Transport); ok {
	31	+	tr = tr.Clone()
	32	+	tr.ResponseHeaderTimeout = c.RequestTimeout
	33	+	blobs.Transport = tr
	34	+	}
	35	+	blobs.CheckRedirect = func(req http.Request, via []http.Request) error {
	36	+	if len(via) >= 10 {
	37	+	return errors.New("too many registry redirects")
	38	+	}
	39	+	if len(via) > 0 && req.URL.Host != via[0].URL.Host {
	40	+	req.Header.Del("Authorization")
	41	+	}
	42	+	return nil
	43	+	}
	44	+	return &metadata, &blobs
	45	+	}
	46	+
	47	+	// idleBody serializes timer reset/stop against expiry. Cancellation interrupts
	48	+	// even a blocked transport read; only completed, nonempty reads renew the lease.
	49	+	type idleBody struct {
	50	+	io.ReadCloser
	51	+	mu sync.Mutex
	52	+	timer *time.Timer
	53	+	cancel context.CancelFunc
	54	+	idle time.Duration
	55	+	expired bool
	56	+	}
	57	+
	58	+	func (b *idleBody) Read(p []byte) (int, error) {
	59	+	n, err := b.ReadCloser.Read(p)
	60	+	if n > 0 {
	61	+	b.mu.Lock()
	62	+	if !b.expired {
	63	+	b.timer.Reset(b.idle)
	64	+	}
	65	+	b.mu.Unlock()
	66	+	}
	67	+	return n, err
	68	+	}
	69	+	func (b *idleBody) Close() error {
	70	+	b.mu.Lock()
	71	+	b.expired = true
	72	+	b.timer.Stop()
	73	+	b.mu.Unlock()
	74	+	b.cancel()
	75	+	return b.ReadCloser.Close()
	76	+	}
	77	+	func (u upstream) blob(ctx context.Context, name, digest string) (http.Response, error) {
	78	+	// Token acquisition has a total timeout; the subsequent blob request has only
	79	+	// a header/body progress deadline and no total transfer deadline.
	80	+	authCtx, authCancel := context.WithTimeout(ctx, u.requestTimeout)
	81	+	tok, err := u.token(authCtx, name, false)
	82	+	authCancel()
	83	+	if err != nil {
	84	+	return nil, err
	85	+	}
	86	+	for attempt := 0; attempt < 2; attempt++ {
	87	+	bodyCtx, cancel := context.WithCancel(ctx)
	88	+	lease := &idleBody{cancel: cancel, idle: u.blobIdle}
	89	+	lease.timer = time.AfterFunc(u.requestTimeout, func() { lease.mu.Lock(); lease.expired = true; cancel(); lease.mu.Unlock() })
	90	+	req, err := http.NewRequestWithContext(bodyCtx, http.MethodGet, u.base+"/v2/"+name+"/blobs/"+digest, nil)
	91	+	if err != nil {
	92	+	lease.timer.Stop()
	93	+	cancel()
	94	+	return nil, err
	95	+	}
	96	+	req.Header.Set("Authorization", "Bearer "+tok)
	97	+	resp, err := u.blobClient.Do(req)
	98	+	if err != nil {
	99	+	lease.timer.Stop()
	100	+	cancel()
	101	+	return nil, err
	102	+	}
	103	+	lease.ReadCloser = resp.Body
	104	+	if resp.StatusCode == http.StatusOK {
	105	+	lease.mu.Lock()
	106	+	if !lease.expired {
	107	+	lease.timer.Reset(u.blobIdle)
	108	+	}
	109	+	lease.mu.Unlock()
	110	+	resp.Body = lease
	111	+	return resp, nil
	112	+	}
	113	+	_ = lease.Close()
	114	+	if (resp.StatusCode == http.StatusUnauthorized \|\| resp.StatusCode == http.StatusForbidden) && attempt == 0 {
	115	+	authCtx, authCancel := context.WithTimeout(ctx, u.requestTimeout)
	116	+	tok, err = u.token(authCtx, name, true)
	117	+	authCancel()
	118	+	if err != nil {
	119	+	return nil, err
	120	+	}
	121	+	continue
	122	+	}
	123	+	return nil, upstreamError{resp.StatusCode}
	124	+	}
	125	+	return nil, upstreamError{http.StatusUnauthorized}
	126	+	}

@@ -0,0 +1,126 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+	package registryproxy
	3	+
	4	+	import (
	5	+	"context"
	6	+	"errors"
	7	+	"io"
	8	+	"net/http"
	9	+	"sync"
	10	+	"time"
	11	+	)
	12	+
	13	+	// upstreamClients returns the metadata client (auth, manifests: a total
	14	+	// timeout and no redirects) and the blob client (no total timeout, a response
	15	+	// header timeout, and redirects that drop the bearer when leaving the host).
	16	+	func upstreamClients(c Config) (http.Client, http.Client) {
	17	+	base := http.Client{}
	18	+	if c.Client != nil {
	19	+	base = *c.Client
	20	+	}
	21	+	metadata := base
	22	+	metadata.Timeout = c.RequestTimeout
	23	+	metadata.CheckRedirect = func(http.Request, []http.Request) error { return http.ErrUseLastResponse }
	24	+	blobs := base
	25	+	blobs.Timeout = 0
	26	+	transport := base.Transport
	27	+	if transport == nil {
	28	+	transport = http.DefaultTransport
	29	+	}
	30	+	if tr, ok := transport.(*http.Transport); ok {
	31	+	tr = tr.Clone()
	32	+	tr.ResponseHeaderTimeout = c.RequestTimeout
	33	+	blobs.Transport = tr
	34	+	}
	35	+	blobs.CheckRedirect = func(req http.Request, via []http.Request) error {
	36	+	if len(via) >= 10 {
	37	+	return errors.New("too many registry redirects")
	38	+	}
	39	+	if len(via) > 0 && req.URL.Host != via[0].URL.Host {
	40	+	req.Header.Del("Authorization")
	41	+	}
	42	+	return nil
	43	+	}
	44	+	return &metadata, &blobs
	45	+	}
	46	+
	47	+	// idleBody serializes timer reset/stop against expiry. Cancellation interrupts
	48	+	// even a blocked transport read; only completed, nonempty reads renew the lease.
	49	+	type idleBody struct {
	50	+	io.ReadCloser
	51	+	mu sync.Mutex
	52	+	timer *time.Timer
	53	+	cancel context.CancelFunc
	54	+	idle time.Duration
	55	+	expired bool
	56	+	}
	57	+
	58	+	func (b *idleBody) Read(p []byte) (int, error) {
	59	+	n, err := b.ReadCloser.Read(p)
	60	+	if n > 0 {
	61	+	b.mu.Lock()
	62	+	if !b.expired {
	63	+	b.timer.Reset(b.idle)
	64	+	}
	65	+	b.mu.Unlock()
	66	+	}
	67	+	return n, err
	68	+	}
	69	+	func (b *idleBody) Close() error {
	70	+	b.mu.Lock()
	71	+	b.expired = true
	72	+	b.timer.Stop()
	73	+	b.mu.Unlock()
	74	+	b.cancel()
	75	+	return b.ReadCloser.Close()
	76	+	}
	77	+	func (u upstream) blob(ctx context.Context, name, digest string) (http.Response, error) {
	78	+	// Token acquisition has a total timeout; the subsequent blob request has only
	79	+	// a header/body progress deadline and no total transfer deadline.
	80	+	authCtx, authCancel := context.WithTimeout(ctx, u.requestTimeout)
	81	+	tok, err := u.token(authCtx, name, false)
	82	+	authCancel()
	83	+	if err != nil {
	84	+	return nil, err
	85	+	}
	86	+	for attempt := 0; attempt < 2; attempt++ {
	87	+	bodyCtx, cancel := context.WithCancel(ctx)
	88	+	lease := &idleBody{cancel: cancel, idle: u.blobIdle}
	89	+	lease.timer = time.AfterFunc(u.requestTimeout, func() { lease.mu.Lock(); lease.expired = true; cancel(); lease.mu.Unlock() })
	90	+	req, err := http.NewRequestWithContext(bodyCtx, http.MethodGet, u.base+"/v2/"+name+"/blobs/"+digest, nil)
	91	+	if err != nil {
	92	+	lease.timer.Stop()
	93	+	cancel()
	94	+	return nil, err
	95	+	}
	96	+	req.Header.Set("Authorization", "Bearer "+tok)
	97	+	resp, err := u.blobClient.Do(req)
	98	+	if err != nil {
	99	+	lease.timer.Stop()
	100	+	cancel()
	101	+	return nil, err
	102	+	}
	103	+	lease.ReadCloser = resp.Body
	104	+	if resp.StatusCode == http.StatusOK {
	105	+	lease.mu.Lock()
	106	+	if !lease.expired {
	107	+	lease.timer.Reset(u.blobIdle)
	108	+	}
	109	+	lease.mu.Unlock()
	110	+	resp.Body = lease
	111	+	return resp, nil
	112	+	}
	113	+	_ = lease.Close()
	114	+	if (resp.StatusCode == http.StatusUnauthorized \|\| resp.StatusCode == http.StatusForbidden) && attempt == 0 {
	115	+	authCtx, authCancel := context.WithTimeout(ctx, u.requestTimeout)
	116	+	tok, err = u.token(authCtx, name, true)
	117	+	authCancel()
	118	+	if err != nil {
	119	+	return nil, err
	120	+	}
	121	+	continue
	122	+	}
	123	+	return nil, upstreamError{resp.StatusCode}
	124	+	}
	125	+	return nil, upstreamError{http.StatusUnauthorized}
	126	+	}

A internal/registryproxy/flights_test.go +431 -0

@@ -0,0 +1,431 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+	package registryproxy
	3	+
	4	+	import (
	5	+	"bytes"
	6	+	"context"
	7	+	"fmt"
	8	+	"io"
	9	+	"net/http"
	10	+	"net/http/httptest"
	11	+	"os"
	12	+	"path/filepath"
	13	+	"strings"
	14	+	"sync"
	15	+	"sync/atomic"
	16	+	"testing"
	17	+	"time"
	18	+
	19	+	"github.com/bnema/zerowrap"
	20	+	)
	21	+
	22	+	// stallingUpstream serves a blob's first bytes and then holds the body until
	23	+	// the request is cancelled; it counts started and cancelled downloads.
	24	+	type stallingUpstream struct {
	25	+	srv *httptest.Server
	26	+	started, cancelled atomic.Int32
	27	+	}
	28	+
	29	+	func newStallingUpstream(t testing.T) stallingUpstream {
	30	+	t.Helper()
	31	+	u := &stallingUpstream{}
	32	+	u.srv = upstreamWith(t, func(w http.ResponseWriter, r *http.Request) {
	33	+	u.started.Add(1)
	34	+	w.Header().Set("Content-Length", "1000000")
	35	+	_, _ = io.WriteString(w, "partial")
	36	+	w.(http.Flusher).Flush()
	37	+	select {
	38	+	case <-r.Context().Done():
	39	+	u.cancelled.Add(1)
	40	+	case <-time.After(10 * time.Second):
	41	+	}
	42	+	})
	43	+	return u
	44	+	}
	45	+
	46	+	func blobPath(i int) string { return "/v2/library/alpine/blobs/" + sum(fmt.Sprint("blob-", i)) }
	47	+
	48	+	// serveCancelled starts a request and cancels it once its download has begun.
	49	+	func serveCancelled(t testing.T, p Proxy, u *stallingUpstream, path string, started int32) {
	50	+	t.Helper()
	51	+	ctx, cancel := context.WithCancel(context.Background())
	52	+	done := make(chan struct{})
	53	+	go func() {
	54	+	defer close(done)
	55	+	p.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, path, nil).WithContext(ctx))
	56	+	}()
	57	+	waitFor(t, p.s, "the download to start", func() bool { return u.started.Load() >= started })
	58	+	cancel()
	59	+	<-done
	60	+	}
	61	+
	62	+	// MAJOR 1a: a guest that requests many digests and disconnects leaves no
	63	+	// download behind: once a flight has no waiter for the grace period, it is
	64	+	// cancelled upstream and its temporary file removed.
	65	+	func TestAbandonedDownloadsCancelled(t *testing.T) {
	66	+	u := newStallingUpstream(t)
	67	+	const n = 20
	68	+	p := proxyWith(t, u.srv, Config{AbandonGrace: 20 * time.Millisecond, MaxDownloads: n})
	69	+	for i := range n {
	70	+	serveCancelled(t, p, u, blobPath(i), int32(i+1))
	71	+	}
	72	+	waitFor(t, p.s, "every flight to end", func() bool { return len(p.s.flights) == 0 && p.s.downloads == 0 && p.s.pending == 0 })
	73	+	for end := time.Now().Add(5 * time.Second); u.cancelled.Load() != n; {
	74	+	if time.Now().After(end) {
	75	+	t.Fatalf("upstream saw %d cancellations of %d", u.cancelled.Load(), n)
	76	+	}
	77	+	time.Sleep(time.Millisecond)
	78	+	}
	79	+	waitFor(t, p.s, "tmp/ to be empty", func() bool {
	80	+	ds, _ := os.ReadDir(filepath.Join(p.s.dir, "tmp"))
	81	+	return len(ds) == 0
	82	+	})
	83	+	}
	84	+
	85	+	// MAJOR 1a: a request that joins within the grace period keeps the download.
	86	+	func TestRejoinWithinGraceKeepsDownload(t *testing.T) {
	87	+	body := strings.Repeat("layer", 4000)
	88	+	release := make(chan struct{})
	89	+	var gets, cancelled atomic.Int32
	90	+	srv := upstreamWith(t, func(w http.ResponseWriter, r *http.Request) {
	91	+	gets.Add(1)
	92	+	w.Header().Set("Content-Length", fmt.Sprint(len(body)))
	93	+	_, _ = io.WriteString(w, body[:10])
	94	+	w.(http.Flusher).Flush()
	95	+	select {
	96	+	case <-release:
	97	+	_, _ = io.WriteString(w, body[10:])
	98	+	case <-r.Context().Done():
	99	+	cancelled.Add(1)
	100	+	}
	101	+	})
	102	+	var once sync.Once
	103	+	t.Cleanup(func() { once.Do(func() { close(release) }) })
	104	+	p := proxyWith(t, srv, Config{AbandonGrace: time.Hour})
	105	+	path := "/v2/library/alpine/blobs/" + sum(body)
	106	+	ctx, cancel := context.WithCancel(context.Background())
	107	+	first := make(chan struct{})
	108	+	go func() {
	109	+	defer close(first)
	110	+	p.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, path, nil).WithContext(ctx))
	111	+	}()
	112	+	key := "blobs/" + sum(body)[7:]
	113	+	waitFor(t, p.s, "the first waiter", func() bool { return waiters(p.s, key) == 1 })
	114	+	cancel()
	115	+	<-first
	116	+	w := httptest.NewRecorder()
	117	+	second := make(chan struct{})
	118	+	go func() { defer close(second); p.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) }()
	119	+	waitFor(t, p.s, "the second waiter", func() bool { return waiters(p.s, key) == 1 })
	120	+	once.Do(func() { close(release) })
	121	+	<-second
	122	+	if w.Code != http.StatusOK \|\| w.Body.String() != body \|\| gets.Load() != 1 \|\| cancelled.Load() != 0 {
	123	+	t.Fatalf("code %d, %d fetches, %d cancelled", w.Code, gets.Load(), cancelled.Load())
	124	+	}
	125	+	}
	126	+
	127	+	// MAJOR 1b: at most MaxDownloads cold blobs download at once; one more waits
	128	+	// for SlotWait and then gets 503 (dockerd retries or pulls from Docker Hub),
	129	+	// and a slot frees once a download ends.
	130	+	func TestColdDownloadCap(t *testing.T) {
	131	+	u := newStallingUpstream(t)
	132	+	const limit = 3
	133	+	const wait = 50 * time.Millisecond
	134	+	p := proxyWith(t, u.srv, Config{AbandonGrace: 10 * time.Millisecond, MaxDownloads: limit, SlotWait: wait})
	135	+	ctx, cancel := context.WithCancel(context.Background())
	136	+	var wg sync.WaitGroup
	137	+	for i := range limit {
	138	+	wg.Go(func() {
	139	+	p.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, blobPath(i), nil).WithContext(ctx))
	140	+	})
	141	+	}
	142	+	waitFor(t, p.s, "the cap to fill", func() bool { return p.s.downloads == limit })
	143	+	start := time.Now()
	144	+	w := request(t, p, http.MethodGet, blobPath(limit))
	145	+	if w.Code != http.StatusServiceUnavailable \|\| w.Header().Get("Retry-After") == "" {
	146	+	t.Fatalf("over the cap: %d", w.Code)
	147	+	}
	148	+	if d := time.Since(start); d < wait {
	149	+	t.Fatalf("503 after %v, before the slot wait", d)
	150	+	}
	151	+	// Joining a running download takes no slot.
	152	+	joined := make(chan int, 1)
	153	+	go func() {
	154	+	jctx, jcancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
	155	+	defer jcancel()
	156	+	jw := httptest.NewRecorder()
	157	+	p.ServeHTTP(jw, httptest.NewRequest(http.MethodGet, blobPath(0), nil).WithContext(jctx))
	158	+	joined <- jw.Code
	159	+	}()
	160	+	if code := <-joined; code == http.StatusServiceUnavailable {
	161	+	t.Fatal("a joiner was refused a slot")
	162	+	}
	163	+	cancel()
	164	+	wg.Wait()
	165	+	waitFor(t, p.s, "the slots to free", func() bool { return p.s.downloads == 0 })
	166	+	if u.started.Load() != limit {
	167	+	t.Fatalf("%d downloads started", u.started.Load())
	168	+	}
	169	+	}
	170	+
	171	+	// MAJOR 2: dockerd's legacy client sends one Accept line per type. All of them
	172	+	// reach upstream, and they make one tag file.
	173	+	func TestMultiLineAccept(t *testing.T) {
	174	+	types := []string{
	175	+	"application/vnd.oci.image.index.v1+json",
	176	+	"application/vnd.docker.distribution.manifest.list.v2+json",
	177	+	"application/vnd.docker.distribution.manifest.v2+json",
	178	+	}
	179	+	var mu sync.Mutex
	180	+	var seen []string
	181	+	srv := upstreamWith(t, func(w http.ResponseWriter, r *http.Request) {
	182	+	mu.Lock()
	183	+	seen = append(seen, strings.Join(r.Header.Values("Accept"), ","))
	184	+	mu.Unlock()
	185	+	w.Header().Set("Content-Type", types[0])
	186	+	w.Header().Set("Docker-Content-Digest", sum(manifestBody))
	187	+	if r.Method == http.MethodGet {
	188	+	_, _ = io.WriteString(w, manifestBody)
	189	+	}
	190	+	})
	191	+	p := proxyWith(t, srv, Config{})
	192	+	for range 2 {
	193	+	r := httptest.NewRequest(http.MethodGet, "/v2/library/alpine/manifests/3", nil)
	194	+	for _, typ := range types {
	195	+	r.Header.Add("Accept", typ)
	196	+	}
	197	+	w := httptest.NewRecorder()
	198	+	p.ServeHTTP(w, r)
	199	+	if w.Code != http.StatusOK {
	200	+	t.Fatal(w.Code)
	201	+	}
	202	+	}
	203	+	mu.Lock()
	204	+	for _, got := range seen {
	205	+	for _, typ := range types {
	206	+	if !strings.Contains(got, typ) {
	207	+	t.Errorf("upstream Accept %q lacks %s", got, typ)
	208	+	}
	209	+	}
	210	+	}
	211	+	mu.Unlock()
	212	+	if n := accounted(t, p.s); n != 1 {
	213	+	t.Fatalf("%d tag files", n)
	214	+	}
	215	+	// The 8 KiB bound applies to the joined lines.
	216	+	r := httptest.NewRequest(http.MethodGet, "/v2/library/alpine/manifests/3", nil)
	217	+	for range 200 {
	218	+	r.Header.Add("Accept", strings.Repeat("x", 50))
	219	+	}
	220	+	w := httptest.NewRecorder()
	221	+	p.ServeHTTP(w, r)
	222	+	if w.Code != http.StatusNotFound {
	223	+	t.Fatalf("oversized Accept: %d", w.Code)
	224	+	}
	225	+	}
	226	+
	227	+	// MINOR: a rate-limited upstream (429) is unavailable: a known tag is served
	228	+	// stale; an unknown one is an error.
	229	+	func TestRateLimitedServesStaleTag(t *testing.T) {
	230	+	f := fake(t)
	231	+	p := proxyFor(t, f)
	232	+	if w := request(t, p, http.MethodGet, "/v2/library/alpine/manifests/3"); w.Code != http.StatusOK {
	233	+	t.Fatal(w.Code)
	234	+	}
	235	+	f.status.Store(http.StatusTooManyRequests)
	236	+	if w := request(t, p, http.MethodGet, "/v2/library/alpine/manifests/3"); w.Code != http.StatusOK \|\| w.Body.String() != manifestBody {
	237	+	t.Fatalf("stale on 429: %d", w.Code)
	238	+	}
	239	+	if w := request(t, p, http.MethodGet, "/v2/library/alpine/manifests/other"); w.Code != http.StatusBadGateway {
	240	+	t.Fatalf("unknown on 429: %d", w.Code)
	241	+	}
	242	+	}
	243	+
	244	+	// MINOR: eviction walks the LRU, evicts least recently used first, frees down
	245	+	// to the low-water mark rather than the bare need, and skips pinned entries.
	246	+	func TestEvictionLRUAndLowWater(t *testing.T) {
	247	+	now := time.Unix(1_700_000_000, 0)
	248	+	const max = 2000 // low water: 5% = 100 bytes
	249	+	s, err := openStore(t.TempDir(), max, func() time.Time { return now })
	250	+	if err != nil {
	251	+	t.Fatal(err)
	252	+	}
	253	+	// 40 blobs of 50 bytes fill the store. Freeing only the need (1 byte) would
	254	+	// evict one blob; the low-water mark (100 bytes) evicts two.
	255	+	const n = 40
	256	+	blob := func(i int) string { return fmt.Sprintf("%03d", i) + strings.Repeat("x", 47) }
	257	+	for i := range n {
	258	+	now = now.Add(time.Second)
	259	+	put(t, s, blob(i))
	260	+	}
	261	+	// Touch 0 and 1, and keep 2 open: the least recently used are then 3, 4...
	262	+	for _, i := range []int{0, 1} {
	263	+	now = now.Add(time.Second)
	264	+	f, _, err := s.open("blobs", sum(blob(i)))
	265	+	if err != nil {
	266	+	t.Fatal(err)
	267	+	}
	268	+	_ = f.Close()
	269	+	}
	270	+	pinned, _, err := s.open("blobs", sum(blob(2)))
	271	+	if err != nil {
	272	+	t.Fatal(err)
	273	+	}
	274	+	defer func() { _ = pinned.Close() }()
	275	+	s.mu.Lock()
	276	+	s.lru.MoveToFront(s.entries["blobs/"+sum(blob(2))[7:]].elem) // oldest of all, but open
	277	+	s.mu.Unlock()
	278	+	now = now.Add(time.Second)
	279	+	put(t, s, "a") // 1 byte over the cap: frees max(1, 100) bytes
	280	+	present := func(i int) bool {
	281	+	_, err := os.Stat(filepath.Join(s.dir, "blobs", sum(blob(i))[7:]))
	282	+	return err == nil
	283	+	}
	284	+	for i := range n {
	285	+	want := i != 3 && i != 4 // 3 and 4 are the least recently used unpinned
	286	+	if present(i) != want {
	287	+	t.Errorf("blob %d present=%v", i, present(i))
	288	+	}
	289	+	}
	290	+	accounted(t, s)
	291	+	s.mu.Lock()
	292	+	free := s.max - s.total
	293	+	s.mu.Unlock()
	294	+	if free < min(lowWater, int64(max)/20)-1 {
	295	+	t.Fatalf("freed only to %d", free)
	296	+	}
	297	+	}
	298	+
	299	+	// MINOR: the tag index follows rewrites, and removing a manifest takes exactly
	300	+	// its own tags along.
	301	+	func TestTagIndex(t *testing.T) {
	302	+	s, err := openStore(t.TempDir(), 1<<20, time.Now)
	303	+	if err != nil {
	304	+	t.Fatal(err)
	305	+	}
	306	+	manifests := []string{`{"schemaVersion":2,"a":1}`, `{"schemaVersion":2,"b":1}`}
	307	+	for _, m := range manifests {
	308	+	f, _, _, err := s.ensure(context.Background(), "manifests", sum(m), func(_ context.Context, w io.Writer) (string, error) {
	309	+	_, err := io.WriteString(w, m)
	310	+	return "application/vnd.oci.image.index.v1+json", err
	311	+	}, keepalive{})
	312	+	if err != nil {
	313	+	t.Fatal(err)
	314	+	}
	315	+	_ = f.Close()
	316	+	}
	317	+	for _, tag := range []string{"x", "y"} {
	318	+	if err := s.setTag("library/alpine", tag, "", sum(manifests[0])); err != nil {
	319	+	t.Fatal(err)
	320	+	}
	321	+	}
	322	+	// y moves to the second manifest.
	323	+	if err := s.setTag("library/alpine", "y", "", sum(manifests[1])); err != nil {
	324	+	t.Fatal(err)
	325	+	}
	326	+	s.mu.Lock()
	327	+	if !s.removeLocked("manifests/" + sum(manifests[0])[7:]) {
	328	+	t.Fatal("remove failed")
	329	+	}
	330	+	n0, n1 := len(s.tagsOf[sum(manifests[0])]), len(s.tagsOf[sum(manifests[1])])
	331	+	s.mu.Unlock()
	332	+	if s.tag("library/alpine", "x", "") != "" \|\| s.tag("library/alpine", "y", "") != sum(manifests[1]) \|\| n0 != 0 \|\| n1 != 1 {
	333	+	t.Fatalf("index: %d %d", n0, n1)
	334	+	}
	335	+	accounted(t, s)
	336	+	}
	337	+
	338	+	// NIT: the committed entry is pinned for its waiters, so an eviction between
	339	+	// the commit and their open cannot remove it.
	340	+	func TestCommittedEntryPinnedForWaiters(t *testing.T) {
	341	+	s, err := openStore(t.TempDir(), 8, time.Now)
	342	+	if err != nil {
	343	+	t.Fatal(err)
	344	+	}
	345	+	s.hookCommitted = func(string) {
	346	+	// Ask for the whole store before the waiters are woken.
	347	+	s.mu.Lock()
	348	+	s.evictLocked(s.max)
	349	+	s.mu.Unlock()
	350	+	}
	351	+	f, _, _, err := s.ensure(context.Background(), "blobs", sum("aaaa"), func(_ context.Context, w io.Writer) (string, error) {
	352	+	_, err := io.WriteString(w, "aaaa")
	353	+	return "", err
	354	+	}, keepalive{})
	355	+	if err != nil {
	356	+	t.Fatalf("waiter lost its entry: %v", err)
	357	+	}
	358	+	s.mu.Lock()
	359	+	pins := s.entries["blobs/"+sum("aaaa")[7:]].pins
	360	+	s.mu.Unlock()
	361	+	if pins != 1 {
	362	+	t.Fatalf("pins %d: the commit pin was not released to the reader's", pins)
	363	+	}
	364	+	_ = f.Close()
	365	+	s.hookCommitted = nil
	366	+	s.mu.Lock()
	367	+	s.evictLocked(s.max)
	368	+	_, kept := s.entries["blobs/"+sum("aaaa")[7:]]
	369	+	s.mu.Unlock()
	370	+	if kept {
	371	+	t.Fatal("unpinned entry not evictable")
	372	+	}
	373	+	}
	374	+
	375	+	// NIT: a request that joins a manifest flight logs hit=false.
	376	+	func TestJoinerLogsMiss(t *testing.T) {
	377	+	release := make(chan struct{})
	378	+	srv := upstreamWith(t, func(w http.ResponseWriter, r *http.Request) {
	379	+	w.Header().Set("Content-Type", "application/vnd.oci.image.index.v1+json")
	380	+	w.Header().Set("Docker-Content-Digest", sum(manifestBody))
	381	+	if r.Method == http.MethodGet {
	382	+	<-release
	383	+	_, _ = io.WriteString(w, manifestBody)
	384	+	}
	385	+	})
	386	+	var logs syncBuffer
	387	+	p := proxyWith(t, srv, Config{Log: zerowrap.New(zerowrap.Config{Level: "info", Format: "json", Output: &logs})})
	388	+	var wg sync.WaitGroup
	389	+	for range 2 {
	390	+	wg.Go(func() { request(t, p, http.MethodGet, "/v2/library/alpine/manifests/3") })
	391	+	}
	392	+	key := "manifests/" + sum(manifestBody)[7:]
	393	+	waitFor(t, p.s, "two waiters", func() bool { return waiters(p.s, key) == 2 })
	394	+	close(release)
	395	+	wg.Wait()
	396	+	if out := logs.String(); strings.Count(out, `"hit":false`) != 2 \|\| strings.Contains(out, `"hit":true`) {
	397	+	t.Fatalf("logs: %s", out)
	398	+	}
	399	+	}
	400	+
	401	+	type syncBuffer struct {
	402	+	mu sync.Mutex
	403	+	b bytes.Buffer
	404	+	}
	405	+
	406	+	func (s *syncBuffer) Write(p []byte) (int, error) {
	407	+	s.mu.Lock()
	408	+	defer s.mu.Unlock()
	409	+	return s.b.Write(p)
	410	+	}
	411	+	func (s *syncBuffer) String() string { s.mu.Lock(); defer s.mu.Unlock(); return s.b.String() }
	412	+
	413	+	// NIT: a .type file without its manifest is removed at startup.
	414	+	func TestOrphanTypeRemoved(t *testing.T) {
	415	+	dir := t.TempDir()
	416	+	if _, err := openStore(dir, 1<<20, time.Now); err != nil {
	417	+	t.Fatal(err)
	418	+	}
	419	+	orphan := filepath.Join(dir, "manifests", sum("gone")[7:]+".type")
	420	+	if err := os.WriteFile(orphan, []byte("application/json"), 0600); err != nil {
	421	+	t.Fatal(err)
	422	+	}
	423	+	s, err := openStore(dir, 1<<20, time.Now)
	424	+	if err != nil {
	425	+	t.Fatal(err)
	426	+	}
	427	+	if _, err := os.Stat(orphan); !os.IsNotExist(err) {
	428	+	t.Fatal("orphan .type kept")
	429	+	}
	430	+	accounted(t, s)
	431	+	}

@@ -0,0 +1,431 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+	package registryproxy
	3	+
	4	+	import (
	5	+	"bytes"
	6	+	"context"
	7	+	"fmt"
	8	+	"io"
	9	+	"net/http"
	10	+	"net/http/httptest"
	11	+	"os"
	12	+	"path/filepath"
	13	+	"strings"
	14	+	"sync"
	15	+	"sync/atomic"
	16	+	"testing"
	17	+	"time"
	18	+
	19	+	"github.com/bnema/zerowrap"
	20	+	)
	21	+
	22	+	// stallingUpstream serves a blob's first bytes and then holds the body until
	23	+	// the request is cancelled; it counts started and cancelled downloads.
	24	+	type stallingUpstream struct {
	25	+	srv *httptest.Server
	26	+	started, cancelled atomic.Int32
	27	+	}
	28	+
	29	+	func newStallingUpstream(t testing.T) stallingUpstream {
	30	+	t.Helper()
	31	+	u := &stallingUpstream{}
	32	+	u.srv = upstreamWith(t, func(w http.ResponseWriter, r *http.Request) {
	33	+	u.started.Add(1)
	34	+	w.Header().Set("Content-Length", "1000000")
	35	+	_, _ = io.WriteString(w, "partial")
	36	+	w.(http.Flusher).Flush()
	37	+	select {
	38	+	case <-r.Context().Done():
	39	+	u.cancelled.Add(1)
	40	+	case <-time.After(10 * time.Second):
	41	+	}
	42	+	})
	43	+	return u
	44	+	}
	45	+
	46	+	func blobPath(i int) string { return "/v2/library/alpine/blobs/" + sum(fmt.Sprint("blob-", i)) }
	47	+
	48	+	// serveCancelled starts a request and cancels it once its download has begun.
	49	+	func serveCancelled(t testing.T, p Proxy, u *stallingUpstream, path string, started int32) {
	50	+	t.Helper()
	51	+	ctx, cancel := context.WithCancel(context.Background())
	52	+	done := make(chan struct{})
	53	+	go func() {
	54	+	defer close(done)
	55	+	p.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, path, nil).WithContext(ctx))
	56	+	}()
	57	+	waitFor(t, p.s, "the download to start", func() bool { return u.started.Load() >= started })
	58	+	cancel()
	59	+	<-done
	60	+	}
	61	+
	62	+	// MAJOR 1a: a guest that requests many digests and disconnects leaves no
	63	+	// download behind: once a flight has no waiter for the grace period, it is
	64	+	// cancelled upstream and its temporary file removed.
	65	+	func TestAbandonedDownloadsCancelled(t *testing.T) {
	66	+	u := newStallingUpstream(t)
	67	+	const n = 20
	68	+	p := proxyWith(t, u.srv, Config{AbandonGrace: 20 * time.Millisecond, MaxDownloads: n})
	69	+	for i := range n {
	70	+	serveCancelled(t, p, u, blobPath(i), int32(i+1))
	71	+	}
	72	+	waitFor(t, p.s, "every flight to end", func() bool { return len(p.s.flights) == 0 && p.s.downloads == 0 && p.s.pending == 0 })
	73	+	for end := time.Now().Add(5 * time.Second); u.cancelled.Load() != n; {
	74	+	if time.Now().After(end) {
	75	+	t.Fatalf("upstream saw %d cancellations of %d", u.cancelled.Load(), n)
	76	+	}
	77	+	time.Sleep(time.Millisecond)
	78	+	}
	79	+	waitFor(t, p.s, "tmp/ to be empty", func() bool {
	80	+	ds, _ := os.ReadDir(filepath.Join(p.s.dir, "tmp"))
	81	+	return len(ds) == 0
	82	+	})
	83	+	}
	84	+
	85	+	// MAJOR 1a: a request that joins within the grace period keeps the download.
	86	+	func TestRejoinWithinGraceKeepsDownload(t *testing.T) {
	87	+	body := strings.Repeat("layer", 4000)
	88	+	release := make(chan struct{})
	89	+	var gets, cancelled atomic.Int32
	90	+	srv := upstreamWith(t, func(w http.ResponseWriter, r *http.Request) {
	91	+	gets.Add(1)
	92	+	w.Header().Set("Content-Length", fmt.Sprint(len(body)))
	93	+	_, _ = io.WriteString(w, body[:10])
	94	+	w.(http.Flusher).Flush()
	95	+	select {
	96	+	case <-release:
	97	+	_, _ = io.WriteString(w, body[10:])
	98	+	case <-r.Context().Done():
	99	+	cancelled.Add(1)
	100	+	}
	101	+	})
	102	+	var once sync.Once
	103	+	t.Cleanup(func() { once.Do(func() { close(release) }) })
	104	+	p := proxyWith(t, srv, Config{AbandonGrace: time.Hour})
	105	+	path := "/v2/library/alpine/blobs/" + sum(body)
	106	+	ctx, cancel := context.WithCancel(context.Background())
	107	+	first := make(chan struct{})
	108	+	go func() {
	109	+	defer close(first)
	110	+	p.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, path, nil).WithContext(ctx))
	111	+	}()
	112	+	key := "blobs/" + sum(body)[7:]
	113	+	waitFor(t, p.s, "the first waiter", func() bool { return waiters(p.s, key) == 1 })
	114	+	cancel()
	115	+	<-first
	116	+	w := httptest.NewRecorder()
	117	+	second := make(chan struct{})
	118	+	go func() { defer close(second); p.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) }()
	119	+	waitFor(t, p.s, "the second waiter", func() bool { return waiters(p.s, key) == 1 })
	120	+	once.Do(func() { close(release) })
	121	+	<-second
	122	+	if w.Code != http.StatusOK \|\| w.Body.String() != body \|\| gets.Load() != 1 \|\| cancelled.Load() != 0 {
	123	+	t.Fatalf("code %d, %d fetches, %d cancelled", w.Code, gets.Load(), cancelled.Load())
	124	+	}
	125	+	}
	126	+
	127	+	// MAJOR 1b: at most MaxDownloads cold blobs download at once; one more waits
	128	+	// for SlotWait and then gets 503 (dockerd retries or pulls from Docker Hub),
	129	+	// and a slot frees once a download ends.
	130	+	func TestColdDownloadCap(t *testing.T) {
	131	+	u := newStallingUpstream(t)
	132	+	const limit = 3
	133	+	const wait = 50 * time.Millisecond
	134	+	p := proxyWith(t, u.srv, Config{AbandonGrace: 10 * time.Millisecond, MaxDownloads: limit, SlotWait: wait})
	135	+	ctx, cancel := context.WithCancel(context.Background())
	136	+	var wg sync.WaitGroup
	137	+	for i := range limit {
	138	+	wg.Go(func() {
	139	+	p.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, blobPath(i), nil).WithContext(ctx))
	140	+	})
	141	+	}
	142	+	waitFor(t, p.s, "the cap to fill", func() bool { return p.s.downloads == limit })
	143	+	start := time.Now()
	144	+	w := request(t, p, http.MethodGet, blobPath(limit))
	145	+	if w.Code != http.StatusServiceUnavailable \|\| w.Header().Get("Retry-After") == "" {
	146	+	t.Fatalf("over the cap: %d", w.Code)
	147	+	}
	148	+	if d := time.Since(start); d < wait {
	149	+	t.Fatalf("503 after %v, before the slot wait", d)
	150	+	}
	151	+	// Joining a running download takes no slot.
	152	+	joined := make(chan int, 1)
	153	+	go func() {
	154	+	jctx, jcancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
	155	+	defer jcancel()
	156	+	jw := httptest.NewRecorder()
	157	+	p.ServeHTTP(jw, httptest.NewRequest(http.MethodGet, blobPath(0), nil).WithContext(jctx))
	158	+	joined <- jw.Code
	159	+	}()
	160	+	if code := <-joined; code == http.StatusServiceUnavailable {
	161	+	t.Fatal("a joiner was refused a slot")
	162	+	}
	163	+	cancel()
	164	+	wg.Wait()
	165	+	waitFor(t, p.s, "the slots to free", func() bool { return p.s.downloads == 0 })
	166	+	if u.started.Load() != limit {
	167	+	t.Fatalf("%d downloads started", u.started.Load())
	168	+	}
	169	+	}
	170	+
	171	+	// MAJOR 2: dockerd's legacy client sends one Accept line per type. All of them
	172	+	// reach upstream, and they make one tag file.
	173	+	func TestMultiLineAccept(t *testing.T) {
	174	+	types := []string{
	175	+	"application/vnd.oci.image.index.v1+json",
	176	+	"application/vnd.docker.distribution.manifest.list.v2+json",
	177	+	"application/vnd.docker.distribution.manifest.v2+json",
	178	+	}
	179	+	var mu sync.Mutex
	180	+	var seen []string
	181	+	srv := upstreamWith(t, func(w http.ResponseWriter, r *http.Request) {
	182	+	mu.Lock()
	183	+	seen = append(seen, strings.Join(r.Header.Values("Accept"), ","))
	184	+	mu.Unlock()
	185	+	w.Header().Set("Content-Type", types[0])
	186	+	w.Header().Set("Docker-Content-Digest", sum(manifestBody))
	187	+	if r.Method == http.MethodGet {
	188	+	_, _ = io.WriteString(w, manifestBody)
	189	+	}
	190	+	})
	191	+	p := proxyWith(t, srv, Config{})
	192	+	for range 2 {
	193	+	r := httptest.NewRequest(http.MethodGet, "/v2/library/alpine/manifests/3", nil)
	194	+	for _, typ := range types {
	195	+	r.Header.Add("Accept", typ)
	196	+	}
	197	+	w := httptest.NewRecorder()
	198	+	p.ServeHTTP(w, r)
	199	+	if w.Code != http.StatusOK {
	200	+	t.Fatal(w.Code)
	201	+	}
	202	+	}
	203	+	mu.Lock()
	204	+	for _, got := range seen {
	205	+	for _, typ := range types {
	206	+	if !strings.Contains(got, typ) {
	207	+	t.Errorf("upstream Accept %q lacks %s", got, typ)
	208	+	}
	209	+	}
	210	+	}
	211	+	mu.Unlock()
	212	+	if n := accounted(t, p.s); n != 1 {
	213	+	t.Fatalf("%d tag files", n)
	214	+	}
	215	+	// The 8 KiB bound applies to the joined lines.
	216	+	r := httptest.NewRequest(http.MethodGet, "/v2/library/alpine/manifests/3", nil)
	217	+	for range 200 {
	218	+	r.Header.Add("Accept", strings.Repeat("x", 50))
	219	+	}
	220	+	w := httptest.NewRecorder()
	221	+	p.ServeHTTP(w, r)
	222	+	if w.Code != http.StatusNotFound {
	223	+	t.Fatalf("oversized Accept: %d", w.Code)
	224	+	}
	225	+	}
	226	+
	227	+	// MINOR: a rate-limited upstream (429) is unavailable: a known tag is served
	228	+	// stale; an unknown one is an error.
	229	+	func TestRateLimitedServesStaleTag(t *testing.T) {
	230	+	f := fake(t)
	231	+	p := proxyFor(t, f)
	232	+	if w := request(t, p, http.MethodGet, "/v2/library/alpine/manifests/3"); w.Code != http.StatusOK {
	233	+	t.Fatal(w.Code)
	234	+	}
	235	+	f.status.Store(http.StatusTooManyRequests)
	236	+	if w := request(t, p, http.MethodGet, "/v2/library/alpine/manifests/3"); w.Code != http.StatusOK \|\| w.Body.String() != manifestBody {
	237	+	t.Fatalf("stale on 429: %d", w.Code)
	238	+	}
	239	+	if w := request(t, p, http.MethodGet, "/v2/library/alpine/manifests/other"); w.Code != http.StatusBadGateway {
	240	+	t.Fatalf("unknown on 429: %d", w.Code)
	241	+	}
	242	+	}
	243	+
	244	+	// MINOR: eviction walks the LRU, evicts least recently used first, frees down
	245	+	// to the low-water mark rather than the bare need, and skips pinned entries.
	246	+	func TestEvictionLRUAndLowWater(t *testing.T) {
	247	+	now := time.Unix(1_700_000_000, 0)
	248	+	const max = 2000 // low water: 5% = 100 bytes
	249	+	s, err := openStore(t.TempDir(), max, func() time.Time { return now })
	250	+	if err != nil {
	251	+	t.Fatal(err)
	252	+	}
	253	+	// 40 blobs of 50 bytes fill the store. Freeing only the need (1 byte) would
	254	+	// evict one blob; the low-water mark (100 bytes) evicts two.
	255	+	const n = 40
	256	+	blob := func(i int) string { return fmt.Sprintf("%03d", i) + strings.Repeat("x", 47) }
	257	+	for i := range n {
	258	+	now = now.Add(time.Second)
	259	+	put(t, s, blob(i))
	260	+	}
	261	+	// Touch 0 and 1, and keep 2 open: the least recently used are then 3, 4...
	262	+	for _, i := range []int{0, 1} {
	263	+	now = now.Add(time.Second)
	264	+	f, _, err := s.open("blobs", sum(blob(i)))
	265	+	if err != nil {
	266	+	t.Fatal(err)
	267	+	}
	268	+	_ = f.Close()
	269	+	}
	270	+	pinned, _, err := s.open("blobs", sum(blob(2)))
	271	+	if err != nil {
	272	+	t.Fatal(err)
	273	+	}
	274	+	defer func() { _ = pinned.Close() }()
	275	+	s.mu.Lock()
	276	+	s.lru.MoveToFront(s.entries["blobs/"+sum(blob(2))[7:]].elem) // oldest of all, but open
	277	+	s.mu.Unlock()
	278	+	now = now.Add(time.Second)
	279	+	put(t, s, "a") // 1 byte over the cap: frees max(1, 100) bytes
	280	+	present := func(i int) bool {
	281	+	_, err := os.Stat(filepath.Join(s.dir, "blobs", sum(blob(i))[7:]))
	282	+	return err == nil
	283	+	}
	284	+	for i := range n {
	285	+	want := i != 3 && i != 4 // 3 and 4 are the least recently used unpinned
	286	+	if present(i) != want {
	287	+	t.Errorf("blob %d present=%v", i, present(i))
	288	+	}
	289	+	}
	290	+	accounted(t, s)
	291	+	s.mu.Lock()
	292	+	free := s.max - s.total
	293	+	s.mu.Unlock()
	294	+	if free < min(lowWater, int64(max)/20)-1 {
	295	+	t.Fatalf("freed only to %d", free)
	296	+	}
	297	+	}
	298	+
	299	+	// MINOR: the tag index follows rewrites, and removing a manifest takes exactly
	300	+	// its own tags along.
	301	+	func TestTagIndex(t *testing.T) {
	302	+	s, err := openStore(t.TempDir(), 1<<20, time.Now)
	303	+	if err != nil {
	304	+	t.Fatal(err)
	305	+	}
	306	+	manifests := []string{`{"schemaVersion":2,"a":1}`, `{"schemaVersion":2,"b":1}`}
	307	+	for _, m := range manifests {
	308	+	f, _, _, err := s.ensure(context.Background(), "manifests", sum(m), func(_ context.Context, w io.Writer) (string, error) {
	309	+	_, err := io.WriteString(w, m)
	310	+	return "application/vnd.oci.image.index.v1+json", err
	311	+	}, keepalive{})
	312	+	if err != nil {
	313	+	t.Fatal(err)
	314	+	}
	315	+	_ = f.Close()
	316	+	}
	317	+	for _, tag := range []string{"x", "y"} {
	318	+	if err := s.setTag("library/alpine", tag, "", sum(manifests[0])); err != nil {
	319	+	t.Fatal(err)
	320	+	}
	321	+	}
	322	+	// y moves to the second manifest.
	323	+	if err := s.setTag("library/alpine", "y", "", sum(manifests[1])); err != nil {
	324	+	t.Fatal(err)
	325	+	}
	326	+	s.mu.Lock()
	327	+	if !s.removeLocked("manifests/" + sum(manifests[0])[7:]) {
	328	+	t.Fatal("remove failed")
	329	+	}
	330	+	n0, n1 := len(s.tagsOf[sum(manifests[0])]), len(s.tagsOf[sum(manifests[1])])
	331	+	s.mu.Unlock()
	332	+	if s.tag("library/alpine", "x", "") != "" \|\| s.tag("library/alpine", "y", "") != sum(manifests[1]) \|\| n0 != 0 \|\| n1 != 1 {
	333	+	t.Fatalf("index: %d %d", n0, n1)
	334	+	}
	335	+	accounted(t, s)
	336	+	}
	337	+
	338	+	// NIT: the committed entry is pinned for its waiters, so an eviction between
	339	+	// the commit and their open cannot remove it.
	340	+	func TestCommittedEntryPinnedForWaiters(t *testing.T) {
	341	+	s, err := openStore(t.TempDir(), 8, time.Now)
	342	+	if err != nil {
	343	+	t.Fatal(err)
	344	+	}
	345	+	s.hookCommitted = func(string) {
	346	+	// Ask for the whole store before the waiters are woken.
	347	+	s.mu.Lock()
	348	+	s.evictLocked(s.max)
	349	+	s.mu.Unlock()
	350	+	}
	351	+	f, _, _, err := s.ensure(context.Background(), "blobs", sum("aaaa"), func(_ context.Context, w io.Writer) (string, error) {
	352	+	_, err := io.WriteString(w, "aaaa")
	353	+	return "", err
	354	+	}, keepalive{})
	355	+	if err != nil {
	356	+	t.Fatalf("waiter lost its entry: %v", err)
	357	+	}
	358	+	s.mu.Lock()
	359	+	pins := s.entries["blobs/"+sum("aaaa")[7:]].pins
	360	+	s.mu.Unlock()
	361	+	if pins != 1 {
	362	+	t.Fatalf("pins %d: the commit pin was not released to the reader's", pins)
	363	+	}
	364	+	_ = f.Close()
	365	+	s.hookCommitted = nil
	366	+	s.mu.Lock()
	367	+	s.evictLocked(s.max)
	368	+	_, kept := s.entries["blobs/"+sum("aaaa")[7:]]
	369	+	s.mu.Unlock()
	370	+	if kept {
	371	+	t.Fatal("unpinned entry not evictable")
	372	+	}
	373	+	}
	374	+
	375	+	// NIT: a request that joins a manifest flight logs hit=false.
	376	+	func TestJoinerLogsMiss(t *testing.T) {
	377	+	release := make(chan struct{})
	378	+	srv := upstreamWith(t, func(w http.ResponseWriter, r *http.Request) {
	379	+	w.Header().Set("Content-Type", "application/vnd.oci.image.index.v1+json")
	380	+	w.Header().Set("Docker-Content-Digest", sum(manifestBody))
	381	+	if r.Method == http.MethodGet {
	382	+	<-release
	383	+	_, _ = io.WriteString(w, manifestBody)
	384	+	}
	385	+	})
	386	+	var logs syncBuffer
	387	+	p := proxyWith(t, srv, Config{Log: zerowrap.New(zerowrap.Config{Level: "info", Format: "json", Output: &logs})})
	388	+	var wg sync.WaitGroup
	389	+	for range 2 {
	390	+	wg.Go(func() { request(t, p, http.MethodGet, "/v2/library/alpine/manifests/3") })
	391	+	}
	392	+	key := "manifests/" + sum(manifestBody)[7:]
	393	+	waitFor(t, p.s, "two waiters", func() bool { return waiters(p.s, key) == 2 })
	394	+	close(release)
	395	+	wg.Wait()
	396	+	if out := logs.String(); strings.Count(out, `"hit":false`) != 2 \|\| strings.Contains(out, `"hit":true`) {
	397	+	t.Fatalf("logs: %s", out)
	398	+	}
	399	+	}
	400	+
	401	+	type syncBuffer struct {
	402	+	mu sync.Mutex
	403	+	b bytes.Buffer
	404	+	}
	405	+
	406	+	func (s *syncBuffer) Write(p []byte) (int, error) {
	407	+	s.mu.Lock()
	408	+	defer s.mu.Unlock()
	409	+	return s.b.Write(p)
	410	+	}
	411	+	func (s *syncBuffer) String() string { s.mu.Lock(); defer s.mu.Unlock(); return s.b.String() }
	412	+
	413	+	// NIT: a .type file without its manifest is removed at startup.
	414	+	func TestOrphanTypeRemoved(t *testing.T) {
	415	+	dir := t.TempDir()
	416	+	if _, err := openStore(dir, 1<<20, time.Now); err != nil {
	417	+	t.Fatal(err)
	418	+	}
	419	+	orphan := filepath.Join(dir, "manifests", sum("gone")[7:]+".type")
	420	+	if err := os.WriteFile(orphan, []byte("application/json"), 0600); err != nil {
	421	+	t.Fatal(err)
	422	+	}
	423	+	s, err := openStore(dir, 1<<20, time.Now)
	424	+	if err != nil {
	425	+	t.Fatal(err)
	426	+	}
	427	+	if _, err := os.Stat(orphan); !os.IsNotExist(err) {
	428	+	t.Fatal("orphan .type kept")
	429	+	}
	430	+	accounted(t, s)
	431	+	}

A internal/registryproxy/hub_test.go +78 -0

@@ -0,0 +1,78 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+	package registryproxy
	3	+
	4	+	import (
	5	+	"bytes"
	6	+	"encoding/json"
	7	+	"github.com/bnema/zerowrap"
	8	+	"net/http/httptest"
	9	+	"os"
	10	+	"os/exec"
	11	+	"strings"
	12	+	"testing"
	13	+	)
	14	+
	15	+	// An opt-in manual check using curl against a real local HTTP listener, with
	16	+	// Docker Hub as upstream. No daemon is required.
	17	+	func TestDockerHubCurl(t *testing.T) {
	18	+	if os.Getenv("ERMITE_RUNNER_IT_REGISTRY") != "1" {
	19	+	t.Skip("set ERMITE_RUNNER_IT_REGISTRY=1 for live Docker Hub curl check")
	20	+	}
	21	+	var logs bytes.Buffer
	22	+	p, err := New(Config{Dir: t.TempDir(), Max: 64 << 20, Log: zerowrap.New(zerowrap.Config{Level: "debug", Format: "json", Output: &logs})})
	23	+	if err != nil {
	24	+	t.Fatal(err)
	25	+	}
	26	+	srv := httptest.NewServer(p)
	27	+	defer srv.Close()
	28	+	curl := func(path string) []byte {
	29	+	t.Helper()
	30	+	out, err := exec.Command("curl", "--max-time", "20", "-fsS", "-H", "Accept: application/vnd.oci.image.index.v1+json, application/vnd.oci.image.manifest.v1+json", srv.URL+path).CombinedOutput()
	31	+	if err != nil {
	32	+	t.Fatalf("curl %s: %v: %s", path, err, out)
	33	+	}
	34	+	t.Logf("curl %s: HTTP 200, %d bytes", path, len(out))
	35	+	return out
	36	+	}
	37	+	var idx struct {
	38	+	Manifests []struct {
	39	+	Digest string
	40	+	Platform struct{ Architecture, OS string }
	41	+	}
	42	+	}
	43	+	b := curl("/v2/library/alpine/manifests/3")
	44	+	if err := json.Unmarshal(b, &idx); err != nil {
	45	+	t.Fatal(err)
	46	+	}
	47	+	if !bytes.Equal(b, curl("/v2/library/alpine/manifests/3")) {
	48	+	t.Fatal("tag changed during check")
	49	+	}
	50	+	var digest string
	51	+	for _, m := range idx.Manifests {
	52	+	if m.Platform.Architecture == "amd64" && m.Platform.OS == "linux" {
	53	+	digest = m.Digest
	54	+	break
	55	+	}
	56	+	}
	57	+	if !validDigest(digest) {
	58	+	t.Fatal("no linux/amd64 manifest")
	59	+	}
	60	+	var manifest struct{ Config struct{ Digest string } }
	61	+	if err := json.Unmarshal(curl("/v2/library/alpine/manifests/"+digest), &manifest); err != nil {
	62	+	t.Fatal(err)
	63	+	}
	64	+	if !validDigest(manifest.Config.Digest) {
	65	+	t.Fatal("no config blob")
	66	+	}
	67	+	path := "/v2/library/alpine/blobs/" + manifest.Config.Digest
	68	+	first := curl(path)
	69	+	second := curl(path)
	70	+	if !bytes.Equal(first, second) {
	71	+	t.Fatal("blob differs")
	72	+	}
	73	+	all := logs.String()
	74	+	t.Log(all)
	75	+	if !strings.Contains(all, `"hit":true`) \|\| strings.Count(all, `"message":"registry blob"`) != 1 {
	76	+	t.Fatal("missing cache hits or repeated blob miss")
	77	+	}
	78	+	}

@@ -0,0 +1,78 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+	package registryproxy
	3	+
	4	+	import (
	5	+	"bytes"
	6	+	"encoding/json"
	7	+	"github.com/bnema/zerowrap"
	8	+	"net/http/httptest"
	9	+	"os"
	10	+	"os/exec"
	11	+	"strings"
	12	+	"testing"
	13	+	)
	14	+
	15	+	// An opt-in manual check using curl against a real local HTTP listener, with
	16	+	// Docker Hub as upstream. No daemon is required.
	17	+	func TestDockerHubCurl(t *testing.T) {
	18	+	if os.Getenv("ERMITE_RUNNER_IT_REGISTRY") != "1" {
	19	+	t.Skip("set ERMITE_RUNNER_IT_REGISTRY=1 for live Docker Hub curl check")
	20	+	}
	21	+	var logs bytes.Buffer
	22	+	p, err := New(Config{Dir: t.TempDir(), Max: 64 << 20, Log: zerowrap.New(zerowrap.Config{Level: "debug", Format: "json", Output: &logs})})
	23	+	if err != nil {
	24	+	t.Fatal(err)
	25	+	}
	26	+	srv := httptest.NewServer(p)
	27	+	defer srv.Close()
	28	+	curl := func(path string) []byte {
	29	+	t.Helper()
	30	+	out, err := exec.Command("curl", "--max-time", "20", "-fsS", "-H", "Accept: application/vnd.oci.image.index.v1+json, application/vnd.oci.image.manifest.v1+json", srv.URL+path).CombinedOutput()
	31	+	if err != nil {
	32	+	t.Fatalf("curl %s: %v: %s", path, err, out)
	33	+	}
	34	+	t.Logf("curl %s: HTTP 200, %d bytes", path, len(out))
	35	+	return out
	36	+	}
	37	+	var idx struct {
	38	+	Manifests []struct {
	39	+	Digest string
	40	+	Platform struct{ Architecture, OS string }
	41	+	}
	42	+	}
	43	+	b := curl("/v2/library/alpine/manifests/3")
	44	+	if err := json.Unmarshal(b, &idx); err != nil {
	45	+	t.Fatal(err)
	46	+	}
	47	+	if !bytes.Equal(b, curl("/v2/library/alpine/manifests/3")) {
	48	+	t.Fatal("tag changed during check")
	49	+	}
	50	+	var digest string
	51	+	for _, m := range idx.Manifests {
	52	+	if m.Platform.Architecture == "amd64" && m.Platform.OS == "linux" {
	53	+	digest = m.Digest
	54	+	break
	55	+	}
	56	+	}
	57	+	if !validDigest(digest) {
	58	+	t.Fatal("no linux/amd64 manifest")
	59	+	}
	60	+	var manifest struct{ Config struct{ Digest string } }
	61	+	if err := json.Unmarshal(curl("/v2/library/alpine/manifests/"+digest), &manifest); err != nil {
	62	+	t.Fatal(err)
	63	+	}
	64	+	if !validDigest(manifest.Config.Digest) {
	65	+	t.Fatal("no config blob")
	66	+	}
	67	+	path := "/v2/library/alpine/blobs/" + manifest.Config.Digest
	68	+	first := curl(path)
	69	+	second := curl(path)
	70	+	if !bytes.Equal(first, second) {
	71	+	t.Fatal("blob differs")
	72	+	}
	73	+	all := logs.String()
	74	+	t.Log(all)
	75	+	if !strings.Contains(all, `"hit":true`) \|\| strings.Count(all, `"message":"registry blob"`) != 1 {
	76	+	t.Fatal("missing cache hits or repeated blob miss")
	77	+	}
	78	+	}

A internal/registryproxy/limits_test.go +494 -0

@@ -0,0 +1,494 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+	package registryproxy
	3	+
	4	+	import (
	5	+	"context"
	6	+	"errors"
	7	+	"fmt"
	8	+	"io"
	9	+	"net/http"
	10	+	"net/http/httptest"
	11	+	"os"
	12	+	"path/filepath"
	13	+	"strings"
	14	+	"sync"
	15	+	"sync/atomic"
	16	+	"testing"
	17	+	"time"
	18	+	)
	19	+
	20	+	// upstreamWith fakes Docker Hub: /token answers an anonymous token and h serves
	21	+	// everything else.
	22	+	func upstreamWith(t testing.T, h http.HandlerFunc) httptest.Server {
	23	+	t.Helper()
	24	+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	25	+	if r.URL.Path == "/token" {
	26	+	_, _ = io.WriteString(w, `{"token":"public","expires_in":300}`)
	27	+	return
	28	+	}
	29	+	h(w, r)
	30	+	}))
	31	+	t.Cleanup(srv.Close)
	32	+	return srv
	33	+	}
	34	+
	35	+	func proxyWith(t testing.T, srv httptest.Server, c Config) *Proxy {
	36	+	t.Helper()
	37	+	if c.Dir == "" {
	38	+	c.Dir = t.TempDir()
	39	+	}
	40	+	if c.Max == 0 {
	41	+	c.Max = 8 << 20
	42	+	}
	43	+	c.Upstream, c.AuthURL = srv.URL, srv.URL+"/token"
	44	+	p, err := New(c)
	45	+	if err != nil {
	46	+	t.Fatal(err)
	47	+	}
	48	+	return p
	49	+	}
	50	+
	51	+	// accounted checks that the store's total is exactly the sum of its entries and
	52	+	// within Max, and returns the number of tag files on disk.
	53	+	func accounted(t testing.T, s store) int {
	54	+	t.Helper()
	55	+	s.mu.Lock()
	56	+	defer s.mu.Unlock()
	57	+	var sum int64
	58	+	tags := 0
	59	+	for k, e := range s.entries {
	60	+	sum += e.size
	61	+	if strings.HasPrefix(k, "tags/") {
	62	+	tags++
	63	+	if e.size != tagCharge {
	64	+	t.Errorf("tag %s charged %d", k, e.size)
	65	+	}
	66	+	}
	67	+	}
	68	+	if sum != s.total \|\| s.total+s.pending > s.max {
	69	+	t.Fatalf("accounting: sum=%d total=%d pending=%d max=%d", sum, s.total, s.pending, s.max)
	70	+	}
	71	+	files, err := os.ReadDir(filepath.Join(s.dir, "tags"))
	72	+	if err != nil {
	73	+	t.Fatal(err)
	74	+	}
	75	+	if len(files) != tags {
	76	+	t.Fatalf("%d tag files, %d tag entries", len(files), tags)
	77	+	}
	78	+	return tags
	79	+	}
	80	+
	81	+	// Accept is reduced to the known manifest types, so its variants map to
	82	+	// a bounded set of keys.
	83	+	func TestNormalizeAccept(t *testing.T) {
	84	+	const index, v2 = "application/vnd.oci.image.index.v1+json", "application/vnd.docker.distribution.manifest.v2+json"
	85	+	all := normalizeAccept("")
	86	+	for _, tc := range []struct{ in, want string }{
	87	+	{index + ";q=0.9, " + strings.ToUpper(v2) + " , " + index, v2 + "," + index},
	88	+	{" " + index + " ;q=0.1", index},
	89	+	{"text/html, /", all},
	90	+	{"", all},
	91	+	} {
	92	+	if got := normalizeAccept(tc.in); got != tc.want {
	93	+	t.Errorf("normalizeAccept(%q) = %q, want %q", tc.in, got, tc.want)
	94	+	}
	95	+	}
	96	+	if strings.Count(all, ",") != len(manifestTypes)-1 {
	97	+	t.Fatalf("default key %q", all)
	98	+	}
	99	+	}
	100	+
	101	+	// A job varying Accept on one tag creates at most one tag file per
	102	+	// subset of the known types, and every tag file is charged.
	103	+	func TestTagFilesBoundedByAccept(t *testing.T) {
	104	+	f := fake(t)
	105	+	p := proxyFor(t, f)
	106	+	for i := range 300 {
	107	+	accept := fmt.Sprintf("application/vnd.oci.image.index.v1+json;q=0.%d, x/junk-%d", i%10, i)
	108	+	switch i % 3 {
	109	+	case 1:
	110	+	accept = fmt.Sprintf("x/only-junk-%d", i)
	111	+	case 2:
	112	+	accept += ", application/vnd.docker.distribution.manifest.list.v2+json"
	113	+	}
	114	+	if w := request(t, p, http.MethodGet, "/v2/library/alpine/manifests/3", "Accept", accept); w.Code != http.StatusOK {
	115	+	t.Fatalf("manifest: %d", w.Code)
	116	+	}
	117	+	}
	118	+	if n := accounted(t, p.s); n == 0 \|\| n > 3 {
	119	+	t.Fatalf("%d tag files for 300 Accept variants", n)
	120	+	}
	121	+	}
	122	+
	123	+	// Tag files share the LRU of the store, so they cannot grow the store
	124	+	// past Max.
	125	+	func TestTagsChargedAndEvicted(t *testing.T) {
	126	+	now := time.Unix(1_700_000_000, 0)
	127	+	const media = "application/vnd.oci.image.index.v1+json"
	128	+	dir := t.TempDir()
	129	+	max := int64(len(manifestBody)+len(media)) + 3*tagCharge
	130	+	s, err := openStore(dir, max, func() time.Time { return now })
	131	+	if err != nil {
	132	+	t.Fatal(err)
	133	+	}
	134	+	m, _, _, err := s.ensure(context.Background(), "manifests", sum(manifestBody), func(_ context.Context, w io.Writer) (string, error) {
	135	+	_, err := io.WriteString(w, manifestBody)
	136	+	return media, err
	137	+	}, keepalive{})
	138	+	if err != nil {
	139	+	t.Fatal(err)
	140	+	}
	141	+	_ = m.Close()
	142	+	for i := range 20 {
	143	+	now = now.Add(time.Second)
	144	+	if err := s.setTag("library/alpine", fmt.Sprint("t", i), "", sum(manifestBody)); err != nil {
	145	+	t.Fatalf("tag %d: %v", i, err)
	146	+	}
	147	+	if n := accounted(t, s); n > 3 {
	148	+	t.Fatalf("%d tags over a cap of 3", n)
	149	+	}
	150	+	}
	151	+	if s.tag("library/alpine", "t19", "") != sum(manifestBody) \|\| s.tag("library/alpine", "t0", "") != "" {
	152	+	t.Fatal("tags not evicted least recently used first")
	153	+	}
	154	+	// Reopening charges the tag files again.
	155	+	s, err = openStore(dir, max, time.Now)
	156	+	if err != nil {
	157	+	t.Fatal(err)
	158	+	}
	159	+	if n := accounted(t, s); n != 3 {
	160	+	t.Fatalf("reopened with %d tags", n)
	161	+	}
	162	+	}
	163	+
	164	+	// An entry that is open for serving is not evicted (its bytes would
	165	+	// stay allocated while uncharged); it can be evicted once closed.
	166	+	func TestPinnedBlobSurvivesEviction(t *testing.T) {
	167	+	dir := t.TempDir()
	168	+	s, err := openStore(dir, 8, time.Now)
	169	+	if err != nil {
	170	+	t.Fatal(err)
	171	+	}
	172	+	put(t, s, "aaaa")
	173	+	f, _, err := s.open("blobs", sum("aaaa"))
	174	+	if err != nil {
	175	+	t.Fatal(err)
	176	+	}
	177	+	fill := func() error {
	178	+	g, _, _, err := s.ensure(context.Background(), "blobs", sum("bbbbbbbb"), func(_ context.Context, w io.Writer) (string, error) {
	179	+	_, err := io.WriteString(w, "bbbbbbbb")
	180	+	return "", err
	181	+	}, keepalive{})
	182	+	if err == nil {
	183	+	_ = g.Close()
	184	+	}
	185	+	return err
	186	+	}
	187	+	if err := fill(); !errors.Is(err, errTooLarge) {
	188	+	t.Fatalf("fill over a pinned entry: %v", err)
	189	+	}
	190	+	if _, err := os.Stat(filepath.Join(dir, "blobs", sum("aaaa")[7:])); err != nil {
	191	+	t.Fatalf("pinned blob removed: %v", err)
	192	+	}
	193	+	accounted(t, s)
	194	+	if err := f.Close(); err != nil {
	195	+	t.Fatal(err)
	196	+	}
	197	+	if err := fill(); err != nil {
	198	+	t.Fatal(err)
	199	+	}
	200	+	if _, err := os.Stat(filepath.Join(dir, "blobs", sum("aaaa")[7:])); !os.IsNotExist(err) {
	201	+	t.Fatal("unpinned blob not evicted")
	202	+	}
	203	+	accounted(t, s)
	204	+	}
	205	+
	206	+	// blockingWriter holds a response in its first Write until release is closed.
	207	+	type blockingWriter struct {
	208	+	*httptest.ResponseRecorder
	209	+	started, release chan struct{}
	210	+	once sync.Once
	211	+	}
	212	+
	213	+	func (b *blockingWriter) Write(p []byte) (int, error) {
	214	+	b.once.Do(func() { close(b.started) })
	215	+	<-b.release
	216	+	return b.ResponseRecorder.Write(p)
	217	+	}
	218	+
	219	+	// A blob that a response is still sending is pinned: a fill that needs
	220	+	// its room fails, and succeeds once the response has closed the file.
	221	+	func TestEvictionWhileServing(t *testing.T) {
	222	+	a, b := strings.Repeat("a", 1024), strings.Repeat("b", 1024)
	223	+	srv := upstreamWith(t, func(w http.ResponseWriter, r *http.Request) {
	224	+	switch {
	225	+	case strings.HasSuffix(r.URL.Path, sum(a)):
	226	+	_, _ = io.WriteString(w, a)
	227	+	case strings.HasSuffix(r.URL.Path, sum(b)):
	228	+	_, _ = io.WriteString(w, b)
	229	+	default:
	230	+	http.NotFound(w, r)
	231	+	}
	232	+	})
	233	+	p := proxyWith(t, srv, Config{Max: 1500})
	234	+	if w := request(t, p, http.MethodGet, "/v2/library/alpine/blobs/"+sum(a)); w.Code != http.StatusOK {
	235	+	t.Fatal(w.Code)
	236	+	}
	237	+	bw := &blockingWriter{ResponseRecorder: httptest.NewRecorder(), started: make(chan struct{}), release: make(chan struct{})}
	238	+	done := make(chan struct{})
	239	+	go func() {
	240	+	defer close(done)
	241	+	p.ServeHTTP(bw, httptest.NewRequest(http.MethodGet, "/v2/library/alpine/blobs/"+sum(a), nil))
	242	+	}()
	243	+	<-bw.started
	244	+	if w := request(t, p, http.MethodGet, "/v2/library/alpine/blobs/"+sum(b)); w.Code != http.StatusBadGateway {
	245	+	t.Fatalf("fill evicted a blob being served: %d", w.Code)
	246	+	}
	247	+	if _, err := os.Stat(filepath.Join(p.s.dir, "blobs", sum(a)[7:])); err != nil {
	248	+	t.Fatalf("served blob removed: %v", err)
	249	+	}
	250	+	accounted(t, p.s)
	251	+	close(bw.release)
	252	+	<-done
	253	+	if bw.Code != http.StatusOK \|\| bw.Body.String() != a {
	254	+	t.Fatalf("served: %d", bw.Code)
	255	+	}
	256	+	if w := request(t, p, http.MethodGet, "/v2/library/alpine/blobs/"+sum(b)); w.Code != http.StatusOK \|\| w.Body.String() != b {
	257	+	t.Fatalf("after release: %d", w.Code)
	258	+	}
	259	+	if _, err := os.Stat(filepath.Join(p.s.dir, "blobs", sum(a)[7:])); !os.IsNotExist(err) {
	260	+	t.Fatal("released blob not evicted")
	261	+	}
	262	+	accounted(t, p.s)
	263	+	}
	264	+
	265	+	// slowBlob streams body in chunks, one every gap.
	266	+	func slowBlob(body string, chunks int, gap time.Duration) http.HandlerFunc {
	267	+	return func(w http.ResponseWriter, r *http.Request) {
	268	+	w.Header().Set("Content-Length", fmt.Sprint(len(body)))
	269	+	size := len(body) / chunks
	270	+	for i := 0; i < len(body); i += size {
	271	+	_, _ = io.WriteString(w, body[i:min(i+size, len(body))])
	272	+	w.(http.Flusher).Flush()
	273	+	select {
	274	+	case <-time.After(gap):
	275	+	case <-r.Context().Done():
	276	+	return
	277	+	}
	278	+	}
	279	+	}
	280	+	}
	281	+
	282	+	// A blob body has no total deadline: a transfer that keeps moving may
	283	+	// last far longer than RequestTimeout and BlobIdleTimeout.
	284	+	func TestBlobTransferOutlivesRequestTimeout(t *testing.T) {
	285	+	body := strings.Repeat("layer", 4000)
	286	+	srv := upstreamWith(t, slowBlob(body, 20, 25*time.Millisecond))
	287	+	p := proxyWith(t, srv, Config{RequestTimeout: 100 * time.Millisecond, BlobIdleTimeout: 150 * time.Millisecond})
	288	+	start := time.Now()
	289	+	w := request(t, p, http.MethodGet, "/v2/library/alpine/blobs/"+sum(body))
	290	+	if w.Code != http.StatusOK \|\| w.Body.String() != body {
	291	+	t.Fatalf("blob: %d", w.Code)
	292	+	}
	293	+	if d := time.Since(start); d < 300*time.Millisecond {
	294	+	t.Fatalf("transfer took %v, not longer than the timeouts", d)
	295	+	}
	296	+	}
	297	+
	298	+	// A blob body that stops moving is cut after BlobIdleTimeout, and auth,
	299	+	// manifests and HEAD keep their total RequestTimeout.
	300	+	func TestStalledUpstreamTimesOut(t *testing.T) {
	301	+	body := strings.Repeat("layer", 4000)
	302	+	srv := upstreamWith(t, func(w http.ResponseWriter, r *http.Request) {
	303	+	if strings.Contains(r.URL.Path, "/blobs/") && r.Method == http.MethodGet {
	304	+	w.Header().Set("Content-Length", fmt.Sprint(len(body)))
	305	+	_, _ = io.WriteString(w, body[:100])
	306	+	w.(http.Flusher).Flush()
	307	+	}
	308	+	select { // a stalled body, or headers that never come
	309	+	case <-r.Context().Done():
	310	+	case <-time.After(5 * time.Second):
	311	+	}
	312	+	})
	313	+	p := proxyWith(t, srv, Config{RequestTimeout: 100 * time.Millisecond, BlobIdleTimeout: 150 * time.Millisecond})
	314	+	for _, tc := range []struct{ method, path string }{
	315	+	{http.MethodGet, "/v2/library/alpine/blobs/" + sum(body)},
	316	+	{http.MethodHead, "/v2/library/alpine/blobs/" + sum(body)},
	317	+	{http.MethodGet, "/v2/library/alpine/manifests/3"},
	318	+	} {
	319	+	start := time.Now()
	320	+	if w := request(t, p, tc.method, tc.path); w.Code != http.StatusBadGateway {
	321	+	t.Errorf("%s %s: %d", tc.method, tc.path, w.Code)
	322	+	}
	323	+	if d := time.Since(start); d > 2*time.Second {
	324	+	t.Errorf("%s %s: cut after %v", tc.method, tc.path, d)
	325	+	}
	326	+	}
	327	+	if _, err := os.Stat(filepath.Join(p.s.dir, "blobs", sum(body)[7:])); !os.IsNotExist(err) {
	328	+	t.Fatal("partial blob stored")
	329	+	}
	330	+	}
	331	+
	332	+	// waitFor polls cond (under the store lock) until it holds.
	333	+	func waitFor(t testing.T, s store, what string, cond func() bool) {
	334	+	t.Helper()
	335	+	for end := time.Now().Add(5 * time.Second); ; {
	336	+	s.mu.Lock()
	337	+	ok := cond()
	338	+	s.mu.Unlock()
	339	+	if ok {
	340	+	return
	341	+	}
	342	+	if time.Now().After(end) {
	343	+	t.Fatalf("timed out waiting for %s", what)
	344	+	}
	345	+	time.Sleep(time.Millisecond)
	346	+	}
	347	+	}
	348	+
	349	+	// waiters is the number of requests waiting for the flight of key (-1: none).
	350	+	func waiters(s *store, key string) int {
	351	+	if fl, ok := s.flights[key]; ok {
	352	+	return fl.waiters
	353	+	}
	354	+	return -1
	355	+	}
	356	+
	357	+	// The shared download does not belong to the client that started it.
	358	+	// That client disconnects mid-download; a second client, joining within the
	359	+	// grace period, still gets the blob and the blob is committed, from one
	360	+	// upstream fetch.
	361	+	func TestBlobFlightSurvivesInitiatorDisconnect(t *testing.T) {
	362	+	body := strings.Repeat("layer", 4000)
	363	+	half, release := make(chan struct{}), make(chan struct{})
	364	+	var releaseOnce sync.Once
	365	+	unblock := func() { releaseOnce.Do(func() { close(release) }) }
	366	+	var gets atomic.Int32
	367	+	srv := upstreamWith(t, func(w http.ResponseWriter, r *http.Request) {
	368	+	gets.Add(1)
	369	+	w.Header().Set("Content-Length", fmt.Sprint(len(body)))
	370	+	_, _ = io.WriteString(w, body[:len(body)/2])
	371	+	w.(http.Flusher).Flush()
	372	+	close(half)
	373	+	<-release
	374	+	_, _ = io.WriteString(w, body[len(body)/2:])
	375	+	})
	376	+	t.Cleanup(unblock) // before srv.Close, which waits for the handler
	377	+	p := proxyWith(t, srv, Config{})
	378	+	path := "/v2/library/alpine/blobs/" + sum(body)
	379	+
	380	+	ctx, cancel := context.WithCancel(context.Background())
	381	+	first := httptest.NewRecorder()
	382	+	done := make(chan struct{})
	383	+	go func() {
	384	+	defer close(done)
	385	+	p.ServeHTTP(first, httptest.NewRequest(http.MethodGet, path, nil).WithContext(ctx))
	386	+	}()
	387	+	<-half
	388	+	cancel()
	389	+	<-done
	390	+	if strings.Contains(first.Body.String(), "layer") {
	391	+	t.Fatal("unverified bytes sent to the initiator")
	392	+	}
	393	+	key := "blobs/" + sum(body)[7:]
	394	+	p.s.mu.Lock()
	395	+	left := waiters(p.s, key)
	396	+	p.s.mu.Unlock()
	397	+	if left != 0 {
	398	+	t.Fatalf("waiters after the disconnect: %d", left)
	399	+	}
	400	+
	401	+	second := httptest.NewRecorder()
	402	+	done = make(chan struct{})
	403	+	go func() {
	404	+	defer close(done)
	405	+	p.ServeHTTP(second, httptest.NewRequest(http.MethodGet, path, nil))
	406	+	}()
	407	+	waitFor(t, p.s, "the second client to join", func() bool { return waiters(p.s, key) == 1 })
	408	+	unblock()
	409	+	<-done
	410	+	if second.Code != http.StatusOK \|\| second.Body.String() != body {
	411	+	t.Fatalf("second client: %d", second.Code)
	412	+	}
	413	+	if _, err := os.Stat(filepath.Join(p.s.dir, "blobs", sum(body)[7:])); err != nil {
	414	+	t.Fatalf("blob not committed: %v", err)
	415	+	}
	416	+	if gets.Load() != 1 {
	417	+	t.Fatalf("%d upstream fetches", gets.Load())
	418	+	}
	419	+	}
	420	+
	421	+	type keepAliveRecorder struct {
	422	+	*httptest.ResponseRecorder
	423	+	calls atomic.Int32
	424	+	}
	425	+
	426	+	func (k *keepAliveRecorder) KeepAlive() { k.calls.Add(1) }
	427	+
	428	+	// A client waiting for a progressing download is kept alive, so
	429	+	// the cache channel's progress deadline follows the download.
	430	+	func TestWaiterKeepAlive(t *testing.T) {
	431	+	body := strings.Repeat("layer", 4000)
	432	+	srv := upstreamWith(t, slowBlob(body, 10, 30*time.Millisecond))
	433	+	p := proxyWith(t, srv, Config{KeepAliveInterval: 10 * time.Millisecond})
	434	+	w := &keepAliveRecorder{ResponseRecorder: httptest.NewRecorder()}
	435	+	p.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/v2/library/alpine/blobs/"+sum(body), nil))
	436	+	if w.Code != http.StatusOK \|\| w.calls.Load() == 0 {
	437	+	t.Fatalf("code %d, %d keepalives", w.Code, w.calls.Load())
	438	+	}
	439	+	}
	440	+
	441	+	// Auth and manifest requests never follow a redirect.
	442	+	func TestMetadataRedirectsNotFollowed(t *testing.T) {
	443	+	var elsewhere atomic.Int32
	444	+	other := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	445	+	elsewhere.Add(1)
	446	+	_, _ = io.WriteString(w, `{"token":"public"}`)
	447	+	}))
	448	+	t.Cleanup(other.Close)
	449	+	for _, redirectAuth := range []bool{true, false} {
	450	+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	451	+	if r.URL.Path == "/token" && !redirectAuth {
	452	+	_, _ = io.WriteString(w, `{"token":"public"}`)
	453	+	return
	454	+	}
	455	+	http.Redirect(w, r, other.URL+r.URL.Path, http.StatusTemporaryRedirect)
	456	+	}))
	457	+	p := proxyWith(t, srv, Config{})
	458	+	for _, ref := range []string{"3", sum(manifestBody)} {
	459	+	if w := request(t, p, http.MethodGet, "/v2/library/alpine/manifests/"+ref); w.Code != http.StatusBadGateway {
	460	+	t.Errorf("auth redirect %v, ref %s: %d", redirectAuth, ref, w.Code)
	461	+	}
	462	+	}
	463	+	srv.Close()
	464	+	}
	465	+	if elsewhere.Load() != 0 {
	466	+	t.Fatalf("%d redirects followed", elsewhere.Load())
	467	+	}
	468	+	}
	469	+
	470	+	// Blob requests follow the CDN redirect, without the bearer token.
	471	+	func TestBlobRedirectStripsBearer(t *testing.T) {
	472	+	var cdnHits atomic.Int32
	473	+	cdn := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	474	+	cdnHits.Add(1)
	475	+	if r.Header.Get("Authorization") != "" {
	476	+	t.Error("bearer sent to the CDN")
	477	+	}
	478	+	_, _ = io.WriteString(w, blobBody)
	479	+	}))
	480	+	t.Cleanup(cdn.Close)
	481	+	srv := upstreamWith(t, func(w http.ResponseWriter, r *http.Request) {
	482	+	if r.Header.Get("Authorization") != "Bearer public" {
	483	+	t.Error("no bearer on the registry")
	484	+	}
	485	+	http.Redirect(w, r, cdn.URL+"/layer", http.StatusTemporaryRedirect)
	486	+	})
	487	+	p := proxyWith(t, srv, Config{})
	488	+	if w := request(t, p, http.MethodGet, "/v2/library/alpine/blobs/"+sum(blobBody)); w.Code != http.StatusOK \|\| w.Body.String() != blobBody {
	489	+	t.Fatalf("blob: %d", w.Code)
	490	+	}
	491	+	if cdnHits.Load() != 1 {
	492	+	t.Fatalf("CDN hits %d", cdnHits.Load())
	493	+	}
	494	+	}

@@ -0,0 +1,494 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+	package registryproxy
	3	+
	4	+	import (
	5	+	"context"
	6	+	"errors"
	7	+	"fmt"
	8	+	"io"
	9	+	"net/http"
	10	+	"net/http/httptest"
	11	+	"os"
	12	+	"path/filepath"
	13	+	"strings"
	14	+	"sync"
	15	+	"sync/atomic"
	16	+	"testing"
	17	+	"time"
	18	+	)
	19	+
	20	+	// upstreamWith fakes Docker Hub: /token answers an anonymous token and h serves
	21	+	// everything else.
	22	+	func upstreamWith(t testing.T, h http.HandlerFunc) httptest.Server {
	23	+	t.Helper()
	24	+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	25	+	if r.URL.Path == "/token" {
	26	+	_, _ = io.WriteString(w, `{"token":"public","expires_in":300}`)
	27	+	return
	28	+	}
	29	+	h(w, r)
	30	+	}))
	31	+	t.Cleanup(srv.Close)
	32	+	return srv
	33	+	}
	34	+
	35	+	func proxyWith(t testing.T, srv httptest.Server, c Config) *Proxy {
	36	+	t.Helper()
	37	+	if c.Dir == "" {
	38	+	c.Dir = t.TempDir()
	39	+	}
	40	+	if c.Max == 0 {
	41	+	c.Max = 8 << 20
	42	+	}
	43	+	c.Upstream, c.AuthURL = srv.URL, srv.URL+"/token"
	44	+	p, err := New(c)
	45	+	if err != nil {
	46	+	t.Fatal(err)
	47	+	}
	48	+	return p
	49	+	}
	50	+
	51	+	// accounted checks that the store's total is exactly the sum of its entries and
	52	+	// within Max, and returns the number of tag files on disk.
	53	+	func accounted(t testing.T, s store) int {
	54	+	t.Helper()
	55	+	s.mu.Lock()
	56	+	defer s.mu.Unlock()
	57	+	var sum int64
	58	+	tags := 0
	59	+	for k, e := range s.entries {
	60	+	sum += e.size
	61	+	if strings.HasPrefix(k, "tags/") {
	62	+	tags++
	63	+	if e.size != tagCharge {
	64	+	t.Errorf("tag %s charged %d", k, e.size)
	65	+	}
	66	+	}
	67	+	}
	68	+	if sum != s.total \|\| s.total+s.pending > s.max {
	69	+	t.Fatalf("accounting: sum=%d total=%d pending=%d max=%d", sum, s.total, s.pending, s.max)
	70	+	}
	71	+	files, err := os.ReadDir(filepath.Join(s.dir, "tags"))
	72	+	if err != nil {
	73	+	t.Fatal(err)
	74	+	}
	75	+	if len(files) != tags {
	76	+	t.Fatalf("%d tag files, %d tag entries", len(files), tags)
	77	+	}
	78	+	return tags
	79	+	}
	80	+
	81	+	// Accept is reduced to the known manifest types, so its variants map to
	82	+	// a bounded set of keys.
	83	+	func TestNormalizeAccept(t *testing.T) {
	84	+	const index, v2 = "application/vnd.oci.image.index.v1+json", "application/vnd.docker.distribution.manifest.v2+json"
	85	+	all := normalizeAccept("")
	86	+	for _, tc := range []struct{ in, want string }{
	87	+	{index + ";q=0.9, " + strings.ToUpper(v2) + " , " + index, v2 + "," + index},
	88	+	{" " + index + " ;q=0.1", index},
	89	+	{"text/html, /", all},
	90	+	{"", all},
	91	+	} {
	92	+	if got := normalizeAccept(tc.in); got != tc.want {
	93	+	t.Errorf("normalizeAccept(%q) = %q, want %q", tc.in, got, tc.want)
	94	+	}
	95	+	}
	96	+	if strings.Count(all, ",") != len(manifestTypes)-1 {
	97	+	t.Fatalf("default key %q", all)
	98	+	}
	99	+	}
	100	+
	101	+	// A job varying Accept on one tag creates at most one tag file per
	102	+	// subset of the known types, and every tag file is charged.
	103	+	func TestTagFilesBoundedByAccept(t *testing.T) {
	104	+	f := fake(t)
	105	+	p := proxyFor(t, f)
	106	+	for i := range 300 {
	107	+	accept := fmt.Sprintf("application/vnd.oci.image.index.v1+json;q=0.%d, x/junk-%d", i%10, i)
	108	+	switch i % 3 {
	109	+	case 1:
	110	+	accept = fmt.Sprintf("x/only-junk-%d", i)
	111	+	case 2:
	112	+	accept += ", application/vnd.docker.distribution.manifest.list.v2+json"
	113	+	}
	114	+	if w := request(t, p, http.MethodGet, "/v2/library/alpine/manifests/3", "Accept", accept); w.Code != http.StatusOK {
	115	+	t.Fatalf("manifest: %d", w.Code)
	116	+	}
	117	+	}
	118	+	if n := accounted(t, p.s); n == 0 \|\| n > 3 {
	119	+	t.Fatalf("%d tag files for 300 Accept variants", n)
	120	+	}
	121	+	}
	122	+
	123	+	// Tag files share the LRU of the store, so they cannot grow the store
	124	+	// past Max.
	125	+	func TestTagsChargedAndEvicted(t *testing.T) {
	126	+	now := time.Unix(1_700_000_000, 0)
	127	+	const media = "application/vnd.oci.image.index.v1+json"
	128	+	dir := t.TempDir()
	129	+	max := int64(len(manifestBody)+len(media)) + 3*tagCharge
	130	+	s, err := openStore(dir, max, func() time.Time { return now })
	131	+	if err != nil {
	132	+	t.Fatal(err)
	133	+	}
	134	+	m, _, _, err := s.ensure(context.Background(), "manifests", sum(manifestBody), func(_ context.Context, w io.Writer) (string, error) {
	135	+	_, err := io.WriteString(w, manifestBody)
	136	+	return media, err
	137	+	}, keepalive{})
	138	+	if err != nil {
	139	+	t.Fatal(err)
	140	+	}
	141	+	_ = m.Close()
	142	+	for i := range 20 {
	143	+	now = now.Add(time.Second)
	144	+	if err := s.setTag("library/alpine", fmt.Sprint("t", i), "", sum(manifestBody)); err != nil {
	145	+	t.Fatalf("tag %d: %v", i, err)
	146	+	}
	147	+	if n := accounted(t, s); n > 3 {
	148	+	t.Fatalf("%d tags over a cap of 3", n)
	149	+	}
	150	+	}
	151	+	if s.tag("library/alpine", "t19", "") != sum(manifestBody) \|\| s.tag("library/alpine", "t0", "") != "" {
	152	+	t.Fatal("tags not evicted least recently used first")
	153	+	}
	154	+	// Reopening charges the tag files again.
	155	+	s, err = openStore(dir, max, time.Now)
	156	+	if err != nil {
	157	+	t.Fatal(err)
	158	+	}
	159	+	if n := accounted(t, s); n != 3 {
	160	+	t.Fatalf("reopened with %d tags", n)
	161	+	}
	162	+	}
	163	+
	164	+	// An entry that is open for serving is not evicted (its bytes would
	165	+	// stay allocated while uncharged); it can be evicted once closed.
	166	+	func TestPinnedBlobSurvivesEviction(t *testing.T) {
	167	+	dir := t.TempDir()
	168	+	s, err := openStore(dir, 8, time.Now)
	169	+	if err != nil {
	170	+	t.Fatal(err)
	171	+	}
	172	+	put(t, s, "aaaa")
	173	+	f, _, err := s.open("blobs", sum("aaaa"))
	174	+	if err != nil {
	175	+	t.Fatal(err)
	176	+	}
	177	+	fill := func() error {
	178	+	g, _, _, err := s.ensure(context.Background(), "blobs", sum("bbbbbbbb"), func(_ context.Context, w io.Writer) (string, error) {
	179	+	_, err := io.WriteString(w, "bbbbbbbb")
	180	+	return "", err
	181	+	}, keepalive{})
	182	+	if err == nil {
	183	+	_ = g.Close()
	184	+	}
	185	+	return err
	186	+	}
	187	+	if err := fill(); !errors.Is(err, errTooLarge) {
	188	+	t.Fatalf("fill over a pinned entry: %v", err)
	189	+	}
	190	+	if _, err := os.Stat(filepath.Join(dir, "blobs", sum("aaaa")[7:])); err != nil {
	191	+	t.Fatalf("pinned blob removed: %v", err)
	192	+	}
	193	+	accounted(t, s)
	194	+	if err := f.Close(); err != nil {
	195	+	t.Fatal(err)
	196	+	}
	197	+	if err := fill(); err != nil {
	198	+	t.Fatal(err)
	199	+	}
	200	+	if _, err := os.Stat(filepath.Join(dir, "blobs", sum("aaaa")[7:])); !os.IsNotExist(err) {
	201	+	t.Fatal("unpinned blob not evicted")
	202	+	}
	203	+	accounted(t, s)
	204	+	}
	205	+
	206	+	// blockingWriter holds a response in its first Write until release is closed.
	207	+	type blockingWriter struct {
	208	+	*httptest.ResponseRecorder
	209	+	started, release chan struct{}
	210	+	once sync.Once
	211	+	}
	212	+
	213	+	func (b *blockingWriter) Write(p []byte) (int, error) {
	214	+	b.once.Do(func() { close(b.started) })
	215	+	<-b.release
	216	+	return b.ResponseRecorder.Write(p)
	217	+	}
	218	+
	219	+	// A blob that a response is still sending is pinned: a fill that needs
	220	+	// its room fails, and succeeds once the response has closed the file.
	221	+	func TestEvictionWhileServing(t *testing.T) {
	222	+	a, b := strings.Repeat("a", 1024), strings.Repeat("b", 1024)
	223	+	srv := upstreamWith(t, func(w http.ResponseWriter, r *http.Request) {
	224	+	switch {
	225	+	case strings.HasSuffix(r.URL.Path, sum(a)):
	226	+	_, _ = io.WriteString(w, a)
	227	+	case strings.HasSuffix(r.URL.Path, sum(b)):
	228	+	_, _ = io.WriteString(w, b)
	229	+	default:
	230	+	http.NotFound(w, r)
	231	+	}
	232	+	})
	233	+	p := proxyWith(t, srv, Config{Max: 1500})
	234	+	if w := request(t, p, http.MethodGet, "/v2/library/alpine/blobs/"+sum(a)); w.Code != http.StatusOK {
	235	+	t.Fatal(w.Code)
	236	+	}
	237	+	bw := &blockingWriter{ResponseRecorder: httptest.NewRecorder(), started: make(chan struct{}), release: make(chan struct{})}
	238	+	done := make(chan struct{})
	239	+	go func() {
	240	+	defer close(done)
	241	+	p.ServeHTTP(bw, httptest.NewRequest(http.MethodGet, "/v2/library/alpine/blobs/"+sum(a), nil))
	242	+	}()
	243	+	<-bw.started
	244	+	if w := request(t, p, http.MethodGet, "/v2/library/alpine/blobs/"+sum(b)); w.Code != http.StatusBadGateway {
	245	+	t.Fatalf("fill evicted a blob being served: %d", w.Code)
	246	+	}
	247	+	if _, err := os.Stat(filepath.Join(p.s.dir, "blobs", sum(a)[7:])); err != nil {
	248	+	t.Fatalf("served blob removed: %v", err)
	249	+	}
	250	+	accounted(t, p.s)
	251	+	close(bw.release)
	252	+	<-done
	253	+	if bw.Code != http.StatusOK \|\| bw.Body.String() != a {
	254	+	t.Fatalf("served: %d", bw.Code)
	255	+	}
	256	+	if w := request(t, p, http.MethodGet, "/v2/library/alpine/blobs/"+sum(b)); w.Code != http.StatusOK \|\| w.Body.String() != b {
	257	+	t.Fatalf("after release: %d", w.Code)
	258	+	}
	259	+	if _, err := os.Stat(filepath.Join(p.s.dir, "blobs", sum(a)[7:])); !os.IsNotExist(err) {
	260	+	t.Fatal("released blob not evicted")
	261	+	}
	262	+	accounted(t, p.s)
	263	+	}
	264	+
	265	+	// slowBlob streams body in chunks, one every gap.
	266	+	func slowBlob(body string, chunks int, gap time.Duration) http.HandlerFunc {
	267	+	return func(w http.ResponseWriter, r *http.Request) {
	268	+	w.Header().Set("Content-Length", fmt.Sprint(len(body)))
	269	+	size := len(body) / chunks
	270	+	for i := 0; i < len(body); i += size {
	271	+	_, _ = io.WriteString(w, body[i:min(i+size, len(body))])
	272	+	w.(http.Flusher).Flush()
	273	+	select {
	274	+	case <-time.After(gap):
	275	+	case <-r.Context().Done():
	276	+	return
	277	+	}
	278	+	}
	279	+	}
	280	+	}
	281	+
	282	+	// A blob body has no total deadline: a transfer that keeps moving may
	283	+	// last far longer than RequestTimeout and BlobIdleTimeout.
	284	+	func TestBlobTransferOutlivesRequestTimeout(t *testing.T) {
	285	+	body := strings.Repeat("layer", 4000)
	286	+	srv := upstreamWith(t, slowBlob(body, 20, 25*time.Millisecond))
	287	+	p := proxyWith(t, srv, Config{RequestTimeout: 100 * time.Millisecond, BlobIdleTimeout: 150 * time.Millisecond})
	288	+	start := time.Now()
	289	+	w := request(t, p, http.MethodGet, "/v2/library/alpine/blobs/"+sum(body))
	290	+	if w.Code != http.StatusOK \|\| w.Body.String() != body {
	291	+	t.Fatalf("blob: %d", w.Code)
	292	+	}
	293	+	if d := time.Since(start); d < 300*time.Millisecond {
	294	+	t.Fatalf("transfer took %v, not longer than the timeouts", d)
	295	+	}
	296	+	}
	297	+
	298	+	// A blob body that stops moving is cut after BlobIdleTimeout, and auth,
	299	+	// manifests and HEAD keep their total RequestTimeout.
	300	+	func TestStalledUpstreamTimesOut(t *testing.T) {
	301	+	body := strings.Repeat("layer", 4000)
	302	+	srv := upstreamWith(t, func(w http.ResponseWriter, r *http.Request) {
	303	+	if strings.Contains(r.URL.Path, "/blobs/") && r.Method == http.MethodGet {
	304	+	w.Header().Set("Content-Length", fmt.Sprint(len(body)))
	305	+	_, _ = io.WriteString(w, body[:100])
	306	+	w.(http.Flusher).Flush()
	307	+	}
	308	+	select { // a stalled body, or headers that never come
	309	+	case <-r.Context().Done():
	310	+	case <-time.After(5 * time.Second):
	311	+	}
	312	+	})
	313	+	p := proxyWith(t, srv, Config{RequestTimeout: 100 * time.Millisecond, BlobIdleTimeout: 150 * time.Millisecond})
	314	+	for _, tc := range []struct{ method, path string }{
	315	+	{http.MethodGet, "/v2/library/alpine/blobs/" + sum(body)},
	316	+	{http.MethodHead, "/v2/library/alpine/blobs/" + sum(body)},
	317	+	{http.MethodGet, "/v2/library/alpine/manifests/3"},
	318	+	} {
	319	+	start := time.Now()
	320	+	if w := request(t, p, tc.method, tc.path); w.Code != http.StatusBadGateway {
	321	+	t.Errorf("%s %s: %d", tc.method, tc.path, w.Code)
	322	+	}
	323	+	if d := time.Since(start); d > 2*time.Second {
	324	+	t.Errorf("%s %s: cut after %v", tc.method, tc.path, d)
	325	+	}
	326	+	}
	327	+	if _, err := os.Stat(filepath.Join(p.s.dir, "blobs", sum(body)[7:])); !os.IsNotExist(err) {
	328	+	t.Fatal("partial blob stored")
	329	+	}
	330	+	}
	331	+
	332	+	// waitFor polls cond (under the store lock) until it holds.
	333	+	func waitFor(t testing.T, s store, what string, cond func() bool) {
	334	+	t.Helper()
	335	+	for end := time.Now().Add(5 * time.Second); ; {
	336	+	s.mu.Lock()
	337	+	ok := cond()
	338	+	s.mu.Unlock()
	339	+	if ok {
	340	+	return
	341	+	}
	342	+	if time.Now().After(end) {
	343	+	t.Fatalf("timed out waiting for %s", what)
	344	+	}
	345	+	time.Sleep(time.Millisecond)
	346	+	}
	347	+	}
	348	+
	349	+	// waiters is the number of requests waiting for the flight of key (-1: none).
	350	+	func waiters(s *store, key string) int {
	351	+	if fl, ok := s.flights[key]; ok {
	352	+	return fl.waiters
	353	+	}
	354	+	return -1
	355	+	}
	356	+
	357	+	// The shared download does not belong to the client that started it.
	358	+	// That client disconnects mid-download; a second client, joining within the
	359	+	// grace period, still gets the blob and the blob is committed, from one
	360	+	// upstream fetch.
	361	+	func TestBlobFlightSurvivesInitiatorDisconnect(t *testing.T) {
	362	+	body := strings.Repeat("layer", 4000)
	363	+	half, release := make(chan struct{}), make(chan struct{})
	364	+	var releaseOnce sync.Once
	365	+	unblock := func() { releaseOnce.Do(func() { close(release) }) }
	366	+	var gets atomic.Int32
	367	+	srv := upstreamWith(t, func(w http.ResponseWriter, r *http.Request) {
	368	+	gets.Add(1)
	369	+	w.Header().Set("Content-Length", fmt.Sprint(len(body)))
	370	+	_, _ = io.WriteString(w, body[:len(body)/2])
	371	+	w.(http.Flusher).Flush()
	372	+	close(half)
	373	+	<-release
	374	+	_, _ = io.WriteString(w, body[len(body)/2:])
	375	+	})
	376	+	t.Cleanup(unblock) // before srv.Close, which waits for the handler
	377	+	p := proxyWith(t, srv, Config{})
	378	+	path := "/v2/library/alpine/blobs/" + sum(body)
	379	+
	380	+	ctx, cancel := context.WithCancel(context.Background())
	381	+	first := httptest.NewRecorder()
	382	+	done := make(chan struct{})
	383	+	go func() {
	384	+	defer close(done)
	385	+	p.ServeHTTP(first, httptest.NewRequest(http.MethodGet, path, nil).WithContext(ctx))
	386	+	}()
	387	+	<-half
	388	+	cancel()
	389	+	<-done
	390	+	if strings.Contains(first.Body.String(), "layer") {
	391	+	t.Fatal("unverified bytes sent to the initiator")
	392	+	}
	393	+	key := "blobs/" + sum(body)[7:]
	394	+	p.s.mu.Lock()
	395	+	left := waiters(p.s, key)
	396	+	p.s.mu.Unlock()
	397	+	if left != 0 {
	398	+	t.Fatalf("waiters after the disconnect: %d", left)
	399	+	}
	400	+
	401	+	second := httptest.NewRecorder()
	402	+	done = make(chan struct{})
	403	+	go func() {
	404	+	defer close(done)
	405	+	p.ServeHTTP(second, httptest.NewRequest(http.MethodGet, path, nil))
	406	+	}()
	407	+	waitFor(t, p.s, "the second client to join", func() bool { return waiters(p.s, key) == 1 })
	408	+	unblock()
	409	+	<-done
	410	+	if second.Code != http.StatusOK \|\| second.Body.String() != body {
	411	+	t.Fatalf("second client: %d", second.Code)
	412	+	}
	413	+	if _, err := os.Stat(filepath.Join(p.s.dir, "blobs", sum(body)[7:])); err != nil {
	414	+	t.Fatalf("blob not committed: %v", err)
	415	+	}
	416	+	if gets.Load() != 1 {
	417	+	t.Fatalf("%d upstream fetches", gets.Load())
	418	+	}
	419	+	}
	420	+
	421	+	type keepAliveRecorder struct {
	422	+	*httptest.ResponseRecorder
	423	+	calls atomic.Int32
	424	+	}
	425	+
	426	+	func (k *keepAliveRecorder) KeepAlive() { k.calls.Add(1) }
	427	+
	428	+	// A client waiting for a progressing download is kept alive, so
	429	+	// the cache channel's progress deadline follows the download.
	430	+	func TestWaiterKeepAlive(t *testing.T) {
	431	+	body := strings.Repeat("layer", 4000)
	432	+	srv := upstreamWith(t, slowBlob(body, 10, 30*time.Millisecond))
	433	+	p := proxyWith(t, srv, Config{KeepAliveInterval: 10 * time.Millisecond})
	434	+	w := &keepAliveRecorder{ResponseRecorder: httptest.NewRecorder()}
	435	+	p.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/v2/library/alpine/blobs/"+sum(body), nil))
	436	+	if w.Code != http.StatusOK \|\| w.calls.Load() == 0 {
	437	+	t.Fatalf("code %d, %d keepalives", w.Code, w.calls.Load())
	438	+	}
	439	+	}
	440	+
	441	+	// Auth and manifest requests never follow a redirect.
	442	+	func TestMetadataRedirectsNotFollowed(t *testing.T) {
	443	+	var elsewhere atomic.Int32
	444	+	other := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	445	+	elsewhere.Add(1)
	446	+	_, _ = io.WriteString(w, `{"token":"public"}`)
	447	+	}))
	448	+	t.Cleanup(other.Close)
	449	+	for _, redirectAuth := range []bool{true, false} {
	450	+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	451	+	if r.URL.Path == "/token" && !redirectAuth {
	452	+	_, _ = io.WriteString(w, `{"token":"public"}`)
	453	+	return
	454	+	}
	455	+	http.Redirect(w, r, other.URL+r.URL.Path, http.StatusTemporaryRedirect)
	456	+	}))
	457	+	p := proxyWith(t, srv, Config{})
	458	+	for _, ref := range []string{"3", sum(manifestBody)} {
	459	+	if w := request(t, p, http.MethodGet, "/v2/library/alpine/manifests/"+ref); w.Code != http.StatusBadGateway {
	460	+	t.Errorf("auth redirect %v, ref %s: %d", redirectAuth, ref, w.Code)
	461	+	}
	462	+	}
	463	+	srv.Close()
	464	+	}
	465	+	if elsewhere.Load() != 0 {
	466	+	t.Fatalf("%d redirects followed", elsewhere.Load())
	467	+	}
	468	+	}
	469	+
	470	+	// Blob requests follow the CDN redirect, without the bearer token.
	471	+	func TestBlobRedirectStripsBearer(t *testing.T) {
	472	+	var cdnHits atomic.Int32
	473	+	cdn := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	474	+	cdnHits.Add(1)
	475	+	if r.Header.Get("Authorization") != "" {
	476	+	t.Error("bearer sent to the CDN")
	477	+	}
	478	+	_, _ = io.WriteString(w, blobBody)
	479	+	}))
	480	+	t.Cleanup(cdn.Close)
	481	+	srv := upstreamWith(t, func(w http.ResponseWriter, r *http.Request) {
	482	+	if r.Header.Get("Authorization") != "Bearer public" {
	483	+	t.Error("no bearer on the registry")
	484	+	}
	485	+	http.Redirect(w, r, cdn.URL+"/layer", http.StatusTemporaryRedirect)
	486	+	})
	487	+	p := proxyWith(t, srv, Config{})
	488	+	if w := request(t, p, http.MethodGet, "/v2/library/alpine/blobs/"+sum(blobBody)); w.Code != http.StatusOK \|\| w.Body.String() != blobBody {
	489	+	t.Fatalf("blob: %d", w.Code)
	490	+	}
	491	+	if cdnHits.Load() != 1 {
	492	+	t.Fatalf("CDN hits %d", cdnHits.Load())
	493	+	}
	494	+	}

A internal/registryproxy/main_test.go +25 -0

@@ -0,0 +1,25 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+	package registryproxy
	3	+
	4	+	import (
	5	+	"fmt"
	6	+	"os"
	7	+	"sync/atomic"
	8	+	"testing"
	9	+	)
	10	+
	11	+	// TestMain fails the run if any store hit an accounting bug (unpinLocked of
	12	+	// an unpinned entry), whichever test caused it.
	13	+	func TestMain(m *testing.M) {
	14	+	var bugs atomic.Int32
	15	+	defaultBugHook = func(msg string) {
	16	+	bugs.Add(1)
	17	+	fmt.Fprintln(os.Stderr, "registry cache bug:", msg)
	18	+	}
	19	+	code := m.Run()
	20	+	if n := bugs.Load(); n > 0 && code == 0 {
	21	+	fmt.Fprintf(os.Stderr, "FAIL: %d registry cache accounting bugs\n", n)
	22	+	code = 1
	23	+	}
	24	+	os.Exit(code)
	25	+	}

@@ -0,0 +1,25 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+	package registryproxy
	3	+
	4	+	import (
	5	+	"fmt"
	6	+	"os"
	7	+	"sync/atomic"
	8	+	"testing"
	9	+	)
	10	+
	11	+	// TestMain fails the run if any store hit an accounting bug (unpinLocked of
	12	+	// an unpinned entry), whichever test caused it.
	13	+	func TestMain(m *testing.M) {
	14	+	var bugs atomic.Int32
	15	+	defaultBugHook = func(msg string) {
	16	+	bugs.Add(1)
	17	+	fmt.Fprintln(os.Stderr, "registry cache bug:", msg)
	18	+	}
	19	+	code := m.Run()
	20	+	if n := bugs.Load(); n > 0 && code == 0 {
	21	+	fmt.Fprintf(os.Stderr, "FAIL: %d registry cache accounting bugs\n", n)
	22	+	code = 1
	23	+	}
	24	+	os.Exit(code)
	25	+	}

A internal/registryproxy/proxy.go +319 -0

@@ -0,0 +1,319 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	// Package registryproxy provides an anonymous, pull-only Docker Hub mirror.
	4	+	// Its handler belongs on a sandbox's private runtime channel, not a public port.
	5	+	package registryproxy
	6	+
	7	+	import (
	8	+	"context"
	9	+	"encoding/hex"
	10	+	"errors"
	11	+	"io"
	12	+	"net/http"
	13	+	"net/url"
	14	+	"regexp"
	15	+	"strconv"
	16	+	"strings"
	17	+	"time"
	18	+
	19	+	"github.com/bnema/zerowrap"
	20	+	)
	21	+
	22	+	const manifestLimit = 4 << 20
	23	+
	24	+	// Config configures a Proxy. Max must be positive. The directory is exclusively
	25	+	// owned by one Proxy; opening it clears incomplete downloads from tmp/.
	26	+	type Config struct {
	27	+	Dir string
	28	+	Max int64
	29	+	Upstream string // default https://registry-1.docker.io
	30	+	AuthURL string // default https://auth.docker.io/token
	31	+	Client *http.Client
	32	+	Now func() time.Time
	33	+	Log zerowrap.Logger
	34	+	// RequestTimeout bounds auth, manifests and HEAD (default 10s).
	35	+	RequestTimeout time.Duration
	36	+	// BlobIdleTimeout bounds time without upstream body progress (default 30s).
	37	+	// A blob body has no total timeout.
	38	+	BlobIdleTimeout time.Duration
	39	+	// KeepAliveInterval is how often a client waiting for a cold blob is kept
	40	+	// alive while the download progresses (default 1s), when its
	41	+	// ResponseWriter has a KeepAlive method.
	42	+	KeepAliveInterval time.Duration
	43	+	// AbandonGrace is how long a download nobody waits for any more keeps
	44	+	// running in case another request joins it; then it is cancelled and its
	45	+	// temporary file removed (default 5s).
	46	+	AbandonGrace time.Duration
	47	+	// MaxDownloads caps the concurrent cold blob downloads across all jobs
	48	+	// (default 16). A request may also carry a per-client Quota (WithQuota).
	49	+	MaxDownloads int
	50	+	// SlotWait is how long a request that would start a download waits for a
	51	+	// free slot, global or of its quota, before it gets 503 (default 30s).
	52	+	SlotWait time.Duration
	53	+	}
	54	+
	55	+	// Proxy implements http.Handler for GET and HEAD under /v2/.
	56	+	type Proxy struct {
	57	+	s *store
	58	+	u *upstream
	59	+	log zerowrap.Logger
	60	+	keepEvery time.Duration
	61	+	}
	62	+
	63	+	// New opens the persistent cache. It starts no goroutines or listeners.
	64	+	func New(c Config) (*Proxy, error) {
	65	+	if c.Now == nil {
	66	+	c.Now = time.Now
	67	+	}
	68	+	if c.Upstream == "" {
	69	+	c.Upstream = "https://registry-1.docker.io"
	70	+	}
	71	+	if c.AuthURL == "" {
	72	+	c.AuthURL = "https://auth.docker.io/token"
	73	+	}
	74	+	for _, s := range []string{c.Upstream, c.AuthURL} {
	75	+	u, err := url.Parse(s)
	76	+	if err != nil \|\| u.Host == "" \|\| u.User != nil \|\| (u.Scheme != "http" && u.Scheme != "https") {
	77	+	return nil, errors.New("invalid registry upstream URL")
	78	+	}
	79	+	}
	80	+	if c.RequestTimeout <= 0 {
	81	+	c.RequestTimeout = 10 * time.Second
	82	+	}
	83	+	if c.BlobIdleTimeout <= 0 {
	84	+	c.BlobIdleTimeout = 30 * time.Second
	85	+	}
	86	+	if c.KeepAliveInterval <= 0 {
	87	+	c.KeepAliveInterval = time.Second
	88	+	}
	89	+	if c.AbandonGrace <= 0 {
	90	+	c.AbandonGrace = 5 * time.Second
	91	+	}
	92	+	if c.MaxDownloads <= 0 {
	93	+	c.MaxDownloads = 16
	94	+	}
	95	+	if c.SlotWait <= 0 {
	96	+	c.SlotWait = 30 * time.Second
	97	+	}
	98	+	client, blobs := upstreamClients(c)
	99	+
	100	+	st, err := openStore(c.Dir, c.Max, c.Now)
	101	+	if err != nil {
	102	+	return nil, err
	103	+	}
	104	+	st.grace, st.maxDownloads, st.slotWait, st.log = c.AbandonGrace, c.MaxDownloads, c.SlotWait, c.Log
	105	+	return &Proxy{s: st, u: &upstream{base: strings.TrimRight(c.Upstream, "/"), auth: c.AuthURL, client: client, blobClient: blobs, requestTimeout: c.RequestTimeout, blobIdle: c.BlobIdleTimeout, now: c.Now, tokens: map[string]bearer{}}, log: c.Log, keepEvery: c.KeepAliveInterval}, nil
	106	+	}
	107	+
	108	+	var namePattern = regexp.MustCompile(`^[a-z0-9]+(?:[._-][a-z0-9]+)(?:/[a-z0-9]+(?:[._-][a-z0-9]+))*$`)
	109	+	var tagPattern = regexp.MustCompile(`^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$`)
	110	+
	111	+	func validDigest(s string) bool {
	112	+	if len(s) != 71 \|\| !strings.HasPrefix(s, "sha256:") {
	113	+	return false
	114	+	}
	115	+	_, err := hex.DecodeString(s[7:])
	116	+	return err == nil && s == strings.ToLower(s)
	117	+	}
	118	+
	119	+	func (p Proxy) ServeHTTP(w http.ResponseWriter, r http.Request) {
	120	+	if r.Method != http.MethodGet && r.Method != http.MethodHead {
	121	+	w.WriteHeader(http.StatusMethodNotAllowed)
	122	+	return
	123	+	}
	124	+	w.Header().Set("Docker-Distribution-Api-Version", "registry/2.0")
	125	+	if r.URL.Path == "/v2/" {
	126	+	w.Header().Set("Content-Type", "application/json")
	127	+	w.Header().Set("Content-Length", "2")
	128	+	if r.Method == http.MethodGet {
	129	+	_, _ = io.WriteString(w, "{}")
	130	+	}
	131	+	return
	132	+	}
	133	+	if !strings.HasPrefix(r.URL.Path, "/v2/") {
	134	+	http.NotFound(w, r)
	135	+	return
	136	+	}
	137	+	path := strings.TrimPrefix(r.URL.Path, "/v2/")
	138	+	i := strings.LastIndex(path, "/")
	139	+	if i < 0 {
	140	+	http.NotFound(w, r)
	141	+	return
	142	+	}
	143	+	ref := path[i+1:]
	144	+	path = path[:i]
	145	+	i = strings.LastIndex(path, "/")
	146	+	if i < 0 {
	147	+	http.NotFound(w, r)
	148	+	return
	149	+	}
	150	+	name, kind := path[:i], path[i+1:]
	151	+	// dockerd's legacy client sends one Accept line per media type.
	152	+	accept := strings.Join(r.Header.Values("Accept"), ",")
	153	+	if len(name) > 255 \|\| !namePattern.MatchString(name) \|\| len(accept) > 8192 {
	154	+	http.NotFound(w, r)
	155	+	return
	156	+	}
	157	+	if kind == "manifests" && (validDigest(ref) \|\| tagPattern.MatchString(ref)) {
	158	+	p.manifest(w, r, name, ref, normalizeAccept(accept))
	159	+	return
	160	+	}
	161	+	if kind == "blobs" && validDigest(ref) {
	162	+	p.blob(w, r, name, ref)
	163	+	return
	164	+	}
	165	+	http.NotFound(w, r)
	166	+	}
	167	+	func serveFile(w http.ResponseWriter, r http.Request, f storedFile, media, digest string) {
	168	+	defer func() { _ = f.Close() }()
	169	+	fi, err := f.Stat()
	170	+	if err != nil {
	171	+	http.Error(w, "registry cache read failed", http.StatusBadGateway)
	172	+	return
	173	+	}
	174	+	w.Header().Set("Content-Type", media)
	175	+	w.Header().Set("Docker-Content-Digest", digest)
	176	+	http.ServeContent(w, r, "", fi.ModTime(), f)
	177	+	}
	178	+
	179	+	// manifest serves a manifest by tag or digest. accept is normalized.
	180	+	func (p Proxy) manifest(w http.ResponseWriter, r http.Request, name, ref, accept string) {
	181	+	start := time.Now()
	182	+	hit := false
	183	+	defer func() {
	184	+	p.log.Info().Str("repo", name).Str("ref", ref).Bool("hit", hit).Int64("ms", time.Since(start).Milliseconds()).Msg("registry manifest")
	185	+	}()
	186	+	ctx, cancel := context.WithTimeout(r.Context(), p.u.requestTimeout)
	187	+	defer cancel()
	188	+	digest := ref
	189	+	tag := !validDigest(ref)
	190	+	if tag {
	191	+	resp, err := p.u.request(ctx, http.MethodHead, name, "manifests", ref, accept)
	192	+	if err != nil {
	193	+	if unavailable(err) {
	194	+	digest = p.s.tag(name, ref, accept)
	195	+	if validDigest(digest) {
	196	+	if f, media, e := p.s.open("manifests", digest); e == nil {
	197	+	hit = true
	198	+	serveFile(w, r, f, media, digest)
	199	+	return
	200	+	}
	201	+	}
	202	+	}
	203	+	http.Error(w, "registry upstream unavailable", proxyStatus(err))
	204	+	return
	205	+	}
	206	+	digest = resp.Header.Get("Docker-Content-Digest")
	207	+	_ = resp.Body.Close()
	208	+	if !validDigest(digest) {
	209	+	http.Error(w, "invalid upstream digest", http.StatusBadGateway)
	210	+	return
	211	+	}
	212	+	}
	213	+	// Shared flight, as for blobs: it must not end with the initiator's request.
	214	+	f, media, cached, err := p.s.ensure(ctx, "manifests", digest, func(flightCtx context.Context, dst io.Writer) (string, error) {
	215	+	fillCtx, fillCancel := context.WithTimeout(flightCtx, p.u.requestTimeout)
	216	+	defer fillCancel()
	217	+	resp, err := p.u.request(fillCtx, http.MethodGet, name, "manifests", digest, accept)
	218	+	if err != nil {
	219	+	return "", err
	220	+	}
	221	+	defer func() { _ = resp.Body.Close() }()
	222	+	b, err := io.ReadAll(io.LimitReader(resp.Body, manifestLimit+1))
	223	+	if err != nil {
	224	+	return "", err
	225	+	}
	226	+	if len(b) > manifestLimit {
	227	+	return "", errTooLarge
	228	+	}
	229	+	_, err = dst.Write(b)
	230	+	return resp.Header.Get("Content-Type"), err
	231	+	}, keepalive{})
	232	+	hit = cached
	233	+	if err != nil {
	234	+	if tag && unavailable(err) {
	235	+	previous := p.s.tag(name, ref, accept)
	236	+	if validDigest(previous) {
	237	+	if old, oldMedia, e := p.s.open("manifests", previous); e == nil {
	238	+	hit = true
	239	+	serveFile(w, r, old, oldMedia, previous)
	240	+	return
	241	+	}
	242	+	}
	243	+	}
	244	+	http.Error(w, "registry fetch failed", proxyStatus(err))
	245	+	return
	246	+	}
	247	+	if tag {
	248	+	_ = p.s.setTag(name, ref, accept, digest)
	249	+	}
	250	+	serveFile(w, r, f, media, digest)
	251	+	}
	252	+	func (p Proxy) blob(w http.ResponseWriter, r http.Request, name, digest string) {
	253	+	if f, media, err := p.s.open("blobs", digest); err == nil {
	254	+	serveFile(w, r, f, media, digest)
	255	+	return
	256	+	}
	257	+	// A cold HEAD is metadata-only and retains the short total timeout.
	258	+	if r.Method == http.MethodHead {
	259	+	ctx, cancel := context.WithTimeout(r.Context(), p.u.requestTimeout)
	260	+	defer cancel()
	261	+	resp, err := p.u.request(ctx, http.MethodHead, name, "blobs", digest, "")
	262	+	if err != nil {
	263	+	http.Error(w, "registry fetch failed", proxyStatus(err))
	264	+	return
	265	+	}
	266	+	defer func() { _ = resp.Body.Close() }()
	267	+	if resp.ContentLength < 0 {
	268	+	http.Error(w, "unknown blob length", http.StatusBadGateway)
	269	+	return
	270	+	}
	271	+	w.Header().Set("Content-Type", "application/octet-stream")
	272	+	w.Header().Set("Content-Length", strconv.FormatInt(resp.ContentLength, 10))
	273	+	w.Header().Set("Docker-Content-Digest", digest)
	274	+	return
	275	+	}
	276	+	// The flight is shared by every client that asks for this blob, so no client
	277	+	// may own it: it downloads on a context of its own, into the temporary file
	278	+	// only, and is bounded by Max and the upstream idle timeout. Every client,
	279	+	// the initiator included, waits for the verified commit and then reads the
	280	+	// stored file (simplest correct option: no client streams unverified bytes,
	281	+	// and a client that disconnects fails only itself). The flight is cancelled
	282	+	// when no client has waited for it for AbandonGrace. At most MaxDownloads
	283	+	// run at once, and at most its Quota per client: a request over either
	284	+	// waits up to SlotWait for a slot, then gets 503, which dockerd retries
	285	+	// (about 50 s) before pulling from Docker Hub. While it waits, a writer
	286	+	// with KeepAlive (the cache channel's progress deadline) is kept alive as
	287	+	// long as the download moves.
	288	+	keep := keepalive{every: p.keepEvery}
	289	+	if k, ok := w.(interface{ KeepAlive() }); ok {
	290	+	keep.fn = k.KeepAlive
	291	+	}
	292	+	f, media, _, err := p.s.ensure(r.Context(), "blobs", digest, func(downloadCtx context.Context, dst io.Writer) (string, error) {
	293	+	start := time.Now()
	294	+	var size int64
	295	+	defer func() {
	296	+	p.log.Debug().Str("digest", digest).Int64("bytes", size).Int64("ms", time.Since(start).Milliseconds()).Msg("registry blob")
	297	+	}()
	298	+	resp, err := p.u.blob(downloadCtx, name, digest)
	299	+	if err != nil {
	300	+	return "", err
	301	+	}
	302	+	defer func() { _ = resp.Body.Close() }()
	303	+	if resp.ContentLength > p.s.max {
	304	+	return "", errTooLarge
	305	+	}
	306	+	size, err = io.Copy(dst, resp.Body)
	307	+	return "application/octet-stream", err
	308	+	}, keep)
	309	+	if errors.Is(err, errBusy) {
	310	+	w.Header().Set("Retry-After", "1")
	311	+	http.Error(w, "registry downloads busy", http.StatusServiceUnavailable)
	312	+	return
	313	+	}
	314	+	if err != nil {
	315	+	http.Error(w, "registry fetch failed", proxyStatus(err))
	316	+	return
	317	+	}
	318	+	serveFile(w, r, f, media, digest)
	319	+	}

@@ -0,0 +1,319 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	// Package registryproxy provides an anonymous, pull-only Docker Hub mirror.
	4	+	// Its handler belongs on a sandbox's private runtime channel, not a public port.
	5	+	package registryproxy
	6	+
	7	+	import (
	8	+	"context"
	9	+	"encoding/hex"
	10	+	"errors"
	11	+	"io"
	12	+	"net/http"
	13	+	"net/url"
	14	+	"regexp"
	15	+	"strconv"
	16	+	"strings"
	17	+	"time"
	18	+
	19	+	"github.com/bnema/zerowrap"
	20	+	)
	21	+
	22	+	const manifestLimit = 4 << 20
	23	+
	24	+	// Config configures a Proxy. Max must be positive. The directory is exclusively
	25	+	// owned by one Proxy; opening it clears incomplete downloads from tmp/.
	26	+	type Config struct {
	27	+	Dir string
	28	+	Max int64
	29	+	Upstream string // default https://registry-1.docker.io
	30	+	AuthURL string // default https://auth.docker.io/token
	31	+	Client *http.Client
	32	+	Now func() time.Time
	33	+	Log zerowrap.Logger
	34	+	// RequestTimeout bounds auth, manifests and HEAD (default 10s).
	35	+	RequestTimeout time.Duration
	36	+	// BlobIdleTimeout bounds time without upstream body progress (default 30s).
	37	+	// A blob body has no total timeout.
	38	+	BlobIdleTimeout time.Duration
	39	+	// KeepAliveInterval is how often a client waiting for a cold blob is kept
	40	+	// alive while the download progresses (default 1s), when its
	41	+	// ResponseWriter has a KeepAlive method.
	42	+	KeepAliveInterval time.Duration
	43	+	// AbandonGrace is how long a download nobody waits for any more keeps
	44	+	// running in case another request joins it; then it is cancelled and its
	45	+	// temporary file removed (default 5s).
	46	+	AbandonGrace time.Duration
	47	+	// MaxDownloads caps the concurrent cold blob downloads across all jobs
	48	+	// (default 16). A request may also carry a per-client Quota (WithQuota).
	49	+	MaxDownloads int
	50	+	// SlotWait is how long a request that would start a download waits for a
	51	+	// free slot, global or of its quota, before it gets 503 (default 30s).
	52	+	SlotWait time.Duration
	53	+	}
	54	+
	55	+	// Proxy implements http.Handler for GET and HEAD under /v2/.
	56	+	type Proxy struct {
	57	+	s *store
	58	+	u *upstream
	59	+	log zerowrap.Logger
	60	+	keepEvery time.Duration
	61	+	}
	62	+
	63	+	// New opens the persistent cache. It starts no goroutines or listeners.
	64	+	func New(c Config) (*Proxy, error) {
	65	+	if c.Now == nil {
	66	+	c.Now = time.Now
	67	+	}
	68	+	if c.Upstream == "" {
	69	+	c.Upstream = "https://registry-1.docker.io"
	70	+	}
	71	+	if c.AuthURL == "" {
	72	+	c.AuthURL = "https://auth.docker.io/token"
	73	+	}
	74	+	for _, s := range []string{c.Upstream, c.AuthURL} {
	75	+	u, err := url.Parse(s)
	76	+	if err != nil \|\| u.Host == "" \|\| u.User != nil \|\| (u.Scheme != "http" && u.Scheme != "https") {
	77	+	return nil, errors.New("invalid registry upstream URL")
	78	+	}
	79	+	}
	80	+	if c.RequestTimeout <= 0 {
	81	+	c.RequestTimeout = 10 * time.Second
	82	+	}
	83	+	if c.BlobIdleTimeout <= 0 {
	84	+	c.BlobIdleTimeout = 30 * time.Second
	85	+	}
	86	+	if c.KeepAliveInterval <= 0 {
	87	+	c.KeepAliveInterval = time.Second
	88	+	}
	89	+	if c.AbandonGrace <= 0 {
	90	+	c.AbandonGrace = 5 * time.Second
	91	+	}
	92	+	if c.MaxDownloads <= 0 {
	93	+	c.MaxDownloads = 16
	94	+	}
	95	+	if c.SlotWait <= 0 {
	96	+	c.SlotWait = 30 * time.Second
	97	+	}
	98	+	client, blobs := upstreamClients(c)
	99	+
	100	+	st, err := openStore(c.Dir, c.Max, c.Now)
	101	+	if err != nil {
	102	+	return nil, err
	103	+	}
	104	+	st.grace, st.maxDownloads, st.slotWait, st.log = c.AbandonGrace, c.MaxDownloads, c.SlotWait, c.Log
	105	+	return &Proxy{s: st, u: &upstream{base: strings.TrimRight(c.Upstream, "/"), auth: c.AuthURL, client: client, blobClient: blobs, requestTimeout: c.RequestTimeout, blobIdle: c.BlobIdleTimeout, now: c.Now, tokens: map[string]bearer{}}, log: c.Log, keepEvery: c.KeepAliveInterval}, nil
	106	+	}
	107	+
	108	+	var namePattern = regexp.MustCompile(`^[a-z0-9]+(?:[._-][a-z0-9]+)(?:/[a-z0-9]+(?:[._-][a-z0-9]+))*$`)
	109	+	var tagPattern = regexp.MustCompile(`^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$`)
	110	+
	111	+	func validDigest(s string) bool {
	112	+	if len(s) != 71 \|\| !strings.HasPrefix(s, "sha256:") {
	113	+	return false
	114	+	}
	115	+	_, err := hex.DecodeString(s[7:])
	116	+	return err == nil && s == strings.ToLower(s)
	117	+	}
	118	+
	119	+	func (p Proxy) ServeHTTP(w http.ResponseWriter, r http.Request) {
	120	+	if r.Method != http.MethodGet && r.Method != http.MethodHead {
	121	+	w.WriteHeader(http.StatusMethodNotAllowed)
	122	+	return
	123	+	}
	124	+	w.Header().Set("Docker-Distribution-Api-Version", "registry/2.0")
	125	+	if r.URL.Path == "/v2/" {
	126	+	w.Header().Set("Content-Type", "application/json")
	127	+	w.Header().Set("Content-Length", "2")
	128	+	if r.Method == http.MethodGet {
	129	+	_, _ = io.WriteString(w, "{}")
	130	+	}
	131	+	return
	132	+	}
	133	+	if !strings.HasPrefix(r.URL.Path, "/v2/") {
	134	+	http.NotFound(w, r)
	135	+	return
	136	+	}
	137	+	path := strings.TrimPrefix(r.URL.Path, "/v2/")
	138	+	i := strings.LastIndex(path, "/")
	139	+	if i < 0 {
	140	+	http.NotFound(w, r)
	141	+	return
	142	+	}
	143	+	ref := path[i+1:]
	144	+	path = path[:i]
	145	+	i = strings.LastIndex(path, "/")
	146	+	if i < 0 {
	147	+	http.NotFound(w, r)
	148	+	return
	149	+	}
	150	+	name, kind := path[:i], path[i+1:]
	151	+	// dockerd's legacy client sends one Accept line per media type.
	152	+	accept := strings.Join(r.Header.Values("Accept"), ",")
	153	+	if len(name) > 255 \|\| !namePattern.MatchString(name) \|\| len(accept) > 8192 {
	154	+	http.NotFound(w, r)
	155	+	return
	156	+	}
	157	+	if kind == "manifests" && (validDigest(ref) \|\| tagPattern.MatchString(ref)) {
	158	+	p.manifest(w, r, name, ref, normalizeAccept(accept))
	159	+	return
	160	+	}
	161	+	if kind == "blobs" && validDigest(ref) {
	162	+	p.blob(w, r, name, ref)
	163	+	return
	164	+	}
	165	+	http.NotFound(w, r)
	166	+	}
	167	+	func serveFile(w http.ResponseWriter, r http.Request, f storedFile, media, digest string) {
	168	+	defer func() { _ = f.Close() }()
	169	+	fi, err := f.Stat()
	170	+	if err != nil {
	171	+	http.Error(w, "registry cache read failed", http.StatusBadGateway)
	172	+	return
	173	+	}
	174	+	w.Header().Set("Content-Type", media)
	175	+	w.Header().Set("Docker-Content-Digest", digest)
	176	+	http.ServeContent(w, r, "", fi.ModTime(), f)
	177	+	}
	178	+
	179	+	// manifest serves a manifest by tag or digest. accept is normalized.
	180	+	func (p Proxy) manifest(w http.ResponseWriter, r http.Request, name, ref, accept string) {
	181	+	start := time.Now()
	182	+	hit := false
	183	+	defer func() {
	184	+	p.log.Info().Str("repo", name).Str("ref", ref).Bool("hit", hit).Int64("ms", time.Since(start).Milliseconds()).Msg("registry manifest")
	185	+	}()
	186	+	ctx, cancel := context.WithTimeout(r.Context(), p.u.requestTimeout)
	187	+	defer cancel()
	188	+	digest := ref
	189	+	tag := !validDigest(ref)
	190	+	if tag {
	191	+	resp, err := p.u.request(ctx, http.MethodHead, name, "manifests", ref, accept)
	192	+	if err != nil {
	193	+	if unavailable(err) {
	194	+	digest = p.s.tag(name, ref, accept)
	195	+	if validDigest(digest) {
	196	+	if f, media, e := p.s.open("manifests", digest); e == nil {
	197	+	hit = true
	198	+	serveFile(w, r, f, media, digest)
	199	+	return
	200	+	}
	201	+	}
	202	+	}
	203	+	http.Error(w, "registry upstream unavailable", proxyStatus(err))
	204	+	return
	205	+	}
	206	+	digest = resp.Header.Get("Docker-Content-Digest")
	207	+	_ = resp.Body.Close()
	208	+	if !validDigest(digest) {
	209	+	http.Error(w, "invalid upstream digest", http.StatusBadGateway)
	210	+	return
	211	+	}
	212	+	}
	213	+	// Shared flight, as for blobs: it must not end with the initiator's request.
	214	+	f, media, cached, err := p.s.ensure(ctx, "manifests", digest, func(flightCtx context.Context, dst io.Writer) (string, error) {
	215	+	fillCtx, fillCancel := context.WithTimeout(flightCtx, p.u.requestTimeout)
	216	+	defer fillCancel()
	217	+	resp, err := p.u.request(fillCtx, http.MethodGet, name, "manifests", digest, accept)
	218	+	if err != nil {
	219	+	return "", err
	220	+	}
	221	+	defer func() { _ = resp.Body.Close() }()
	222	+	b, err := io.ReadAll(io.LimitReader(resp.Body, manifestLimit+1))
	223	+	if err != nil {
	224	+	return "", err
	225	+	}
	226	+	if len(b) > manifestLimit {
	227	+	return "", errTooLarge
	228	+	}
	229	+	_, err = dst.Write(b)
	230	+	return resp.Header.Get("Content-Type"), err
	231	+	}, keepalive{})
	232	+	hit = cached
	233	+	if err != nil {
	234	+	if tag && unavailable(err) {
	235	+	previous := p.s.tag(name, ref, accept)
	236	+	if validDigest(previous) {
	237	+	if old, oldMedia, e := p.s.open("manifests", previous); e == nil {
	238	+	hit = true
	239	+	serveFile(w, r, old, oldMedia, previous)
	240	+	return
	241	+	}
	242	+	}
	243	+	}
	244	+	http.Error(w, "registry fetch failed", proxyStatus(err))
	245	+	return
	246	+	}
	247	+	if tag {
	248	+	_ = p.s.setTag(name, ref, accept, digest)
	249	+	}
	250	+	serveFile(w, r, f, media, digest)
	251	+	}
	252	+	func (p Proxy) blob(w http.ResponseWriter, r http.Request, name, digest string) {
	253	+	if f, media, err := p.s.open("blobs", digest); err == nil {
	254	+	serveFile(w, r, f, media, digest)
	255	+	return
	256	+	}
	257	+	// A cold HEAD is metadata-only and retains the short total timeout.
	258	+	if r.Method == http.MethodHead {
	259	+	ctx, cancel := context.WithTimeout(r.Context(), p.u.requestTimeout)
	260	+	defer cancel()
	261	+	resp, err := p.u.request(ctx, http.MethodHead, name, "blobs", digest, "")
	262	+	if err != nil {
	263	+	http.Error(w, "registry fetch failed", proxyStatus(err))
	264	+	return
	265	+	}
	266	+	defer func() { _ = resp.Body.Close() }()
	267	+	if resp.ContentLength < 0 {
	268	+	http.Error(w, "unknown blob length", http.StatusBadGateway)
	269	+	return
	270	+	}
	271	+	w.Header().Set("Content-Type", "application/octet-stream")
	272	+	w.Header().Set("Content-Length", strconv.FormatInt(resp.ContentLength, 10))
	273	+	w.Header().Set("Docker-Content-Digest", digest)
	274	+	return
	275	+	}
	276	+	// The flight is shared by every client that asks for this blob, so no client
	277	+	// may own it: it downloads on a context of its own, into the temporary file
	278	+	// only, and is bounded by Max and the upstream idle timeout. Every client,
	279	+	// the initiator included, waits for the verified commit and then reads the
	280	+	// stored file (simplest correct option: no client streams unverified bytes,
	281	+	// and a client that disconnects fails only itself). The flight is cancelled
	282	+	// when no client has waited for it for AbandonGrace. At most MaxDownloads
	283	+	// run at once, and at most its Quota per client: a request over either
	284	+	// waits up to SlotWait for a slot, then gets 503, which dockerd retries
	285	+	// (about 50 s) before pulling from Docker Hub. While it waits, a writer
	286	+	// with KeepAlive (the cache channel's progress deadline) is kept alive as
	287	+	// long as the download moves.
	288	+	keep := keepalive{every: p.keepEvery}
	289	+	if k, ok := w.(interface{ KeepAlive() }); ok {
	290	+	keep.fn = k.KeepAlive
	291	+	}
	292	+	f, media, _, err := p.s.ensure(r.Context(), "blobs", digest, func(downloadCtx context.Context, dst io.Writer) (string, error) {
	293	+	start := time.Now()
	294	+	var size int64
	295	+	defer func() {
	296	+	p.log.Debug().Str("digest", digest).Int64("bytes", size).Int64("ms", time.Since(start).Milliseconds()).Msg("registry blob")
	297	+	}()
	298	+	resp, err := p.u.blob(downloadCtx, name, digest)
	299	+	if err != nil {
	300	+	return "", err
	301	+	}
	302	+	defer func() { _ = resp.Body.Close() }()
	303	+	if resp.ContentLength > p.s.max {
	304	+	return "", errTooLarge
	305	+	}
	306	+	size, err = io.Copy(dst, resp.Body)
	307	+	return "application/octet-stream", err
	308	+	}, keep)
	309	+	if errors.Is(err, errBusy) {
	310	+	w.Header().Set("Retry-After", "1")
	311	+	http.Error(w, "registry downloads busy", http.StatusServiceUnavailable)
	312	+	return
	313	+	}
	314	+	if err != nil {
	315	+	http.Error(w, "registry fetch failed", proxyStatus(err))
	316	+	return
	317	+	}
	318	+	serveFile(w, r, f, media, digest)
	319	+	}

A internal/registryproxy/proxy_test.go +214 -0

@@ -0,0 +1,214 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+	package registryproxy
	3	+
	4	+	import (
	5	+	"fmt"
	6	+	"io"
	7	+	"net/http"
	8	+	"net/http/httptest"
	9	+	"os"
	10	+	"path/filepath"
	11	+	"strings"
	12	+	"sync/atomic"
	13	+	"testing"
	14	+	)
	15	+
	16	+	const manifestBody = `{"schemaVersion":2}`
	17	+	const blobBody = "a layer of bytes"
	18	+
	19	+	type fakeRegistry struct {
	20	+	server *httptest.Server
	21	+	heads, gets, blobs, tokens atomic.Int32
	22	+	status atomic.Int32
	23	+	corrupt atomic.Bool
	24	+	}
	25	+
	26	+	func fake(t testing.T) fakeRegistry {
	27	+	t.Helper()
	28	+	f := &fakeRegistry{}
	29	+	cdn := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	30	+	if r.Header.Get("Authorization") != "" {
	31	+	t.Error("bearer leaked to CDN")
	32	+	}
	33	+	w.Header().Set("Content-Length", fmt.Sprint(len(blobBody)))
	34	+	if r.Method != http.MethodHead {
	35	+	b := blobBody
	36	+	if f.corrupt.Load() {
	37	+	b = strings.Repeat("x", len(b))
	38	+	}
	39	+	_, _ = io.WriteString(w, b)
	40	+	}
	41	+	}))
	42	+	t.Cleanup(cdn.Close)
	43	+	f.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	44	+	if r.URL.Path == "/token" {
	45	+	f.tokens.Add(1)
	46	+	if r.Header.Get("Authorization") != "" {
	47	+	t.Error("guest credentials forwarded to auth")
	48	+	}
	49	+	if r.URL.Query().Get("scope") != "repository:library/alpine:pull" {
	50	+	t.Error("wrong scope")
	51	+	}
	52	+	_, _ = io.WriteString(w, `{"token":"public","expires_in":300}`)
	53	+	return
	54	+	}
	55	+	if r.Header.Get("Authorization") != "Bearer public" {
	56	+	t.Errorf("wrong bearer %q", r.Header.Get("Authorization"))
	57	+	}
	58	+	if s := f.status.Load(); s != 0 {
	59	+	w.WriteHeader(int(s))
	60	+	return
	61	+	}
	62	+	if strings.Contains(r.URL.Path, "/manifests/") {
	63	+	w.Header().Set("Content-Type", "application/vnd.oci.image.index.v1+json")
	64	+	w.Header().Set("Docker-Content-Digest", sum(manifestBody))
	65	+	w.Header().Set("Content-Length", fmt.Sprint(len(manifestBody)))
	66	+	if r.Method == http.MethodHead {
	67	+	f.heads.Add(1)
	68	+	} else {
	69	+	f.gets.Add(1)
	70	+	_, _ = io.WriteString(w, manifestBody)
	71	+	}
	72	+	return
	73	+	}
	74	+	if strings.Contains(r.URL.Path, "/blobs/") {
	75	+	if r.Method == http.MethodGet {
	76	+	f.blobs.Add(1)
	77	+	}
	78	+	http.Redirect(w, r, cdn.URL+"/layer", http.StatusTemporaryRedirect)
	79	+	return
	80	+	}
	81	+	http.NotFound(w, r)
	82	+	}))
	83	+	t.Cleanup(f.server.Close)
	84	+	return f
	85	+	}
	86	+	func proxyFor(t testing.T, f fakeRegistry) *Proxy {
	87	+	t.Helper()
	88	+	p, err := New(Config{Dir: t.TempDir(), Max: 8 << 20, Upstream: f.server.URL, AuthURL: f.server.URL + "/token"})
	89	+	if err != nil {
	90	+	t.Fatal(err)
	91	+	}
	92	+	return p
	93	+	}
	94	+	func request(t testing.T, p http.Handler, method, path string, headers ...string) httptest.ResponseRecorder {
	95	+	t.Helper()
	96	+	r := httptest.NewRequest(method, path, nil)
	97	+	r.Header.Set("Authorization", "Bearer guest-secret")
	98	+	for i := 0; i < len(headers); i += 2 {
	99	+	r.Header.Set(headers[i], headers[i+1])
	100	+	}
	101	+	w := httptest.NewRecorder()
	102	+	p.ServeHTTP(w, r)
	103	+	return w
	104	+	}
	105	+	func TestPullAndOfflineTag(t *testing.T) {
	106	+	f := fake(t)
	107	+	p := proxyFor(t, f)
	108	+	for range 2 {
	109	+	w := request(t, p, http.MethodGet, "/v2/library/alpine/manifests/3")
	110	+	if w.Code != http.StatusOK \|\| w.Body.String() != manifestBody {
	111	+	t.Fatalf("manifest: %d %s", w.Code, w.Body)
	112	+	}
	113	+	w = request(t, p, http.MethodGet, "/v2/library/alpine/blobs/"+sum(blobBody))
	114	+	if w.Code != http.StatusOK \|\| w.Body.String() != blobBody {
	115	+	t.Fatalf("blob: %d %s", w.Code, w.Body)
	116	+	}
	117	+	}
	118	+	if f.heads.Load() != 2 \|\| f.gets.Load() != 1 \|\| f.blobs.Load() != 1 \|\| f.tokens.Load() != 1 {
	119	+	t.Fatalf("counts HEAD=%d GET=%d blobs=%d tokens=%d", f.heads.Load(), f.gets.Load(), f.blobs.Load(), f.tokens.Load())
	120	+	}
	121	+	for _, down := range []int32{http.StatusServiceUnavailable, 0} {
	122	+	f.status.Store(down)
	123	+	if down == 0 {
	124	+	f.server.Close()
	125	+	}
	126	+	if w := request(t, p, http.MethodGet, "/v2/library/alpine/manifests/3"); w.Code != http.StatusOK \|\| w.Body.String() != manifestBody {
	127	+	t.Fatalf("stale: %d %s", w.Code, w.Body)
	128	+	}
	129	+	if w := request(t, p, http.MethodGet, "/v2/library/alpine/manifests/unknown"); w.Code != http.StatusBadGateway {
	130	+	t.Fatalf("unknown: %d", w.Code)
	131	+	}
	132	+	}
	133	+	}
	134	+	func TestBlobHeadAndRange(t *testing.T) {
	135	+	f := fake(t)
	136	+	p := proxyFor(t, f)
	137	+	path := "/v2/library/alpine/blobs/" + sum(blobBody)
	138	+	for _, cached := range []bool{false, true} {
	139	+	if cached {
	140	+	if w := request(t, p, http.MethodGet, path); w.Code != http.StatusOK {
	141	+	t.Fatal(w.Code)
	142	+	}
	143	+	}
	144	+	w := request(t, p, http.MethodHead, path)
	145	+	if w.Code != http.StatusOK \|\| w.Body.Len() != 0 \|\| w.Header().Get("Content-Length") != fmt.Sprint(len(blobBody)) {
	146	+	t.Fatalf("HEAD %d %v %q", w.Code, w.Header(), w.Body.String())
	147	+	}
	148	+	}
	149	+	w := request(t, p, http.MethodGet, path, "Range", "bytes=2-5")
	150	+	if w.Code != http.StatusPartialContent \|\| w.Body.String() != blobBody[2:6] {
	151	+	t.Fatalf("range: %d %s", w.Code, w.Body)
	152	+	}
	153	+	}
	154	+
	155	+	// A blob is verified before any byte of it is sent: a corrupt one is a 502
	156	+	// with no body, and is not stored.
	157	+	func TestCorruptBlobRejected(t *testing.T) {
	158	+	f := fake(t)
	159	+	f.corrupt.Store(true)
	160	+	p := proxyFor(t, f)
	161	+	w := request(t, p, http.MethodGet, "/v2/library/alpine/blobs/"+sum(blobBody))
	162	+	if w.Code != http.StatusBadGateway \|\| strings.Contains(w.Body.String(), "xxxx") {
	163	+	t.Fatalf("corrupt blob: %d %q", w.Code, w.Body)
	164	+	}
	165	+	if _, err := os.Stat(filepath.Join(p.s.dir, "blobs", sum(blobBody)[7:])); !os.IsNotExist(err) {
	166	+	t.Fatal("corrupt blob stored")
	167	+	}
	168	+	ds, _ := os.ReadDir(filepath.Join(p.s.dir, "tmp"))
	169	+	if len(ds) != 0 {
	170	+	t.Fatal("partial temp retained")
	171	+	}
	172	+	}
	173	+	func TestDeniedAndValidation(t *testing.T) {
	174	+	f := fake(t)
	175	+	p := proxyFor(t, f)
	176	+	f.status.Store(http.StatusUnauthorized)
	177	+	if w := request(t, p, http.MethodGet, "/v2/library/alpine/manifests/3"); w.Code != http.StatusNotFound {
	178	+	t.Fatal(w.Code)
	179	+	}
	180	+	if f.tokens.Load() != 2 {
	181	+	t.Fatal("no fresh token retry")
	182	+	}
	183	+	for _, path := range []string{"/v2/../library/alpine/manifests/3", "/v2/Library/alpine/manifests/3", "/v2/library/alpine/manifests/.bad", "/v2/library/alpine/manifests/sha256:bad", "/v2/library/alpine/blobs/sha256:bad", "/v2/library/alpine/tags/list"} {
	184	+	if w := request(t, p, http.MethodGet, path); w.Code != http.StatusNotFound {
	185	+	t.Errorf("%s: %d", path, w.Code)
	186	+	}
	187	+	}
	188	+	if w := request(t, p, http.MethodPut, "/v2/"); w.Code != http.StatusMethodNotAllowed {
	189	+	t.Fatal(w.Code)
	190	+	}
	191	+	}
	192	+	func TestManifestLimitsAndDigest(t *testing.T) {
	193	+	for _, body := range []string{"tampered", strings.Repeat("x", manifestLimit+1)} {
	194	+	up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	195	+	if r.URL.Path == "/token" {
	196	+	_, _ = io.WriteString(w, `{"token":"public"}`)
	197	+	return
	198	+	}
	199	+	_, _ = io.WriteString(w, body)
	200	+	}))
	201	+	p, err := New(Config{Dir: t.TempDir(), Max: 8 << 20, Upstream: up.URL, AuthURL: up.URL + "/token"})
	202	+	if err != nil {
	203	+	t.Fatal(err)
	204	+	}
	205	+	w := request(t, p, http.MethodGet, "/v2/library/alpine/manifests/"+sum(manifestBody))
	206	+	up.Close()
	207	+	if w.Code != http.StatusBadGateway {
	208	+	t.Fatal(w.Code)
	209	+	}
	210	+	if _, _, err := p.s.open("manifests", sum(manifestBody)); err == nil {
	211	+	t.Fatal("invalid manifest stored")
	212	+	}
	213	+	}
	214	+	}

@@ -0,0 +1,214 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+	package registryproxy
	3	+
	4	+	import (
	5	+	"fmt"
	6	+	"io"
	7	+	"net/http"
	8	+	"net/http/httptest"
	9	+	"os"
	10	+	"path/filepath"
	11	+	"strings"
	12	+	"sync/atomic"
	13	+	"testing"
	14	+	)
	15	+
	16	+	const manifestBody = `{"schemaVersion":2}`
	17	+	const blobBody = "a layer of bytes"
	18	+
	19	+	type fakeRegistry struct {
	20	+	server *httptest.Server
	21	+	heads, gets, blobs, tokens atomic.Int32
	22	+	status atomic.Int32
	23	+	corrupt atomic.Bool
	24	+	}
	25	+
	26	+	func fake(t testing.T) fakeRegistry {
	27	+	t.Helper()
	28	+	f := &fakeRegistry{}
	29	+	cdn := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	30	+	if r.Header.Get("Authorization") != "" {
	31	+	t.Error("bearer leaked to CDN")
	32	+	}
	33	+	w.Header().Set("Content-Length", fmt.Sprint(len(blobBody)))
	34	+	if r.Method != http.MethodHead {
	35	+	b := blobBody
	36	+	if f.corrupt.Load() {
	37	+	b = strings.Repeat("x", len(b))
	38	+	}
	39	+	_, _ = io.WriteString(w, b)
	40	+	}
	41	+	}))
	42	+	t.Cleanup(cdn.Close)
	43	+	f.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	44	+	if r.URL.Path == "/token" {
	45	+	f.tokens.Add(1)
	46	+	if r.Header.Get("Authorization") != "" {
	47	+	t.Error("guest credentials forwarded to auth")
	48	+	}
	49	+	if r.URL.Query().Get("scope") != "repository:library/alpine:pull" {
	50	+	t.Error("wrong scope")
	51	+	}
	52	+	_, _ = io.WriteString(w, `{"token":"public","expires_in":300}`)
	53	+	return
	54	+	}
	55	+	if r.Header.Get("Authorization") != "Bearer public" {
	56	+	t.Errorf("wrong bearer %q", r.Header.Get("Authorization"))
	57	+	}
	58	+	if s := f.status.Load(); s != 0 {
	59	+	w.WriteHeader(int(s))
	60	+	return
	61	+	}
	62	+	if strings.Contains(r.URL.Path, "/manifests/") {
	63	+	w.Header().Set("Content-Type", "application/vnd.oci.image.index.v1+json")
	64	+	w.Header().Set("Docker-Content-Digest", sum(manifestBody))
	65	+	w.Header().Set("Content-Length", fmt.Sprint(len(manifestBody)))
	66	+	if r.Method == http.MethodHead {
	67	+	f.heads.Add(1)
	68	+	} else {
	69	+	f.gets.Add(1)
	70	+	_, _ = io.WriteString(w, manifestBody)
	71	+	}
	72	+	return
	73	+	}
	74	+	if strings.Contains(r.URL.Path, "/blobs/") {
	75	+	if r.Method == http.MethodGet {
	76	+	f.blobs.Add(1)
	77	+	}
	78	+	http.Redirect(w, r, cdn.URL+"/layer", http.StatusTemporaryRedirect)
	79	+	return
	80	+	}
	81	+	http.NotFound(w, r)
	82	+	}))
	83	+	t.Cleanup(f.server.Close)
	84	+	return f
	85	+	}
	86	+	func proxyFor(t testing.T, f fakeRegistry) *Proxy {
	87	+	t.Helper()
	88	+	p, err := New(Config{Dir: t.TempDir(), Max: 8 << 20, Upstream: f.server.URL, AuthURL: f.server.URL + "/token"})
	89	+	if err != nil {
	90	+	t.Fatal(err)
	91	+	}
	92	+	return p
	93	+	}
	94	+	func request(t testing.T, p http.Handler, method, path string, headers ...string) httptest.ResponseRecorder {
	95	+	t.Helper()
	96	+	r := httptest.NewRequest(method, path, nil)
	97	+	r.Header.Set("Authorization", "Bearer guest-secret")
	98	+	for i := 0; i < len(headers); i += 2 {
	99	+	r.Header.Set(headers[i], headers[i+1])
	100	+	}
	101	+	w := httptest.NewRecorder()
	102	+	p.ServeHTTP(w, r)
	103	+	return w
	104	+	}
	105	+	func TestPullAndOfflineTag(t *testing.T) {
	106	+	f := fake(t)
	107	+	p := proxyFor(t, f)
	108	+	for range 2 {
	109	+	w := request(t, p, http.MethodGet, "/v2/library/alpine/manifests/3")
	110	+	if w.Code != http.StatusOK \|\| w.Body.String() != manifestBody {
	111	+	t.Fatalf("manifest: %d %s", w.Code, w.Body)
	112	+	}
	113	+	w = request(t, p, http.MethodGet, "/v2/library/alpine/blobs/"+sum(blobBody))
	114	+	if w.Code != http.StatusOK \|\| w.Body.String() != blobBody {
	115	+	t.Fatalf("blob: %d %s", w.Code, w.Body)
	116	+	}
	117	+	}
	118	+	if f.heads.Load() != 2 \|\| f.gets.Load() != 1 \|\| f.blobs.Load() != 1 \|\| f.tokens.Load() != 1 {
	119	+	t.Fatalf("counts HEAD=%d GET=%d blobs=%d tokens=%d", f.heads.Load(), f.gets.Load(), f.blobs.Load(), f.tokens.Load())
	120	+	}
	121	+	for _, down := range []int32{http.StatusServiceUnavailable, 0} {
	122	+	f.status.Store(down)
	123	+	if down == 0 {
	124	+	f.server.Close()
	125	+	}
	126	+	if w := request(t, p, http.MethodGet, "/v2/library/alpine/manifests/3"); w.Code != http.StatusOK \|\| w.Body.String() != manifestBody {
	127	+	t.Fatalf("stale: %d %s", w.Code, w.Body)
	128	+	}
	129	+	if w := request(t, p, http.MethodGet, "/v2/library/alpine/manifests/unknown"); w.Code != http.StatusBadGateway {
	130	+	t.Fatalf("unknown: %d", w.Code)
	131	+	}
	132	+	}
	133	+	}
	134	+	func TestBlobHeadAndRange(t *testing.T) {
	135	+	f := fake(t)
	136	+	p := proxyFor(t, f)
	137	+	path := "/v2/library/alpine/blobs/" + sum(blobBody)
	138	+	for _, cached := range []bool{false, true} {
	139	+	if cached {
	140	+	if w := request(t, p, http.MethodGet, path); w.Code != http.StatusOK {
	141	+	t.Fatal(w.Code)
	142	+	}
	143	+	}
	144	+	w := request(t, p, http.MethodHead, path)
	145	+	if w.Code != http.StatusOK \|\| w.Body.Len() != 0 \|\| w.Header().Get("Content-Length") != fmt.Sprint(len(blobBody)) {
	146	+	t.Fatalf("HEAD %d %v %q", w.Code, w.Header(), w.Body.String())
	147	+	}
	148	+	}
	149	+	w := request(t, p, http.MethodGet, path, "Range", "bytes=2-5")
	150	+	if w.Code != http.StatusPartialContent \|\| w.Body.String() != blobBody[2:6] {
	151	+	t.Fatalf("range: %d %s", w.Code, w.Body)
	152	+	}
	153	+	}
	154	+
	155	+	// A blob is verified before any byte of it is sent: a corrupt one is a 502
	156	+	// with no body, and is not stored.
	157	+	func TestCorruptBlobRejected(t *testing.T) {
	158	+	f := fake(t)
	159	+	f.corrupt.Store(true)
	160	+	p := proxyFor(t, f)
	161	+	w := request(t, p, http.MethodGet, "/v2/library/alpine/blobs/"+sum(blobBody))
	162	+	if w.Code != http.StatusBadGateway \|\| strings.Contains(w.Body.String(), "xxxx") {
	163	+	t.Fatalf("corrupt blob: %d %q", w.Code, w.Body)
	164	+	}
	165	+	if _, err := os.Stat(filepath.Join(p.s.dir, "blobs", sum(blobBody)[7:])); !os.IsNotExist(err) {
	166	+	t.Fatal("corrupt blob stored")
	167	+	}
	168	+	ds, _ := os.ReadDir(filepath.Join(p.s.dir, "tmp"))
	169	+	if len(ds) != 0 {
	170	+	t.Fatal("partial temp retained")
	171	+	}
	172	+	}
	173	+	func TestDeniedAndValidation(t *testing.T) {
	174	+	f := fake(t)
	175	+	p := proxyFor(t, f)
	176	+	f.status.Store(http.StatusUnauthorized)
	177	+	if w := request(t, p, http.MethodGet, "/v2/library/alpine/manifests/3"); w.Code != http.StatusNotFound {
	178	+	t.Fatal(w.Code)
	179	+	}
	180	+	if f.tokens.Load() != 2 {
	181	+	t.Fatal("no fresh token retry")
	182	+	}
	183	+	for _, path := range []string{"/v2/../library/alpine/manifests/3", "/v2/Library/alpine/manifests/3", "/v2/library/alpine/manifests/.bad", "/v2/library/alpine/manifests/sha256:bad", "/v2/library/alpine/blobs/sha256:bad", "/v2/library/alpine/tags/list"} {
	184	+	if w := request(t, p, http.MethodGet, path); w.Code != http.StatusNotFound {
	185	+	t.Errorf("%s: %d", path, w.Code)
	186	+	}
	187	+	}
	188	+	if w := request(t, p, http.MethodPut, "/v2/"); w.Code != http.StatusMethodNotAllowed {
	189	+	t.Fatal(w.Code)
	190	+	}
	191	+	}
	192	+	func TestManifestLimitsAndDigest(t *testing.T) {
	193	+	for _, body := range []string{"tampered", strings.Repeat("x", manifestLimit+1)} {
	194	+	up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	195	+	if r.URL.Path == "/token" {
	196	+	_, _ = io.WriteString(w, `{"token":"public"}`)
	197	+	return
	198	+	}
	199	+	_, _ = io.WriteString(w, body)
	200	+	}))
	201	+	p, err := New(Config{Dir: t.TempDir(), Max: 8 << 20, Upstream: up.URL, AuthURL: up.URL + "/token"})
	202	+	if err != nil {
	203	+	t.Fatal(err)
	204	+	}
	205	+	w := request(t, p, http.MethodGet, "/v2/library/alpine/manifests/"+sum(manifestBody))
	206	+	up.Close()
	207	+	if w.Code != http.StatusBadGateway {
	208	+	t.Fatal(w.Code)
	209	+	}
	210	+	if _, _, err := p.s.open("manifests", sum(manifestBody)); err == nil {
	211	+	t.Fatal("invalid manifest stored")
	212	+	}
	213	+	}
	214	+	}

A internal/registryproxy/quota.go +50 -0

@@ -0,0 +1,50 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+	package registryproxy
	3	+
	4	+	import (
	5	+	"context"
	6	+	"sync/atomic"
	7	+	)
	8	+
	9	+	// Quota bounds the cold blob downloads started on behalf of one client, such as
	10	+	// the channel of one sandbox, so that no client can take every download slot of
	11	+	// the Proxy. A download counts against the quota of the request that started it
	12	+	// until it really ends, even when only other clients still wait for it;
	13	+	// requests that join a download in progress take none.
	14	+	type Quota struct {
	15	+	max int64
	16	+	used atomic.Int64
	17	+	}
	18	+
	19	+	// NewQuota returns a quota of n concurrent downloads; n must be positive.
	20	+	func NewQuota(n int) *Quota { return &Quota{max: int64(max(n, 1))} }
	21	+
	22	+	// WithQuota attaches q to the context of a request to the Proxy. A request
	23	+	// without a quota is bound by the Proxy's global cap only.
	24	+	func WithQuota(ctx context.Context, q *Quota) context.Context {
	25	+	return context.WithValue(ctx, quotaKey{}, q)
	26	+	}
	27	+
	28	+	type quotaKey struct{}
	29	+
	30	+	// QuotaOf returns the quota attached by WithQuota, or nil.
	31	+	func QuotaOf(ctx context.Context) *Quota { return quotaFrom(ctx) }
	32	+
	33	+	func quotaFrom(ctx context.Context) *Quota {
	34	+	q, _ := ctx.Value(quotaKey{}).(*Quota)
	35	+	return q
	36	+	}
	37	+
	38	+	func (q *Quota) tryAcquire() bool {
	39	+	for {
	40	+	n := q.used.Load()
	41	+	if n >= q.max {
	42	+	return false
	43	+	}
	44	+	if q.used.CompareAndSwap(n, n+1) {
	45	+	return true
	46	+	}
	47	+	}
	48	+	}
	49	+
	50	+	func (q *Quota) release() { q.used.Add(-1) }

@@ -0,0 +1,50 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+	package registryproxy
	3	+
	4	+	import (
	5	+	"context"
	6	+	"sync/atomic"
	7	+	)
	8	+
	9	+	// Quota bounds the cold blob downloads started on behalf of one client, such as
	10	+	// the channel of one sandbox, so that no client can take every download slot of
	11	+	// the Proxy. A download counts against the quota of the request that started it
	12	+	// until it really ends, even when only other clients still wait for it;
	13	+	// requests that join a download in progress take none.
	14	+	type Quota struct {
	15	+	max int64
	16	+	used atomic.Int64
	17	+	}
	18	+
	19	+	// NewQuota returns a quota of n concurrent downloads; n must be positive.
	20	+	func NewQuota(n int) *Quota { return &Quota{max: int64(max(n, 1))} }
	21	+
	22	+	// WithQuota attaches q to the context of a request to the Proxy. A request
	23	+	// without a quota is bound by the Proxy's global cap only.
	24	+	func WithQuota(ctx context.Context, q *Quota) context.Context {
	25	+	return context.WithValue(ctx, quotaKey{}, q)
	26	+	}
	27	+
	28	+	type quotaKey struct{}
	29	+
	30	+	// QuotaOf returns the quota attached by WithQuota, or nil.
	31	+	func QuotaOf(ctx context.Context) *Quota { return quotaFrom(ctx) }
	32	+
	33	+	func quotaFrom(ctx context.Context) *Quota {
	34	+	q, _ := ctx.Value(quotaKey{}).(*Quota)
	35	+	return q
	36	+	}
	37	+
	38	+	func (q *Quota) tryAcquire() bool {
	39	+	for {
	40	+	n := q.used.Load()
	41	+	if n >= q.max {
	42	+	return false
	43	+	}
	44	+	if q.used.CompareAndSwap(n, n+1) {
	45	+	return true
	46	+	}
	47	+	}
	48	+	}
	49	+
	50	+	func (q *Quota) release() { q.used.Add(-1) }

A internal/registryproxy/quota_test.go +252 -0

@@ -0,0 +1,252 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+	package registryproxy
	3	+
	4	+	import (
	5	+	"context"
	6	+	"io"
	7	+	"net/http"
	8	+	"net/http/httptest"
	9	+	"os"
	10	+	"path/filepath"
	11	+	"strings"
	12	+	"sync"
	13	+	"sync/atomic"
	14	+	"testing"
	15	+	"time"
	16	+	)
	17	+
	18	+	// serveAsync serves path with ctx on its own goroutine, into w (a new
	19	+	// recorder when nil); the status code comes on the returned channel.
	20	+	func serveAsync(p Proxy, ctx context.Context, path string, w keepAliveRecorder) <-chan int {
	21	+	if w == nil {
	22	+	w = &keepAliveRecorder{ResponseRecorder: httptest.NewRecorder()}
	23	+	}
	24	+	code := make(chan int, 1)
	25	+	go func() {
	26	+	p.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil).WithContext(ctx))
	27	+	code <- w.Code
	28	+	}()
	29	+	return code
	30	+	}
	31	+
	32	+	// A sandbox that saturates its quota does not hold back another sandbox: its
	33	+	// download starts at once while the global cap still has room.
	34	+	func TestQuotaIsolatesChannels(t *testing.T) {
	35	+	u := newStallingUpstream(t)
	36	+	p := proxyWith(t, u.srv, Config{MaxDownloads: 16, SlotWait: time.Hour, AbandonGrace: 10 * time.Millisecond})
	37	+	a, b := NewQuota(3), NewQuota(3)
	38	+	ctxA, cancelA := context.WithCancel(WithQuota(context.Background(), a))
	39	+	defer cancelA()
	40	+	for i := range 3 {
	41	+	serveAsync(p, ctxA, blobPath(i), nil)
	42	+	}
	43	+	waitFor(t, p.s, "A's quota to fill", func() bool { return a.used.Load() == 3 })
	44	+	// A's fourth waits for its own quota and starts nothing.
	45	+	serveAsync(p, ctxA, blobPath(3), nil)
	46	+	ctxB, cancelB := context.WithCancel(WithQuota(context.Background(), b))
	47	+	defer cancelB()
	48	+	start := time.Now()
	49	+	serveAsync(p, ctxB, blobPath(10), nil)
	50	+	waitFor(t, p.s, "B's download to start", func() bool { return b.used.Load() == 1 })
	51	+	if d := time.Since(start); d > time.Second {
	52	+	t.Fatalf("B started after %v", d)
	53	+	}
	54	+	if got := u.started.Load(); got != 4 {
	55	+	t.Fatalf("%d downloads started, want A's 3 and B's 1", got)
	56	+	}
	57	+	cancelA()
	58	+	cancelB()
	59	+	waitFor(t, p.s, "every slot to free", func() bool { return p.s.downloads == 0 })
	60	+	if a.used.Load() != 0 \|\| b.used.Load() != 0 {
	61	+	t.Fatalf("quota not released: %d %d", a.used.Load(), b.used.Load())
	62	+	}
	63	+	}
	64	+
	65	+	// A request over its quota waits, kept alive, and starts its download as soon
	66	+	// as one of its channel's ends, with no 503.
	67	+	func TestQuotaWaiterGetsFreedSlot(t *testing.T) {
	68	+	body := strings.Repeat("layer", 100)
	69	+	release := make(chan struct{})
	70	+	var once sync.Once
	71	+	unblock := func() { once.Do(func() { close(release) }) }
	72	+	t.Cleanup(unblock)
	73	+	srv := upstreamWith(t, func(w http.ResponseWriter, r *http.Request) {
	74	+	if strings.HasSuffix(r.URL.Path, sum(body)) {
	75	+	_, _ = io.WriteString(w, body)
	76	+	return
	77	+	}
	78	+	w.Header().Set("Content-Length", "1000")
	79	+	_, _ = io.WriteString(w, "x")
	80	+	w.(http.Flusher).Flush()
	81	+	select {
	82	+	case <-release:
	83	+	case <-r.Context().Done():
	84	+	}
	85	+	})
	86	+	p := proxyWith(t, srv, Config{SlotWait: time.Hour, KeepAliveInterval: 5 * time.Millisecond})
	87	+	q := NewQuota(1)
	88	+	ctx := WithQuota(context.Background(), q)
	89	+	first := serveAsync(p, ctx, blobPath(0), nil)
	90	+	waitFor(t, p.s, "the quota to fill", func() bool { return q.used.Load() == 1 })
	91	+	w := &keepAliveRecorder{ResponseRecorder: httptest.NewRecorder()}
	92	+	second := serveAsync(p, ctx, "/v2/library/alpine/blobs/"+sum(body), w)
	93	+	for end := time.Now().Add(5 * time.Second); w.calls.Load() == 0; { // waiting for a slot, kept alive
	94	+	select {
	95	+	case code := <-second:
	96	+	t.Fatalf("did not wait for a slot: %d", code)
	97	+	default:
	98	+	}
	99	+	if time.Now().After(end) {
	100	+	t.Fatal("not kept alive while waiting for a slot")
	101	+	}
	102	+	time.Sleep(time.Millisecond)
	103	+	}
	104	+	unblock() // the first download ends (digest mismatch), freeing the slot
	105	+	if code := <-first; code != http.StatusBadGateway {
	106	+	t.Fatalf("first: %d", code)
	107	+	}
	108	+	if code := <-second; code != http.StatusOK \|\| w.Body.String() != body {
	109	+	t.Fatalf("second: %d", code)
	110	+	}
	111	+	}
	112	+
	113	+	// After SlotWait without a slot, the request gets 503.
	114	+	func TestQuotaWaitTimesOut(t *testing.T) {
	115	+	u := newStallingUpstream(t)
	116	+	const wait = 50 * time.Millisecond
	117	+	p := proxyWith(t, u.srv, Config{SlotWait: wait, AbandonGrace: 10 * time.Millisecond})
	118	+	q := NewQuota(1)
	119	+	ctx, cancel := context.WithCancel(WithQuota(context.Background(), q))
	120	+	defer cancel()
	121	+	serveAsync(p, ctx, blobPath(0), nil)
	122	+	waitFor(t, p.s, "the quota to fill", func() bool { return q.used.Load() == 1 })
	123	+	start := time.Now()
	124	+	code := <-serveAsync(p, ctx, blobPath(1), nil)
	125	+	if code != http.StatusServiceUnavailable {
	126	+	t.Fatalf("over the quota: %d", code)
	127	+	}
	128	+	if d := time.Since(start); d < wait {
	129	+	t.Fatalf("503 after %v", d)
	130	+	}
	131	+	// A request that disconnects while waiting leaves at once, with no slot.
	132	+	cctx, ccancel := context.WithCancel(WithQuota(context.Background(), q))
	133	+	waiting := serveAsync(p, cctx, blobPath(2), nil)
	134	+	ccancel()
	135	+	<-waiting
	136	+	if u.started.Load() != 1 {
	137	+	t.Fatalf("%d downloads started", u.started.Load())
	138	+	}
	139	+	}
	140	+
	141	+	// Joining a download in progress takes none of the joiner's quota.
	142	+	func TestQuotaJoinersExempt(t *testing.T) {
	143	+	u := newStallingUpstream(t)
	144	+	p := proxyWith(t, u.srv, Config{SlotWait: 20 * time.Millisecond, AbandonGrace: 10 * time.Millisecond})
	145	+	a, b := NewQuota(1), NewQuota(1)
	146	+	ctxA, cancelA := context.WithCancel(WithQuota(context.Background(), a))
	147	+	defer cancelA()
	148	+	serveAsync(p, ctxA, blobPath(0), nil)
	149	+	waitFor(t, p.s, "A's download", func() bool { return a.used.Load() == 1 })
	150	+	ctxB, cancelB := context.WithCancel(WithQuota(context.Background(), b))
	151	+	defer cancelB()
	152	+	serveAsync(p, ctxB, blobPath(0), nil) // joins A's
	153	+	key := "blobs/" + sum("blob-0")[7:]
	154	+	waitFor(t, p.s, "B to join", func() bool { return waiters(p.s, key) == 2 })
	155	+	if b.used.Load() != 0 {
	156	+	t.Fatal("a joiner took quota")
	157	+	}
	158	+	// B's quota is still free for a download of its own.
	159	+	serveAsync(p, ctxB, blobPath(1), nil)
	160	+	waitFor(t, p.s, "B's own download", func() bool { return b.used.Load() == 1 })
	161	+	}
	162	+
	163	+	// A panic in a download fails its flight and frees its slots; it does not
	164	+	// leave the waiters hanging.
	165	+	func TestPanickingFetchFailsFlight(t *testing.T) {
	166	+	s, err := openStore(t.TempDir(), 1<<20, time.Now)
	167	+	if err != nil {
	168	+	t.Fatal(err)
	169	+	}
	170	+	s.maxDownloads = 1
	171	+	q := NewQuota(1)
	172	+	ctx, cancel := context.WithTimeout(WithQuota(context.Background(), q), 5*time.Second)
	173	+	defer cancel()
	174	+	_, _, _, err = s.ensure(ctx, "blobs", sum("x"), func(context.Context, io.Writer) (string, error) {
	175	+	panic("boom")
	176	+	}, keepalive{})
	177	+	if err == nil \|\| !strings.Contains(err.Error(), "panicked") {
	178	+	t.Fatalf("err %v", err)
	179	+	}
	180	+	s.mu.Lock()
	181	+	defer s.mu.Unlock()
	182	+	if s.downloads != 0 \|\| len(s.flights) != 0 \|\| q.used.Load() != 0 \|\| s.pending != 0 {
	183	+	t.Fatalf("leaked: downloads=%d flights=%d quota=%d pending=%d", s.downloads, len(s.flights), q.used.Load(), s.pending)
	184	+	}
	185	+	if ds, _ := os.ReadDir(filepath.Join(s.dir, "tmp")); len(ds) != 0 {
	186	+	t.Fatal("temp file kept")
	187	+	}
	188	+	}
	189	+
	190	+	// Eviction restarts from the front after a manifest removal that fails, and
	191	+	// does not loop on it.
	192	+	func TestEvictionAfterFailedManifestRemoval(t *testing.T) {
	193	+	now := time.Unix(1_700_000_000, 0)
	194	+	const media = "application/vnd.oci.image.index.v1+json"
	195	+	s, err := openStore(t.TempDir(), 1<<20, func() time.Time { return now })
	196	+	if err != nil {
	197	+	t.Fatal(err)
	198	+	}
	199	+	m := `{"schemaVersion":2}`
	200	+	f, _, _, err := s.ensure(context.Background(), "manifests", sum(m), func(_ context.Context, w io.Writer) (string, error) {
	201	+	_, err := io.WriteString(w, m)
	202	+	return media, err
	203	+	}, keepalive{})
	204	+	if err != nil {
	205	+	t.Fatal(err)
	206	+	}
	207	+	_ = f.Close()
	208	+	now = now.Add(time.Second)
	209	+	if err := s.setTag("library/alpine", "3", "", sum(m)); err != nil {
	210	+	t.Fatal(err)
	211	+	}
	212	+	now = now.Add(time.Second)
	213	+	put(t, s, "blob")
	214	+	// The manifest's .type cannot be removed: replace it with a non-empty
	215	+	// directory. Its tag goes first all the same.
	216	+	typ := filepath.Join(s.dir, "manifests", sum(m)[7:]+".type")
	217	+	if err := os.Remove(typ); err != nil {
	218	+	t.Fatal(err)
	219	+	}
	220	+	if err := os.MkdirAll(filepath.Join(typ, "x"), 0700); err != nil {
	221	+	t.Fatal(err)
	222	+	}
	223	+	s.mu.Lock()
	224	+	s.evictLocked(s.max) // ask for everything
	225	+	_, manifestKept := s.entries["manifests/"+sum(m)[7:]]
	226	+	_, blobKept := s.entries["blobs/"+sum("blob")[7:]]
	227	+	tags := len(s.tagsOf)
	228	+	s.mu.Unlock()
	229	+	if !manifestKept \|\| blobKept \|\| tags != 0 {
	230	+	t.Fatalf("manifest kept=%v blob kept=%v tags=%d", manifestKept, blobKept, tags)
	231	+	}
	232	+	accounted(t, s)
	233	+	}
	234	+
	235	+	// unpinLocked of an unpinned entry is reported (and fails the test run through
	236	+	// TestMain's hook); here the hook is replaced to observe it.
	237	+	func TestUnpinBugReported(t *testing.T) {
	238	+	s, err := openStore(t.TempDir(), 1<<20, time.Now)
	239	+	if err != nil {
	240	+	t.Fatal(err)
	241	+	}
	242	+	var bugs atomic.Int32
	243	+	s.hookBug = func(string) { bugs.Add(1) }
	244	+	put(t, s, "aaaa")
	245	+	s.mu.Lock()
	246	+	s.unpinLocked("blobs/" + sum("aaaa")[7:])
	247	+	pins := s.entries["blobs/"+sum("aaaa")[7:]].pins
	248	+	s.mu.Unlock()
	249	+	if bugs.Load() != 1 \|\| pins != 0 {
	250	+	t.Fatalf("bugs %d pins %d", bugs.Load(), pins)
	251	+	}
	252	+	}

@@ -0,0 +1,252 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+	package registryproxy
	3	+
	4	+	import (
	5	+	"context"
	6	+	"io"
	7	+	"net/http"
	8	+	"net/http/httptest"
	9	+	"os"
	10	+	"path/filepath"
	11	+	"strings"
	12	+	"sync"
	13	+	"sync/atomic"
	14	+	"testing"
	15	+	"time"
	16	+	)
	17	+
	18	+	// serveAsync serves path with ctx on its own goroutine, into w (a new
	19	+	// recorder when nil); the status code comes on the returned channel.
	20	+	func serveAsync(p Proxy, ctx context.Context, path string, w keepAliveRecorder) <-chan int {
	21	+	if w == nil {
	22	+	w = &keepAliveRecorder{ResponseRecorder: httptest.NewRecorder()}
	23	+	}
	24	+	code := make(chan int, 1)
	25	+	go func() {
	26	+	p.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil).WithContext(ctx))
	27	+	code <- w.Code
	28	+	}()
	29	+	return code
	30	+	}
	31	+
	32	+	// A sandbox that saturates its quota does not hold back another sandbox: its
	33	+	// download starts at once while the global cap still has room.
	34	+	func TestQuotaIsolatesChannels(t *testing.T) {
	35	+	u := newStallingUpstream(t)
	36	+	p := proxyWith(t, u.srv, Config{MaxDownloads: 16, SlotWait: time.Hour, AbandonGrace: 10 * time.Millisecond})
	37	+	a, b := NewQuota(3), NewQuota(3)
	38	+	ctxA, cancelA := context.WithCancel(WithQuota(context.Background(), a))
	39	+	defer cancelA()
	40	+	for i := range 3 {
	41	+	serveAsync(p, ctxA, blobPath(i), nil)
	42	+	}
	43	+	waitFor(t, p.s, "A's quota to fill", func() bool { return a.used.Load() == 3 })
	44	+	// A's fourth waits for its own quota and starts nothing.
	45	+	serveAsync(p, ctxA, blobPath(3), nil)
	46	+	ctxB, cancelB := context.WithCancel(WithQuota(context.Background(), b))
	47	+	defer cancelB()
	48	+	start := time.Now()
	49	+	serveAsync(p, ctxB, blobPath(10), nil)
	50	+	waitFor(t, p.s, "B's download to start", func() bool { return b.used.Load() == 1 })
	51	+	if d := time.Since(start); d > time.Second {
	52	+	t.Fatalf("B started after %v", d)
	53	+	}
	54	+	if got := u.started.Load(); got != 4 {
	55	+	t.Fatalf("%d downloads started, want A's 3 and B's 1", got)
	56	+	}
	57	+	cancelA()
	58	+	cancelB()
	59	+	waitFor(t, p.s, "every slot to free", func() bool { return p.s.downloads == 0 })
	60	+	if a.used.Load() != 0 \|\| b.used.Load() != 0 {
	61	+	t.Fatalf("quota not released: %d %d", a.used.Load(), b.used.Load())
	62	+	}
	63	+	}
	64	+
	65	+	// A request over its quota waits, kept alive, and starts its download as soon
	66	+	// as one of its channel's ends, with no 503.
	67	+	func TestQuotaWaiterGetsFreedSlot(t *testing.T) {
	68	+	body := strings.Repeat("layer", 100)
	69	+	release := make(chan struct{})
	70	+	var once sync.Once
	71	+	unblock := func() { once.Do(func() { close(release) }) }
	72	+	t.Cleanup(unblock)
	73	+	srv := upstreamWith(t, func(w http.ResponseWriter, r *http.Request) {
	74	+	if strings.HasSuffix(r.URL.Path, sum(body)) {
	75	+	_, _ = io.WriteString(w, body)
	76	+	return
	77	+	}
	78	+	w.Header().Set("Content-Length", "1000")
	79	+	_, _ = io.WriteString(w, "x")
	80	+	w.(http.Flusher).Flush()
	81	+	select {
	82	+	case <-release:
	83	+	case <-r.Context().Done():
	84	+	}
	85	+	})
	86	+	p := proxyWith(t, srv, Config{SlotWait: time.Hour, KeepAliveInterval: 5 * time.Millisecond})
	87	+	q := NewQuota(1)
	88	+	ctx := WithQuota(context.Background(), q)
	89	+	first := serveAsync(p, ctx, blobPath(0), nil)
	90	+	waitFor(t, p.s, "the quota to fill", func() bool { return q.used.Load() == 1 })
	91	+	w := &keepAliveRecorder{ResponseRecorder: httptest.NewRecorder()}
	92	+	second := serveAsync(p, ctx, "/v2/library/alpine/blobs/"+sum(body), w)
	93	+	for end := time.Now().Add(5 * time.Second); w.calls.Load() == 0; { // waiting for a slot, kept alive
	94	+	select {
	95	+	case code := <-second:
	96	+	t.Fatalf("did not wait for a slot: %d", code)
	97	+	default:
	98	+	}
	99	+	if time.Now().After(end) {
	100	+	t.Fatal("not kept alive while waiting for a slot")
	101	+	}
	102	+	time.Sleep(time.Millisecond)
	103	+	}
	104	+	unblock() // the first download ends (digest mismatch), freeing the slot
	105	+	if code := <-first; code != http.StatusBadGateway {
	106	+	t.Fatalf("first: %d", code)
	107	+	}
	108	+	if code := <-second; code != http.StatusOK \|\| w.Body.String() != body {
	109	+	t.Fatalf("second: %d", code)
	110	+	}
	111	+	}
	112	+
	113	+	// After SlotWait without a slot, the request gets 503.
	114	+	func TestQuotaWaitTimesOut(t *testing.T) {
	115	+	u := newStallingUpstream(t)
	116	+	const wait = 50 * time.Millisecond
	117	+	p := proxyWith(t, u.srv, Config{SlotWait: wait, AbandonGrace: 10 * time.Millisecond})
	118	+	q := NewQuota(1)
	119	+	ctx, cancel := context.WithCancel(WithQuota(context.Background(), q))
	120	+	defer cancel()
	121	+	serveAsync(p, ctx, blobPath(0), nil)
	122	+	waitFor(t, p.s, "the quota to fill", func() bool { return q.used.Load() == 1 })
	123	+	start := time.Now()
	124	+	code := <-serveAsync(p, ctx, blobPath(1), nil)
	125	+	if code != http.StatusServiceUnavailable {
	126	+	t.Fatalf("over the quota: %d", code)
	127	+	}
	128	+	if d := time.Since(start); d < wait {
	129	+	t.Fatalf("503 after %v", d)
	130	+	}
	131	+	// A request that disconnects while waiting leaves at once, with no slot.
	132	+	cctx, ccancel := context.WithCancel(WithQuota(context.Background(), q))
	133	+	waiting := serveAsync(p, cctx, blobPath(2), nil)
	134	+	ccancel()
	135	+	<-waiting
	136	+	if u.started.Load() != 1 {
	137	+	t.Fatalf("%d downloads started", u.started.Load())
	138	+	}
	139	+	}
	140	+
	141	+	// Joining a download in progress takes none of the joiner's quota.
	142	+	func TestQuotaJoinersExempt(t *testing.T) {
	143	+	u := newStallingUpstream(t)
	144	+	p := proxyWith(t, u.srv, Config{SlotWait: 20 * time.Millisecond, AbandonGrace: 10 * time.Millisecond})
	145	+	a, b := NewQuota(1), NewQuota(1)
	146	+	ctxA, cancelA := context.WithCancel(WithQuota(context.Background(), a))
	147	+	defer cancelA()
	148	+	serveAsync(p, ctxA, blobPath(0), nil)
	149	+	waitFor(t, p.s, "A's download", func() bool { return a.used.Load() == 1 })
	150	+	ctxB, cancelB := context.WithCancel(WithQuota(context.Background(), b))
	151	+	defer cancelB()
	152	+	serveAsync(p, ctxB, blobPath(0), nil) // joins A's
	153	+	key := "blobs/" + sum("blob-0")[7:]
	154	+	waitFor(t, p.s, "B to join", func() bool { return waiters(p.s, key) == 2 })
	155	+	if b.used.Load() != 0 {
	156	+	t.Fatal("a joiner took quota")
	157	+	}
	158	+	// B's quota is still free for a download of its own.
	159	+	serveAsync(p, ctxB, blobPath(1), nil)
	160	+	waitFor(t, p.s, "B's own download", func() bool { return b.used.Load() == 1 })
	161	+	}
	162	+
	163	+	// A panic in a download fails its flight and frees its slots; it does not
	164	+	// leave the waiters hanging.
	165	+	func TestPanickingFetchFailsFlight(t *testing.T) {
	166	+	s, err := openStore(t.TempDir(), 1<<20, time.Now)
	167	+	if err != nil {
	168	+	t.Fatal(err)
	169	+	}
	170	+	s.maxDownloads = 1
	171	+	q := NewQuota(1)
	172	+	ctx, cancel := context.WithTimeout(WithQuota(context.Background(), q), 5*time.Second)
	173	+	defer cancel()
	174	+	_, _, _, err = s.ensure(ctx, "blobs", sum("x"), func(context.Context, io.Writer) (string, error) {
	175	+	panic("boom")
	176	+	}, keepalive{})
	177	+	if err == nil \|\| !strings.Contains(err.Error(), "panicked") {
	178	+	t.Fatalf("err %v", err)
	179	+	}
	180	+	s.mu.Lock()
	181	+	defer s.mu.Unlock()
	182	+	if s.downloads != 0 \|\| len(s.flights) != 0 \|\| q.used.Load() != 0 \|\| s.pending != 0 {
	183	+	t.Fatalf("leaked: downloads=%d flights=%d quota=%d pending=%d", s.downloads, len(s.flights), q.used.Load(), s.pending)
	184	+	}
	185	+	if ds, _ := os.ReadDir(filepath.Join(s.dir, "tmp")); len(ds) != 0 {
	186	+	t.Fatal("temp file kept")
	187	+	}
	188	+	}
	189	+
	190	+	// Eviction restarts from the front after a manifest removal that fails, and
	191	+	// does not loop on it.
	192	+	func TestEvictionAfterFailedManifestRemoval(t *testing.T) {
	193	+	now := time.Unix(1_700_000_000, 0)
	194	+	const media = "application/vnd.oci.image.index.v1+json"
	195	+	s, err := openStore(t.TempDir(), 1<<20, func() time.Time { return now })
	196	+	if err != nil {
	197	+	t.Fatal(err)
	198	+	}
	199	+	m := `{"schemaVersion":2}`
	200	+	f, _, _, err := s.ensure(context.Background(), "manifests", sum(m), func(_ context.Context, w io.Writer) (string, error) {
	201	+	_, err := io.WriteString(w, m)
	202	+	return media, err
	203	+	}, keepalive{})
	204	+	if err != nil {
	205	+	t.Fatal(err)
	206	+	}
	207	+	_ = f.Close()
	208	+	now = now.Add(time.Second)
	209	+	if err := s.setTag("library/alpine", "3", "", sum(m)); err != nil {
	210	+	t.Fatal(err)
	211	+	}
	212	+	now = now.Add(time.Second)
	213	+	put(t, s, "blob")
	214	+	// The manifest's .type cannot be removed: replace it with a non-empty
	215	+	// directory. Its tag goes first all the same.
	216	+	typ := filepath.Join(s.dir, "manifests", sum(m)[7:]+".type")
	217	+	if err := os.Remove(typ); err != nil {
	218	+	t.Fatal(err)
	219	+	}
	220	+	if err := os.MkdirAll(filepath.Join(typ, "x"), 0700); err != nil {
	221	+	t.Fatal(err)
	222	+	}
	223	+	s.mu.Lock()
	224	+	s.evictLocked(s.max) // ask for everything
	225	+	_, manifestKept := s.entries["manifests/"+sum(m)[7:]]
	226	+	_, blobKept := s.entries["blobs/"+sum("blob")[7:]]
	227	+	tags := len(s.tagsOf)
	228	+	s.mu.Unlock()
	229	+	if !manifestKept \|\| blobKept \|\| tags != 0 {
	230	+	t.Fatalf("manifest kept=%v blob kept=%v tags=%d", manifestKept, blobKept, tags)
	231	+	}
	232	+	accounted(t, s)
	233	+	}
	234	+
	235	+	// unpinLocked of an unpinned entry is reported (and fails the test run through
	236	+	// TestMain's hook); here the hook is replaced to observe it.
	237	+	func TestUnpinBugReported(t *testing.T) {
	238	+	s, err := openStore(t.TempDir(), 1<<20, time.Now)
	239	+	if err != nil {
	240	+	t.Fatal(err)
	241	+	}
	242	+	var bugs atomic.Int32
	243	+	s.hookBug = func(string) { bugs.Add(1) }
	244	+	put(t, s, "aaaa")
	245	+	s.mu.Lock()
	246	+	s.unpinLocked("blobs/" + sum("aaaa")[7:])
	247	+	pins := s.entries["blobs/"+sum("aaaa")[7:]].pins
	248	+	s.mu.Unlock()
	249	+	if bugs.Load() != 1 \|\| pins != 0 {
	250	+	t.Fatalf("bugs %d pins %d", bugs.Load(), pins)
	251	+	}
	252	+	}

A internal/registryproxy/store.go +727 -0

@@ -0,0 +1,727 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package registryproxy
	4	+
	5	+	import (
	6	+	"container/list"
	7	+	"context"
	8	+	"crypto/sha256"
	9	+	"encoding/hex"
	10	+	"errors"
	11	+	"fmt"
	12	+	"io"
	13	+	"os"
	14	+	"path/filepath"
	15	+	"sort"
	16	+	"strings"
	17	+	"sync"
	18	+	"sync/atomic"
	19	+	"time"
	20	+
	21	+	"github.com/bnema/zerowrap"
	22	+	)
	23	+
	24	+	var errTooLarge = errors.New("registry cache capacity exceeded")
	25	+	var errDigest = errors.New("registry digest mismatch")
	26	+
	27	+	// errBusy: no download slot came free within the slot wait. The proxy answers
	28	+	// 503, and dockerd retries the layer (about 50 s) before pulling from Docker Hub.
	29	+	var errBusy = errors.New("registry downloads busy")
	30	+
	31	+	const tagCharge int64 = 4096
	32	+
	33	+	// lowWater bounds the extra room an eviction frees (with 5% of Max), so that a
	34	+	// full store evicts once per that many bytes, not on every write.
	35	+	const lowWater int64 = 64 << 20
	36	+
	37	+	type entry struct {
	38	+	key string
	39	+	size int64
	40	+	used time.Time
	41	+	pins int
	42	+	digest string // tag target, empty for content
	43	+	elem *list.Element // in store.lru; nil once removed
	44	+	}
	45	+
	46	+	// flight is one download, shared by every request for its digest. Its fields
	47	+	// below written are guarded by store.mu.
	48	+	type flight struct {
	49	+	key string
	50	+	done chan struct{}
	51	+	err error
	52	+	written atomic.Int64 // bytes received so far, for the waiters' keepalive
	53	+
	54	+	waiters int
	55	+	gen int // bumped each time waiters drops to zero
	56	+	finished bool
	57	+	pinned bool // the committed entry holds a pin for the waiters
	58	+	blob bool // takes a download slot
	59	+	quota *Quota // the starter's quota, held until the flight ends
	60	+	cancel context.CancelFunc
	61	+	}
	62	+
	63	+	type store struct {
	64	+	dir string
	65	+	max, total, pending int64
	66	+	now func() time.Time
	67	+	// grace is how long a flight with no waiter left keeps downloading in
	68	+	// case another request joins it; then it is cancelled.
	69	+	grace time.Duration
	70	+	// maxDownloads caps the concurrent blob downloads (0: no cap).
	71	+	maxDownloads, downloads int
	72	+	// slotWait is how long a request waits for a download slot before errBusy.
	73	+	slotWait time.Duration
	74	+	// slotFreed is closed and replaced each time a download slot frees.
	75	+	slotFreed chan struct{}
	76	+	log zerowrap.Logger
	77	+
	78	+	mu sync.Mutex
	79	+	entries map[string]*entry
	80	+	lru list.List // entry, least recently used first
	81	+	tagsOf map[string]map[string]struct{} // manifest digest -> tag keys
	82	+	flights map[string]*flight
	83	+
	84	+	hookCommitted func(key string) // tests: after a commit, before waking waiters
	85	+	hookBug func(msg string) // tests: called on an accounting bug
	86	+	}
	87	+
	88	+	// defaultBugHook is each new store's hookBug. Tests set it in TestMain, before
	89	+	// any store exists, so that an accounting bug fails the run.
	90	+	var defaultBugHook func(msg string)
	91	+
	92	+	// storedFile pins its entry until the descriptor is closed. In particular, an
	93	+	// unlinked-but-open blob cannot silently escape the disk accounting.
	94	+	type storedFile struct {
	95	+	*os.File
	96	+	s *store
	97	+	key string
	98	+	once sync.Once
	99	+	err error
	100	+	}
	101	+
	102	+	func (f *storedFile) Close() error {
	103	+	f.once.Do(func() {
	104	+	f.err = f.File.Close()
	105	+	f.s.mu.Lock()
	106	+	defer f.s.mu.Unlock()
	107	+	f.s.unpinLocked(f.key)
	108	+	})
	109	+	return f.err
	110	+	}
	111	+
	112	+	func openStore(dir string, max int64, now func() time.Time) (*store, error) {
	113	+	if max <= 0 {
	114	+	return nil, errors.New("registry cache cap must be positive")
	115	+	}
	116	+	s := &store{dir: dir, max: max, now: now, entries: map[string]entry{}, lru: list.New(), tagsOf: map[string]map[string]struct{}{}, flights: map[string]flight{}, slotFreed: make(chan struct{}), hookBug: defaultBugHook}
	117	+	for _, kind := range []string{"blobs", "manifests", "tags", "tmp"} {
	118	+	if err := os.MkdirAll(filepath.Join(dir, kind), 0700); err != nil {
	119	+	return nil, err
	120	+	}
	121	+	}
	122	+	temps, err := os.ReadDir(filepath.Join(dir, "tmp"))
	123	+	if err != nil {
	124	+	return nil, err
	125	+	}
	126	+	for _, de := range temps {
	127	+	if err := os.Remove(filepath.Join(dir, "tmp", de.Name())); err != nil {
	128	+	return nil, err
	129	+	}
	130	+	}
	131	+	var types []string
	132	+	for _, kind := range []string{"blobs", "manifests", "tags"} {
	133	+	ds, err := os.ReadDir(filepath.Join(dir, kind))
	134	+	if err != nil {
	135	+	return nil, err
	136	+	}
	137	+	for _, de := range ds {
	138	+	if base, ok := strings.CutSuffix(de.Name(), ".type"); ok && kind == "manifests" {
	139	+	types = append(types, base)
	140	+	continue
	141	+	}
	142	+	if !validDigest("sha256:" + de.Name()) {
	143	+	continue
	144	+	}
	145	+	fi, err := de.Info()
	146	+	if err != nil {
	147	+	return nil, err
	148	+	}
	149	+	if !fi.Mode().IsRegular() {
	150	+	continue
	151	+	}
	152	+	key := kind + "/" + de.Name()
	153	+	e := &entry{key: key, size: fi.Size(), used: fi.ModTime()}
	154	+	if kind == "manifests" {
	155	+	info, err := os.Stat(filepath.Join(dir, key) + ".type")
	156	+	if err != nil {
	157	+	_ = os.Remove(filepath.Join(dir, key))
	158	+	continue
	159	+	}
	160	+	e.size += info.Size()
	161	+	}
	162	+	if kind == "tags" {
	163	+	digest := s.readTagFile(key)
	164	+	if !validDigest(digest) {
	165	+	_ = os.Remove(filepath.Join(dir, key))
	166	+	continue
	167	+	}
	168	+	if _, ok := s.entries["manifests/"+digest[7:]]; !ok {
	169	+	_ = os.Remove(filepath.Join(dir, key))
	170	+	continue
	171	+	}
	172	+	e.size = tagCharge
	173	+	e.digest = digest
	174	+	}
	175	+	s.addLocked(e)
	176	+	}
	177	+	}
	178	+	// A .type whose manifest is gone (a crash between the two writes).
	179	+	for _, base := range types {
	180	+	if _, ok := s.entries["manifests/"+base]; !ok {
	181	+	_ = os.Remove(filepath.Join(dir, "manifests", base+".type"))
	182	+	}
	183	+	}
	184	+	// Order the LRU by the access times kept in the mtimes.
	185	+	all := make([]*entry, 0, len(s.entries))
	186	+	for _, e := range s.entries {
	187	+	all = append(all, e)
	188	+	}
	189	+	sort.Slice(all, func(i, j int) bool { return all[i].used.Before(all[j].used) })
	190	+	s.lru.Init()
	191	+	for _, e := range all {
	192	+	e.elem = s.lru.PushBack(e)
	193	+	}
	194	+	s.evictLocked(0)
	195	+	return s, nil
	196	+	}
	197	+
	198	+	func (s store) addLocked(e entry) {
	199	+	s.entries[e.key] = e
	200	+	e.elem = s.lru.PushBack(e)
	201	+	s.total += e.size
	202	+	if e.digest != "" {
	203	+	s.indexTagLocked(e.key, e.digest)
	204	+	}
	205	+	}
	206	+
	207	+	func (s *store) indexTagLocked(key, digest string) {
	208	+	set := s.tagsOf[digest]
	209	+	if set == nil {
	210	+	set = map[string]struct{}{}
	211	+	s.tagsOf[digest] = set
	212	+	}
	213	+	set[key] = struct{}{}
	214	+	}
	215	+
	216	+	func (s *store) unindexTagLocked(key, digest string) {
	217	+	if set := s.tagsOf[digest]; set != nil {
	218	+	delete(set, key)
	219	+	if len(set) == 0 {
	220	+	delete(s.tagsOf, digest)
	221	+	}
	222	+	}
	223	+	}
	224	+
	225	+	func (s store) touchLocked(e entry) {
	226	+	e.used = s.now()
	227	+	s.lru.MoveToBack(e.elem)
	228	+	_ = os.Chtimes(filepath.Join(s.dir, e.key), e.used, e.used)
	229	+	}
	230	+
	231	+	func (s *store) unpinLocked(key string) {
	232	+	e, ok := s.entries[key]
	233	+	if !ok {
	234	+	return
	235	+	}
	236	+	if e.pins <= 0 {
	237	+	// A bug: the entry could have been evicted while open.
	238	+	s.log.Error().Str("key", key).Msg("registry cache unpin of an unpinned entry")
	239	+	if s.hookBug != nil {
	240	+	s.hookBug("unpin of an unpinned entry " + key)
	241	+	}
	242	+	return
	243	+	}
	244	+	e.pins--
	245	+	}
	246	+
	247	+	// removeLocked only releases a charge after the corresponding file is gone.
	248	+	// Failures leave the entry charged so a full or unwritable disk fails closed.
	249	+	func (s *store) removeLocked(key string) bool {
	250	+	e, ok := s.entries[key]
	251	+	if !ok {
	252	+	return true
	253	+	}
	254	+	if e.pins > 0 {
	255	+	return false
	256	+	}
	257	+	if hexDigest, ok := strings.CutPrefix(key, "manifests/"); ok {
	258	+	// Prune references first. If a tag cannot be removed, retain its target too.
	259	+	for k := range s.tagsOf["sha256:"+hexDigest] {
	260	+	if !s.removeLocked(k) {
	261	+	return false
	262	+	}
	263	+	}
	264	+	if err := os.Remove(filepath.Join(s.dir, key) + ".type"); err != nil && !os.IsNotExist(err) {
	265	+	return false
	266	+	}
	267	+	}
	268	+	if err := os.Remove(filepath.Join(s.dir, key)); err != nil && !os.IsNotExist(err) {
	269	+	return false
	270	+	}
	271	+	s.total -= e.size
	272	+	delete(s.entries, key)
	273	+	s.lru.Remove(e.elem)
	274	+	e.elem = nil
	275	+	if e.digest != "" {
	276	+	s.unindexTagLocked(key, e.digest)
	277	+	}
	278	+	return true
	279	+	}
	280	+
	281	+	// evictLocked makes room for need bytes. Once over the cap, it walks the LRU
	282	+	// (no sort) and evicts unpinned entries until max(need, low water) bytes are
	283	+	// free, so a full store evicts once per low-water bytes, not on every write.
	284	+	func (s *store) evictLocked(need int64) {
	285	+	if s.total+s.pending+need <= s.max {
	286	+	return
	287	+	}
	288	+	target := s.max - max(need, min(lowWater, s.max/20))
	289	+	for el := s.lru.Front(); el != nil && s.total+s.pending > target; {
	290	+	e := el.Value.(*entry)
	291	+	next := el.Next()
	292	+	if e.pins == 0 {
	293	+	removed := s.removeLocked(e.key)
	294	+	if strings.HasPrefix(e.key, "manifests/") {
	295	+	// Removing a manifest removes its tags first, even when it then
	296	+	// fails: next may be gone. Start over; only pinned or
	297	+	// unremovable entries are walked again.
	298	+	next = s.lru.Front()
	299	+	if !removed {
	300	+	// Do not walk into it again: it stays where it is, and
	301	+	// the walk resumes after it.
	302	+	next = e.elem.Next()
	303	+	}
	304	+	}
	305	+	}
	306	+	el = next
	307	+	}
	308	+	}
	309	+
	310	+	func (s store) openLocked(key string) (storedFile, string, error) {
	311	+	e, ok := s.entries[key]
	312	+	if !ok {
	313	+	return nil, "", os.ErrNotExist
	314	+	}
	315	+	f, err := os.Open(filepath.Join(s.dir, key))
	316	+	if err != nil {
	317	+	return nil, "", err
	318	+	}
	319	+	media := "application/octet-stream"
	320	+	if strings.HasPrefix(key, "manifests/") {
	321	+	b, err := os.ReadFile(filepath.Join(s.dir, key) + ".type")
	322	+	if err != nil {
	323	+	_ = f.Close()
	324	+	return nil, "", err
	325	+	}
	326	+	media = string(b)
	327	+	}
	328	+	s.touchLocked(e)
	329	+	e.pins++
	330	+	return &storedFile{File: f, s: s, key: key}, media, nil
	331	+	}
	332	+	func (s store) open(kind, digest string) (storedFile, string, error) {
	333	+	if !validDigest(digest) \|\| (kind != "blobs" && kind != "manifests") {
	334	+	return nil, "", errDigest
	335	+	}
	336	+	s.mu.Lock()
	337	+	defer s.mu.Unlock()
	338	+	return s.openLocked(kind + "/" + digest[7:])
	339	+	}
	340	+
	341	+	// fetch writes the upstream content to w. ctx belongs to the flight: it ends
	342	+	// when the last waiter has been gone for the grace period.
	343	+	type fetch func(ctx context.Context, w io.Writer) (string, error)
	344	+
	345	+	type keepalive struct {
	346	+	every time.Duration
	347	+	fn func()
	348	+	}
	349	+
	350	+	// ensure returns the stored content, pinned for the caller; hit reports that it
	351	+	// was already stored. Otherwise the request joins or starts the flight of that
	352	+	// digest. The flight runs on its own goroutine and context, so no single
	353	+	// request, the initiator included, can cancel it while another one waits; it is
	354	+	// cancelled once no request has waited for it for the grace period. Every
	355	+	// waiter waits for the verified commit and then reads the stored file. While
	356	+	// waiting, keep.fn is called every keep.every when the download has made
	357	+	// progress, so that the waiter's connection deadlines can follow it.
	358	+	//
	359	+	// Starting a blob flight takes a global download slot and one of the quota
	360	+	// carried by ctx (WithQuota), both held until the flight really ends; joining
	361	+	// one takes neither. When a slot is missing, the request waits for one, kept
	362	+	// alive, for at most the slot wait, and then fails with errBusy.
	363	+	func (s store) ensure(ctx context.Context, kind, digest string, fetch fetch, keep keepalive) (f storedFile, media string, hit bool, err error) {
	364	+	if !validDigest(digest) \|\| (kind != "blobs" && kind != "manifests") {
	365	+	return nil, "", false, errDigest
	366	+	}
	367	+	key := kind + "/" + digest[7:]
	368	+	blob := kind == "blobs"
	369	+	quota := quotaFrom(ctx)
	370	+	var slotDeadline <-chan time.Time
	371	+	var fl *flight
	372	+	for {
	373	+	s.mu.Lock()
	374	+	if f, media, err := s.openLocked(key); err == nil {
	375	+	s.mu.Unlock()
	376	+	return f, media, true, nil
	377	+	}
	378	+	var ok bool
	379	+	if fl, ok = s.flights[key]; ok {
	380	+	break // join: no slot needed
	381	+	}
	382	+	if !blob \|\| s.takeSlotLocked(quota) {
	383	+	dctx, cancel := context.WithCancel(context.Background())
	384	+	fl = &flight{key: key, done: make(chan struct{}), cancel: cancel, blob: blob}
	385	+	if blob {
	386	+	fl.quota = quota
	387	+	}
	388	+	s.flights[key] = fl
	389	+	go s.run(dctx, fl, digest, fetch)
	390	+	break
	391	+	}
	392	+	freed := s.slotFreed
	393	+	s.mu.Unlock()
	394	+	if slotDeadline == nil {
	395	+	t := time.NewTimer(s.slotWait)
	396	+	defer t.Stop()
	397	+	slotDeadline = t.C
	398	+	}
	399	+	if err := waitSlot(ctx, freed, slotDeadline, keep); err != nil {
	400	+	return nil, "", false, err
	401	+	}
	402	+	}
	403	+	fl.waiters++
	404	+	s.mu.Unlock()
	405	+
	406	+	err = waitFlight(ctx, fl, keep)
	407	+	s.mu.Lock()
	408	+	defer s.mu.Unlock()
	409	+	if err == nil {
	410	+	f, media, err = s.openLocked(key)
	411	+	}
	412	+	s.leaveLocked(fl)
	413	+	return f, media, false, err
	414	+	}
	415	+
	416	+	// leaveLocked drops a waiter. The last one releases the commit pin, or, before
	417	+	// the commit, starts the grace period after which the flight is cancelled.
	418	+	func (s store) leaveLocked(fl flight) {
	419	+	fl.waiters--
	420	+	if fl.waiters > 0 {
	421	+	return
	422	+	}
	423	+	if fl.finished {
	424	+	if fl.pinned {
	425	+	fl.pinned = false
	426	+	s.unpinLocked(fl.key)
	427	+	}
	428	+	return
	429	+	}
	430	+	fl.gen++
	431	+	gen := fl.gen
	432	+	// A manifest is small and bounded by the request timeout: it is cancelled at
	433	+	// once, so its flights never outnumber the requests in progress. A blob
	434	+	// holds a download slot until it really ends, which bounds the abandoned
	435	+	// ones to MaxDownloads.
	436	+	if s.grace <= 0 \|\| !fl.blob {
	437	+	s.abandonLocked(fl)
	438	+	return
	439	+	}
	440	+	time.AfterFunc(s.grace, func() {
	441	+	s.mu.Lock()
	442	+	defer s.mu.Unlock()
	443	+	if fl.waiters == 0 && !fl.finished && fl.gen == gen {
	444	+	s.abandonLocked(fl)
	445	+	}
	446	+	})
	447	+	}
	448	+
	449	+	// abandonLocked cancels a flight nobody waits for. A later request for the
	450	+	// digest starts a new one.
	451	+	func (s store) abandonLocked(fl flight) {
	452	+	fl.cancel()
	453	+	if s.flights[fl.key] == fl {
	454	+	delete(s.flights, fl.key)
	455	+	}
	456	+	}
	457	+
	458	+	// takeSlotLocked takes a global download slot and one of quota (nil: none),
	459	+	// both or neither.
	460	+	func (s store) takeSlotLocked(quota Quota) bool {
	461	+	if s.maxDownloads > 0 && s.downloads >= s.maxDownloads {
	462	+	return false
	463	+	}
	464	+	if quota != nil && !quota.tryAcquire() {
	465	+	return false
	466	+	}
	467	+	s.downloads++
	468	+	return true
	469	+	}
	470	+
	471	+	// waitSlot waits until a slot may have freed (freed is closed), keeping the
	472	+	// client alive meanwhile. It fails with errBusy at the deadline.
	473	+	func waitSlot(ctx context.Context, freed <-chan struct{}, deadline <-chan time.Time, keep keepalive) error {
	474	+	var tick <-chan time.Time
	475	+	if keep.every > 0 && keep.fn != nil {
	476	+	t := time.NewTicker(keep.every)
	477	+	defer t.Stop()
	478	+	tick = t.C
	479	+	}
	480	+	for {
	481	+	select {
	482	+	case <-freed:
	483	+	return nil
	484	+	case <-ctx.Done():
	485	+	return ctx.Err()
	486	+	case <-deadline:
	487	+	return errBusy
	488	+	case <-tick:
	489	+	keep.fn() // bounded by the slot wait
	490	+	}
	491	+	}
	492	+	}
	493	+
	494	+	// finishLocked ends a flight and releases its slots. A committed entry is
	495	+	// pinned on behalf of the waiters, so that no eviction removes it before they
	496	+	// open it.
	497	+	func (s store) finishLocked(fl flight, err error, committed *entry) {
	498	+	fl.err = err
	499	+	fl.finished = true
	500	+	if err == nil && committed != nil && fl.waiters > 0 {
	501	+	committed.pins++
	502	+	fl.pinned = true
	503	+	}
	504	+	if s.flights[fl.key] == fl {
	505	+	delete(s.flights, fl.key)
	506	+	}
	507	+	if fl.blob {
	508	+	s.downloads--
	509	+	if fl.quota != nil {
	510	+	fl.quota.release()
	511	+	}
	512	+	close(s.slotFreed)
	513	+	s.slotFreed = make(chan struct{})
	514	+	}
	515	+	}
	516	+
	517	+	// run downloads and finishes the flight. A panic in fetch fails the flight
	518	+	// (and frees its slots) rather than leave its waiters hanging.
	519	+	func (s store) run(ctx context.Context, fl flight, digest string, fetch fetch) {
	520	+	var err error
	521	+	defer func() {
	522	+	if v := recover(); v != nil {
	523	+	err = fmt.Errorf("registry download panicked: %v", v)
	524	+	s.log.Error().Str("key", fl.key).Err(err).Msg("registry download")
	525	+	}
	526	+	fl.cancel()
	527	+	s.mu.Lock()
	528	+	if !fl.finished {
	529	+	if err == nil {
	530	+	err = errors.New("registry download ended without a commit")
	531	+	}
	532	+	s.finishLocked(fl, err, nil)
	533	+	}
	534	+	s.mu.Unlock()
	535	+	if fl.err == nil && s.hookCommitted != nil {
	536	+	s.hookCommitted(fl.key)
	537	+	}
	538	+	close(fl.done)
	539	+	}()
	540	+	err = s.fill(ctx, fl, digest, fetch)
	541	+	}
	542	+
	543	+	func waitFlight(ctx context.Context, fl *flight, keep keepalive) error {
	544	+	var tick <-chan time.Time
	545	+	if keep.every > 0 && keep.fn != nil {
	546	+	t := time.NewTicker(keep.every)
	547	+	defer t.Stop()
	548	+	tick = t.C
	549	+	}
	550	+	last := fl.written.Load()
	551	+	for {
	552	+	select {
	553	+	case <-ctx.Done():
	554	+	return ctx.Err()
	555	+	case <-fl.done:
	556	+	return fl.err
	557	+	case <-tick:
	558	+	if n := fl.written.Load(); n != last {
	559	+	last = n
	560	+	keep.fn()
	561	+	}
	562	+	}
	563	+	}
	564	+	}
	565	+
	566	+	type chargedWriter struct {
	567	+	s *store
	568	+	w io.Writer
	569	+	n int64
	570	+	written *atomic.Int64
	571	+	}
	572	+
	573	+	func (w *chargedWriter) Write(b []byte) (int, error) {
	574	+	s := w.s
	575	+	s.mu.Lock()
	576	+	need := int64(len(b))
	577	+	s.evictLocked(need)
	578	+	if need > s.max-s.total-s.pending {
	579	+	s.mu.Unlock()
	580	+	return 0, errTooLarge
	581	+	}
	582	+	s.pending += need
	583	+	w.n += need
	584	+	s.mu.Unlock()
	585	+	n, err := w.w.Write(b)
	586	+	if n < len(b) {
	587	+	s.mu.Lock()
	588	+	s.pending -= int64(len(b) - n)
	589	+	w.n -= int64(len(b) - n)
	590	+	s.mu.Unlock()
	591	+	}
	592	+	w.written.Add(int64(n))
	593	+	return n, err
	594	+	}
	595	+
	596	+	// fill downloads into tmp/, verifies the digest and commits. On success it has
	597	+	// finished the flight (finishLocked) under the same lock as the commit.
	598	+	func (s store) fill(ctx context.Context, fl flight, digest string, fetch fetch) error {
	599	+	key := fl.key
	600	+	f, err := os.CreateTemp(filepath.Join(s.dir, "tmp"), "download-")
	601	+	if err != nil {
	602	+	return err
	603	+	}
	604	+	defer func() { _ = f.Close(); _ = os.Remove(f.Name()) }()
	605	+	h := sha256.New()
	606	+	cw := &chargedWriter{s: s, w: io.MultiWriter(f, h), written: &fl.written}
	607	+	defer func() { s.mu.Lock(); s.pending -= cw.n; s.mu.Unlock() }()
	608	+	media, err := fetch(ctx, cw)
	609	+	if err != nil {
	610	+	return err
	611	+	}
	612	+	if "sha256:"+hex.EncodeToString(h.Sum(nil)) != digest {
	613	+	return errDigest
	614	+	}
	615	+	if err = f.Close(); err != nil {
	616	+	return err
	617	+	}
	618	+	s.mu.Lock()
	619	+	defer s.mu.Unlock()
	620	+	if e, ok := s.entries[key]; ok {
	621	+	// Same digest, same bytes: keep the stored file and its pins rather than
	622	+	// renaming over a file that may be open (its bytes would leave the
	623	+	// accounting while still allocated).
	624	+	s.finishLocked(fl, nil, e)
	625	+	return nil
	626	+	}
	627	+	path := filepath.Join(s.dir, key)
	628	+	metadata := int64(0)
	629	+	if strings.HasPrefix(key, "manifests/") {
	630	+	if len(media) > 1024 {
	631	+	return errors.New("manifest media type too large")
	632	+	}
	633	+	metadata = int64(len(media))
	634	+	s.evictLocked(metadata)
	635	+	if metadata > s.max-s.total-s.pending {
	636	+	return errTooLarge
	637	+	}
	638	+	if err := os.WriteFile(path+".type", []byte(media), 0600); err != nil {
	639	+	return err
	640	+	}
	641	+	}
	642	+	if err := os.Rename(f.Name(), path); err != nil {
	643	+	if metadata > 0 {
	644	+	_ = os.Remove(path + ".type")
	645	+	}
	646	+	return err
	647	+	}
	648	+	e := &entry{key: key, size: cw.n + metadata, used: s.now()}
	649	+	s.addLocked(e)
	650	+	s.pending -= cw.n
	651	+	cw.n = 0
	652	+	s.finishLocked(fl, nil, e)
	653	+	return nil
	654	+	}
	655	+	func tagKey(name, tag, accept string) string {
	656	+	return fmt.Sprintf("%x", sha256.Sum256([]byte(name+"\x00"+tag+"\x00"+normalizeAccept(accept))))
	657	+	}
	658	+	func (s *store) readTagFile(key string) string {
	659	+	f, err := os.Open(filepath.Join(s.dir, key))
	660	+	if err != nil {
	661	+	return ""
	662	+	}
	663	+	defer func() { _ = f.Close() }()
	664	+	b, err := io.ReadAll(io.LimitReader(f, 72))
	665	+	if err != nil \|\| !validDigest(string(b)) {
	666	+	return ""
	667	+	}
	668	+	return string(b)
	669	+	}
	670	+	func (s *store) tag(name, tag, accept string) string {
	671	+	s.mu.Lock()
	672	+	defer s.mu.Unlock()
	673	+	e, ok := s.entries["tags/"+tagKey(name, tag, accept)]
	674	+	if !ok {
	675	+	return ""
	676	+	}
	677	+	s.touchLocked(e)
	678	+	return e.digest
	679	+	}
	680	+	func (s *store) setTag(name, tag, accept, digest string) error {
	681	+	if !validDigest(digest) {
	682	+	return errDigest
	683	+	}
	684	+	s.mu.Lock()
	685	+	defer s.mu.Unlock()
	686	+	manifest, ok := s.entries["manifests/"+digest[7:]]
	687	+	if !ok {
	688	+	return os.ErrNotExist
	689	+	}
	690	+	// Protect the target while making room for its metadata.
	691	+	manifest.pins++
	692	+	defer func() { manifest.pins-- }()
	693	+	key := "tags/" + tagKey(name, tag, accept)
	694	+	old, exists := s.entries[key]
	695	+	if exists {
	696	+	// Already charged: rewriting it needs no room.
	697	+	s.touchLocked(old)
	698	+	} else {
	699	+	s.evictLocked(tagCharge)
	700	+	if tagCharge > s.max-s.total-s.pending {
	701	+	return errTooLarge
	702	+	}
	703	+	}
	704	+	f, err := os.CreateTemp(filepath.Join(s.dir, "tmp"), "tag-")
	705	+	if err != nil {
	706	+	return err
	707	+	}
	708	+	defer func() { _ = f.Close(); _ = os.Remove(f.Name()) }()
	709	+	if _, err = f.WriteString(digest); err != nil {
	710	+	return err
	711	+	}
	712	+	if err = f.Close(); err != nil {
	713	+	return err
	714	+	}
	715	+	if err = os.Rename(f.Name(), filepath.Join(s.dir, key)); err != nil {
	716	+	return err
	717	+	}
	718	+	if old, ok := s.entries[key]; ok {
	719	+	s.unindexTagLocked(key, old.digest)
	720	+	old.digest = digest
	721	+	old.used = s.now()
	722	+	s.indexTagLocked(key, digest)
	723	+	return nil
	724	+	}
	725	+	s.addLocked(&entry{key: key, size: tagCharge, used: s.now(), digest: digest})
	726	+	return nil
	727	+	}

@@ -0,0 +1,727 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package registryproxy
	4	+
	5	+	import (
	6	+	"container/list"
	7	+	"context"
	8	+	"crypto/sha256"
	9	+	"encoding/hex"
	10	+	"errors"
	11	+	"fmt"
	12	+	"io"
	13	+	"os"
	14	+	"path/filepath"
	15	+	"sort"
	16	+	"strings"
	17	+	"sync"
	18	+	"sync/atomic"
	19	+	"time"
	20	+
	21	+	"github.com/bnema/zerowrap"
	22	+	)
	23	+
	24	+	var errTooLarge = errors.New("registry cache capacity exceeded")
	25	+	var errDigest = errors.New("registry digest mismatch")
	26	+
	27	+	// errBusy: no download slot came free within the slot wait. The proxy answers
	28	+	// 503, and dockerd retries the layer (about 50 s) before pulling from Docker Hub.
	29	+	var errBusy = errors.New("registry downloads busy")
	30	+
	31	+	const tagCharge int64 = 4096
	32	+
	33	+	// lowWater bounds the extra room an eviction frees (with 5% of Max), so that a
	34	+	// full store evicts once per that many bytes, not on every write.
	35	+	const lowWater int64 = 64 << 20
	36	+
	37	+	type entry struct {
	38	+	key string
	39	+	size int64
	40	+	used time.Time
	41	+	pins int
	42	+	digest string // tag target, empty for content
	43	+	elem *list.Element // in store.lru; nil once removed
	44	+	}
	45	+
	46	+	// flight is one download, shared by every request for its digest. Its fields
	47	+	// below written are guarded by store.mu.
	48	+	type flight struct {
	49	+	key string
	50	+	done chan struct{}
	51	+	err error
	52	+	written atomic.Int64 // bytes received so far, for the waiters' keepalive
	53	+
	54	+	waiters int
	55	+	gen int // bumped each time waiters drops to zero
	56	+	finished bool
	57	+	pinned bool // the committed entry holds a pin for the waiters
	58	+	blob bool // takes a download slot
	59	+	quota *Quota // the starter's quota, held until the flight ends
	60	+	cancel context.CancelFunc
	61	+	}
	62	+
	63	+	type store struct {
	64	+	dir string
	65	+	max, total, pending int64
	66	+	now func() time.Time
	67	+	// grace is how long a flight with no waiter left keeps downloading in
	68	+	// case another request joins it; then it is cancelled.
	69	+	grace time.Duration
	70	+	// maxDownloads caps the concurrent blob downloads (0: no cap).
	71	+	maxDownloads, downloads int
	72	+	// slotWait is how long a request waits for a download slot before errBusy.
	73	+	slotWait time.Duration
	74	+	// slotFreed is closed and replaced each time a download slot frees.
	75	+	slotFreed chan struct{}
	76	+	log zerowrap.Logger
	77	+
	78	+	mu sync.Mutex
	79	+	entries map[string]*entry
	80	+	lru list.List // entry, least recently used first
	81	+	tagsOf map[string]map[string]struct{} // manifest digest -> tag keys
	82	+	flights map[string]*flight
	83	+
	84	+	hookCommitted func(key string) // tests: after a commit, before waking waiters
	85	+	hookBug func(msg string) // tests: called on an accounting bug
	86	+	}
	87	+
	88	+	// defaultBugHook is each new store's hookBug. Tests set it in TestMain, before
	89	+	// any store exists, so that an accounting bug fails the run.
	90	+	var defaultBugHook func(msg string)
	91	+
	92	+	// storedFile pins its entry until the descriptor is closed. In particular, an
	93	+	// unlinked-but-open blob cannot silently escape the disk accounting.
	94	+	type storedFile struct {
	95	+	*os.File
	96	+	s *store
	97	+	key string
	98	+	once sync.Once
	99	+	err error
	100	+	}
	101	+
	102	+	func (f *storedFile) Close() error {
	103	+	f.once.Do(func() {
	104	+	f.err = f.File.Close()
	105	+	f.s.mu.Lock()
	106	+	defer f.s.mu.Unlock()
	107	+	f.s.unpinLocked(f.key)
	108	+	})
	109	+	return f.err
	110	+	}
	111	+
	112	+	func openStore(dir string, max int64, now func() time.Time) (*store, error) {
	113	+	if max <= 0 {
	114	+	return nil, errors.New("registry cache cap must be positive")
	115	+	}
	116	+	s := &store{dir: dir, max: max, now: now, entries: map[string]entry{}, lru: list.New(), tagsOf: map[string]map[string]struct{}{}, flights: map[string]flight{}, slotFreed: make(chan struct{}), hookBug: defaultBugHook}
	117	+	for _, kind := range []string{"blobs", "manifests", "tags", "tmp"} {
	118	+	if err := os.MkdirAll(filepath.Join(dir, kind), 0700); err != nil {
	119	+	return nil, err
	120	+	}
	121	+	}
	122	+	temps, err := os.ReadDir(filepath.Join(dir, "tmp"))
	123	+	if err != nil {
	124	+	return nil, err
	125	+	}
	126	+	for _, de := range temps {
	127	+	if err := os.Remove(filepath.Join(dir, "tmp", de.Name())); err != nil {
	128	+	return nil, err
	129	+	}
	130	+	}
	131	+	var types []string
	132	+	for _, kind := range []string{"blobs", "manifests", "tags"} {
	133	+	ds, err := os.ReadDir(filepath.Join(dir, kind))
	134	+	if err != nil {
	135	+	return nil, err
	136	+	}
	137	+	for _, de := range ds {
	138	+	if base, ok := strings.CutSuffix(de.Name(), ".type"); ok && kind == "manifests" {
	139	+	types = append(types, base)
	140	+	continue
	141	+	}
	142	+	if !validDigest("sha256:" + de.Name()) {
	143	+	continue
	144	+	}
	145	+	fi, err := de.Info()
	146	+	if err != nil {
	147	+	return nil, err
	148	+	}
	149	+	if !fi.Mode().IsRegular() {
	150	+	continue
	151	+	}
	152	+	key := kind + "/" + de.Name()
	153	+	e := &entry{key: key, size: fi.Size(), used: fi.ModTime()}
	154	+	if kind == "manifests" {
	155	+	info, err := os.Stat(filepath.Join(dir, key) + ".type")
	156	+	if err != nil {
	157	+	_ = os.Remove(filepath.Join(dir, key))
	158	+	continue
	159	+	}
	160	+	e.size += info.Size()
	161	+	}
	162	+	if kind == "tags" {
	163	+	digest := s.readTagFile(key)
	164	+	if !validDigest(digest) {
	165	+	_ = os.Remove(filepath.Join(dir, key))
	166	+	continue
	167	+	}
	168	+	if _, ok := s.entries["manifests/"+digest[7:]]; !ok {
	169	+	_ = os.Remove(filepath.Join(dir, key))
	170	+	continue
	171	+	}
	172	+	e.size = tagCharge
	173	+	e.digest = digest
	174	+	}
	175	+	s.addLocked(e)
	176	+	}
	177	+	}
	178	+	// A .type whose manifest is gone (a crash between the two writes).
	179	+	for _, base := range types {
	180	+	if _, ok := s.entries["manifests/"+base]; !ok {
	181	+	_ = os.Remove(filepath.Join(dir, "manifests", base+".type"))
	182	+	}
	183	+	}
	184	+	// Order the LRU by the access times kept in the mtimes.
	185	+	all := make([]*entry, 0, len(s.entries))
	186	+	for _, e := range s.entries {
	187	+	all = append(all, e)
	188	+	}
	189	+	sort.Slice(all, func(i, j int) bool { return all[i].used.Before(all[j].used) })
	190	+	s.lru.Init()
	191	+	for _, e := range all {
	192	+	e.elem = s.lru.PushBack(e)
	193	+	}
	194	+	s.evictLocked(0)
	195	+	return s, nil
	196	+	}
	197	+
	198	+	func (s store) addLocked(e entry) {
	199	+	s.entries[e.key] = e
	200	+	e.elem = s.lru.PushBack(e)
	201	+	s.total += e.size
	202	+	if e.digest != "" {
	203	+	s.indexTagLocked(e.key, e.digest)
	204	+	}
	205	+	}
	206	+
	207	+	func (s *store) indexTagLocked(key, digest string) {
	208	+	set := s.tagsOf[digest]
	209	+	if set == nil {
	210	+	set = map[string]struct{}{}
	211	+	s.tagsOf[digest] = set
	212	+	}
	213	+	set[key] = struct{}{}
	214	+	}
	215	+
	216	+	func (s *store) unindexTagLocked(key, digest string) {
	217	+	if set := s.tagsOf[digest]; set != nil {
	218	+	delete(set, key)
	219	+	if len(set) == 0 {
	220	+	delete(s.tagsOf, digest)
	221	+	}
	222	+	}
	223	+	}
	224	+
	225	+	func (s store) touchLocked(e entry) {
	226	+	e.used = s.now()
	227	+	s.lru.MoveToBack(e.elem)
	228	+	_ = os.Chtimes(filepath.Join(s.dir, e.key), e.used, e.used)
	229	+	}
	230	+
	231	+	func (s *store) unpinLocked(key string) {
	232	+	e, ok := s.entries[key]
	233	+	if !ok {
	234	+	return
	235	+	}
	236	+	if e.pins <= 0 {
	237	+	// A bug: the entry could have been evicted while open.
	238	+	s.log.Error().Str("key", key).Msg("registry cache unpin of an unpinned entry")
	239	+	if s.hookBug != nil {
	240	+	s.hookBug("unpin of an unpinned entry " + key)
	241	+	}
	242	+	return
	243	+	}
	244	+	e.pins--
	245	+	}
	246	+
	247	+	// removeLocked only releases a charge after the corresponding file is gone.
	248	+	// Failures leave the entry charged so a full or unwritable disk fails closed.
	249	+	func (s *store) removeLocked(key string) bool {
	250	+	e, ok := s.entries[key]
	251	+	if !ok {
	252	+	return true
	253	+	}
	254	+	if e.pins > 0 {
	255	+	return false
	256	+	}
	257	+	if hexDigest, ok := strings.CutPrefix(key, "manifests/"); ok {
	258	+	// Prune references first. If a tag cannot be removed, retain its target too.
	259	+	for k := range s.tagsOf["sha256:"+hexDigest] {
	260	+	if !s.removeLocked(k) {
	261	+	return false
	262	+	}
	263	+	}
	264	+	if err := os.Remove(filepath.Join(s.dir, key) + ".type"); err != nil && !os.IsNotExist(err) {
	265	+	return false
	266	+	}
	267	+	}
	268	+	if err := os.Remove(filepath.Join(s.dir, key)); err != nil && !os.IsNotExist(err) {
	269	+	return false
	270	+	}
	271	+	s.total -= e.size
	272	+	delete(s.entries, key)
	273	+	s.lru.Remove(e.elem)
	274	+	e.elem = nil
	275	+	if e.digest != "" {
	276	+	s.unindexTagLocked(key, e.digest)
	277	+	}
	278	+	return true
	279	+	}
	280	+
	281	+	// evictLocked makes room for need bytes. Once over the cap, it walks the LRU
	282	+	// (no sort) and evicts unpinned entries until max(need, low water) bytes are
	283	+	// free, so a full store evicts once per low-water bytes, not on every write.
	284	+	func (s *store) evictLocked(need int64) {
	285	+	if s.total+s.pending+need <= s.max {
	286	+	return
	287	+	}
	288	+	target := s.max - max(need, min(lowWater, s.max/20))
	289	+	for el := s.lru.Front(); el != nil && s.total+s.pending > target; {
	290	+	e := el.Value.(*entry)
	291	+	next := el.Next()
	292	+	if e.pins == 0 {
	293	+	removed := s.removeLocked(e.key)
	294	+	if strings.HasPrefix(e.key, "manifests/") {
	295	+	// Removing a manifest removes its tags first, even when it then
	296	+	// fails: next may be gone. Start over; only pinned or
	297	+	// unremovable entries are walked again.
	298	+	next = s.lru.Front()
	299	+	if !removed {
	300	+	// Do not walk into it again: it stays where it is, and
	301	+	// the walk resumes after it.
	302	+	next = e.elem.Next()
	303	+	}
	304	+	}
	305	+	}
	306	+	el = next
	307	+	}
	308	+	}
	309	+
	310	+	func (s store) openLocked(key string) (storedFile, string, error) {
	311	+	e, ok := s.entries[key]
	312	+	if !ok {
	313	+	return nil, "", os.ErrNotExist
	314	+	}
	315	+	f, err := os.Open(filepath.Join(s.dir, key))
	316	+	if err != nil {
	317	+	return nil, "", err
	318	+	}
	319	+	media := "application/octet-stream"
	320	+	if strings.HasPrefix(key, "manifests/") {
	321	+	b, err := os.ReadFile(filepath.Join(s.dir, key) + ".type")
	322	+	if err != nil {
	323	+	_ = f.Close()
	324	+	return nil, "", err
	325	+	}
	326	+	media = string(b)
	327	+	}
	328	+	s.touchLocked(e)
	329	+	e.pins++
	330	+	return &storedFile{File: f, s: s, key: key}, media, nil
	331	+	}
	332	+	func (s store) open(kind, digest string) (storedFile, string, error) {
	333	+	if !validDigest(digest) \|\| (kind != "blobs" && kind != "manifests") {
	334	+	return nil, "", errDigest
	335	+	}
	336	+	s.mu.Lock()
	337	+	defer s.mu.Unlock()
	338	+	return s.openLocked(kind + "/" + digest[7:])
	339	+	}
	340	+
	341	+	// fetch writes the upstream content to w. ctx belongs to the flight: it ends
	342	+	// when the last waiter has been gone for the grace period.
	343	+	type fetch func(ctx context.Context, w io.Writer) (string, error)
	344	+
	345	+	type keepalive struct {
	346	+	every time.Duration
	347	+	fn func()
	348	+	}
	349	+
	350	+	// ensure returns the stored content, pinned for the caller; hit reports that it
	351	+	// was already stored. Otherwise the request joins or starts the flight of that
	352	+	// digest. The flight runs on its own goroutine and context, so no single
	353	+	// request, the initiator included, can cancel it while another one waits; it is
	354	+	// cancelled once no request has waited for it for the grace period. Every
	355	+	// waiter waits for the verified commit and then reads the stored file. While
	356	+	// waiting, keep.fn is called every keep.every when the download has made
	357	+	// progress, so that the waiter's connection deadlines can follow it.
	358	+	//
	359	+	// Starting a blob flight takes a global download slot and one of the quota
	360	+	// carried by ctx (WithQuota), both held until the flight really ends; joining
	361	+	// one takes neither. When a slot is missing, the request waits for one, kept
	362	+	// alive, for at most the slot wait, and then fails with errBusy.
	363	+	func (s store) ensure(ctx context.Context, kind, digest string, fetch fetch, keep keepalive) (f storedFile, media string, hit bool, err error) {
	364	+	if !validDigest(digest) \|\| (kind != "blobs" && kind != "manifests") {
	365	+	return nil, "", false, errDigest
	366	+	}
	367	+	key := kind + "/" + digest[7:]
	368	+	blob := kind == "blobs"
	369	+	quota := quotaFrom(ctx)
	370	+	var slotDeadline <-chan time.Time
	371	+	var fl *flight
	372	+	for {
	373	+	s.mu.Lock()
	374	+	if f, media, err := s.openLocked(key); err == nil {
	375	+	s.mu.Unlock()
	376	+	return f, media, true, nil
	377	+	}
	378	+	var ok bool
	379	+	if fl, ok = s.flights[key]; ok {
	380	+	break // join: no slot needed
	381	+	}
	382	+	if !blob \|\| s.takeSlotLocked(quota) {
	383	+	dctx, cancel := context.WithCancel(context.Background())
	384	+	fl = &flight{key: key, done: make(chan struct{}), cancel: cancel, blob: blob}
	385	+	if blob {
	386	+	fl.quota = quota
	387	+	}
	388	+	s.flights[key] = fl
	389	+	go s.run(dctx, fl, digest, fetch)
	390	+	break
	391	+	}
	392	+	freed := s.slotFreed
	393	+	s.mu.Unlock()
	394	+	if slotDeadline == nil {
	395	+	t := time.NewTimer(s.slotWait)
	396	+	defer t.Stop()
	397	+	slotDeadline = t.C
	398	+	}
	399	+	if err := waitSlot(ctx, freed, slotDeadline, keep); err != nil {
	400	+	return nil, "", false, err
	401	+	}
	402	+	}
	403	+	fl.waiters++
	404	+	s.mu.Unlock()
	405	+
	406	+	err = waitFlight(ctx, fl, keep)
	407	+	s.mu.Lock()
	408	+	defer s.mu.Unlock()
	409	+	if err == nil {
	410	+	f, media, err = s.openLocked(key)
	411	+	}
	412	+	s.leaveLocked(fl)
	413	+	return f, media, false, err
	414	+	}
	415	+
	416	+	// leaveLocked drops a waiter. The last one releases the commit pin, or, before
	417	+	// the commit, starts the grace period after which the flight is cancelled.
	418	+	func (s store) leaveLocked(fl flight) {
	419	+	fl.waiters--
	420	+	if fl.waiters > 0 {
	421	+	return
	422	+	}
	423	+	if fl.finished {
	424	+	if fl.pinned {
	425	+	fl.pinned = false
	426	+	s.unpinLocked(fl.key)
	427	+	}
	428	+	return
	429	+	}
	430	+	fl.gen++
	431	+	gen := fl.gen
	432	+	// A manifest is small and bounded by the request timeout: it is cancelled at
	433	+	// once, so its flights never outnumber the requests in progress. A blob
	434	+	// holds a download slot until it really ends, which bounds the abandoned
	435	+	// ones to MaxDownloads.
	436	+	if s.grace <= 0 \|\| !fl.blob {
	437	+	s.abandonLocked(fl)
	438	+	return
	439	+	}
	440	+	time.AfterFunc(s.grace, func() {
	441	+	s.mu.Lock()
	442	+	defer s.mu.Unlock()
	443	+	if fl.waiters == 0 && !fl.finished && fl.gen == gen {
	444	+	s.abandonLocked(fl)
	445	+	}
	446	+	})
	447	+	}
	448	+
	449	+	// abandonLocked cancels a flight nobody waits for. A later request for the
	450	+	// digest starts a new one.
	451	+	func (s store) abandonLocked(fl flight) {
	452	+	fl.cancel()
	453	+	if s.flights[fl.key] == fl {
	454	+	delete(s.flights, fl.key)
	455	+	}
	456	+	}
	457	+
	458	+	// takeSlotLocked takes a global download slot and one of quota (nil: none),
	459	+	// both or neither.
	460	+	func (s store) takeSlotLocked(quota Quota) bool {
	461	+	if s.maxDownloads > 0 && s.downloads >= s.maxDownloads {
	462	+	return false
	463	+	}
	464	+	if quota != nil && !quota.tryAcquire() {
	465	+	return false
	466	+	}
	467	+	s.downloads++
	468	+	return true
	469	+	}
	470	+
	471	+	// waitSlot waits until a slot may have freed (freed is closed), keeping the
	472	+	// client alive meanwhile. It fails with errBusy at the deadline.
	473	+	func waitSlot(ctx context.Context, freed <-chan struct{}, deadline <-chan time.Time, keep keepalive) error {
	474	+	var tick <-chan time.Time
	475	+	if keep.every > 0 && keep.fn != nil {
	476	+	t := time.NewTicker(keep.every)
	477	+	defer t.Stop()
	478	+	tick = t.C
	479	+	}
	480	+	for {
	481	+	select {
	482	+	case <-freed:
	483	+	return nil
	484	+	case <-ctx.Done():
	485	+	return ctx.Err()
	486	+	case <-deadline:
	487	+	return errBusy
	488	+	case <-tick:
	489	+	keep.fn() // bounded by the slot wait
	490	+	}
	491	+	}
	492	+	}
	493	+
	494	+	// finishLocked ends a flight and releases its slots. A committed entry is
	495	+	// pinned on behalf of the waiters, so that no eviction removes it before they
	496	+	// open it.
	497	+	func (s store) finishLocked(fl flight, err error, committed *entry) {
	498	+	fl.err = err
	499	+	fl.finished = true
	500	+	if err == nil && committed != nil && fl.waiters > 0 {
	501	+	committed.pins++
	502	+	fl.pinned = true
	503	+	}
	504	+	if s.flights[fl.key] == fl {
	505	+	delete(s.flights, fl.key)
	506	+	}
	507	+	if fl.blob {
	508	+	s.downloads--
	509	+	if fl.quota != nil {
	510	+	fl.quota.release()
	511	+	}
	512	+	close(s.slotFreed)
	513	+	s.slotFreed = make(chan struct{})
	514	+	}
	515	+	}
	516	+
	517	+	// run downloads and finishes the flight. A panic in fetch fails the flight
	518	+	// (and frees its slots) rather than leave its waiters hanging.
	519	+	func (s store) run(ctx context.Context, fl flight, digest string, fetch fetch) {
	520	+	var err error
	521	+	defer func() {
	522	+	if v := recover(); v != nil {
	523	+	err = fmt.Errorf("registry download panicked: %v", v)
	524	+	s.log.Error().Str("key", fl.key).Err(err).Msg("registry download")
	525	+	}
	526	+	fl.cancel()
	527	+	s.mu.Lock()
	528	+	if !fl.finished {
	529	+	if err == nil {
	530	+	err = errors.New("registry download ended without a commit")
	531	+	}
	532	+	s.finishLocked(fl, err, nil)
	533	+	}
	534	+	s.mu.Unlock()
	535	+	if fl.err == nil && s.hookCommitted != nil {
	536	+	s.hookCommitted(fl.key)
	537	+	}
	538	+	close(fl.done)
	539	+	}()
	540	+	err = s.fill(ctx, fl, digest, fetch)
	541	+	}
	542	+
	543	+	func waitFlight(ctx context.Context, fl *flight, keep keepalive) error {
	544	+	var tick <-chan time.Time
	545	+	if keep.every > 0 && keep.fn != nil {
	546	+	t := time.NewTicker(keep.every)
	547	+	defer t.Stop()
	548	+	tick = t.C
	549	+	}
	550	+	last := fl.written.Load()
	551	+	for {
	552	+	select {
	553	+	case <-ctx.Done():
	554	+	return ctx.Err()
	555	+	case <-fl.done:
	556	+	return fl.err
	557	+	case <-tick:
	558	+	if n := fl.written.Load(); n != last {
	559	+	last = n
	560	+	keep.fn()
	561	+	}
	562	+	}
	563	+	}
	564	+	}
	565	+
	566	+	type chargedWriter struct {
	567	+	s *store
	568	+	w io.Writer
	569	+	n int64
	570	+	written *atomic.Int64
	571	+	}
	572	+
	573	+	func (w *chargedWriter) Write(b []byte) (int, error) {
	574	+	s := w.s
	575	+	s.mu.Lock()
	576	+	need := int64(len(b))
	577	+	s.evictLocked(need)
	578	+	if need > s.max-s.total-s.pending {
	579	+	s.mu.Unlock()
	580	+	return 0, errTooLarge
	581	+	}
	582	+	s.pending += need
	583	+	w.n += need
	584	+	s.mu.Unlock()
	585	+	n, err := w.w.Write(b)
	586	+	if n < len(b) {
	587	+	s.mu.Lock()
	588	+	s.pending -= int64(len(b) - n)
	589	+	w.n -= int64(len(b) - n)
	590	+	s.mu.Unlock()
	591	+	}
	592	+	w.written.Add(int64(n))
	593	+	return n, err
	594	+	}
	595	+
	596	+	// fill downloads into tmp/, verifies the digest and commits. On success it has
	597	+	// finished the flight (finishLocked) under the same lock as the commit.
	598	+	func (s store) fill(ctx context.Context, fl flight, digest string, fetch fetch) error {
	599	+	key := fl.key
	600	+	f, err := os.CreateTemp(filepath.Join(s.dir, "tmp"), "download-")
	601	+	if err != nil {
	602	+	return err
	603	+	}
	604	+	defer func() { _ = f.Close(); _ = os.Remove(f.Name()) }()
	605	+	h := sha256.New()
	606	+	cw := &chargedWriter{s: s, w: io.MultiWriter(f, h), written: &fl.written}
	607	+	defer func() { s.mu.Lock(); s.pending -= cw.n; s.mu.Unlock() }()
	608	+	media, err := fetch(ctx, cw)
	609	+	if err != nil {
	610	+	return err
	611	+	}
	612	+	if "sha256:"+hex.EncodeToString(h.Sum(nil)) != digest {
	613	+	return errDigest
	614	+	}
	615	+	if err = f.Close(); err != nil {
	616	+	return err
	617	+	}
	618	+	s.mu.Lock()
	619	+	defer s.mu.Unlock()
	620	+	if e, ok := s.entries[key]; ok {
	621	+	// Same digest, same bytes: keep the stored file and its pins rather than
	622	+	// renaming over a file that may be open (its bytes would leave the
	623	+	// accounting while still allocated).
	624	+	s.finishLocked(fl, nil, e)
	625	+	return nil
	626	+	}
	627	+	path := filepath.Join(s.dir, key)
	628	+	metadata := int64(0)
	629	+	if strings.HasPrefix(key, "manifests/") {
	630	+	if len(media) > 1024 {
	631	+	return errors.New("manifest media type too large")
	632	+	}
	633	+	metadata = int64(len(media))
	634	+	s.evictLocked(metadata)
	635	+	if metadata > s.max-s.total-s.pending {
	636	+	return errTooLarge
	637	+	}
	638	+	if err := os.WriteFile(path+".type", []byte(media), 0600); err != nil {
	639	+	return err
	640	+	}
	641	+	}
	642	+	if err := os.Rename(f.Name(), path); err != nil {
	643	+	if metadata > 0 {
	644	+	_ = os.Remove(path + ".type")
	645	+	}
	646	+	return err
	647	+	}
	648	+	e := &entry{key: key, size: cw.n + metadata, used: s.now()}
	649	+	s.addLocked(e)
	650	+	s.pending -= cw.n
	651	+	cw.n = 0
	652	+	s.finishLocked(fl, nil, e)
	653	+	return nil
	654	+	}
	655	+	func tagKey(name, tag, accept string) string {
	656	+	return fmt.Sprintf("%x", sha256.Sum256([]byte(name+"\x00"+tag+"\x00"+normalizeAccept(accept))))
	657	+	}
	658	+	func (s *store) readTagFile(key string) string {
	659	+	f, err := os.Open(filepath.Join(s.dir, key))
	660	+	if err != nil {
	661	+	return ""
	662	+	}
	663	+	defer func() { _ = f.Close() }()
	664	+	b, err := io.ReadAll(io.LimitReader(f, 72))
	665	+	if err != nil \|\| !validDigest(string(b)) {
	666	+	return ""
	667	+	}
	668	+	return string(b)
	669	+	}
	670	+	func (s *store) tag(name, tag, accept string) string {
	671	+	s.mu.Lock()
	672	+	defer s.mu.Unlock()
	673	+	e, ok := s.entries["tags/"+tagKey(name, tag, accept)]
	674	+	if !ok {
	675	+	return ""
	676	+	}
	677	+	s.touchLocked(e)
	678	+	return e.digest
	679	+	}
	680	+	func (s *store) setTag(name, tag, accept, digest string) error {
	681	+	if !validDigest(digest) {
	682	+	return errDigest
	683	+	}
	684	+	s.mu.Lock()
	685	+	defer s.mu.Unlock()
	686	+	manifest, ok := s.entries["manifests/"+digest[7:]]
	687	+	if !ok {
	688	+	return os.ErrNotExist
	689	+	}
	690	+	// Protect the target while making room for its metadata.
	691	+	manifest.pins++
	692	+	defer func() { manifest.pins-- }()
	693	+	key := "tags/" + tagKey(name, tag, accept)
	694	+	old, exists := s.entries[key]
	695	+	if exists {
	696	+	// Already charged: rewriting it needs no room.
	697	+	s.touchLocked(old)
	698	+	} else {
	699	+	s.evictLocked(tagCharge)
	700	+	if tagCharge > s.max-s.total-s.pending {
	701	+	return errTooLarge
	702	+	}
	703	+	}
	704	+	f, err := os.CreateTemp(filepath.Join(s.dir, "tmp"), "tag-")
	705	+	if err != nil {
	706	+	return err
	707	+	}
	708	+	defer func() { _ = f.Close(); _ = os.Remove(f.Name()) }()
	709	+	if _, err = f.WriteString(digest); err != nil {
	710	+	return err
	711	+	}
	712	+	if err = f.Close(); err != nil {
	713	+	return err
	714	+	}
	715	+	if err = os.Rename(f.Name(), filepath.Join(s.dir, key)); err != nil {
	716	+	return err
	717	+	}
	718	+	if old, ok := s.entries[key]; ok {
	719	+	s.unindexTagLocked(key, old.digest)
	720	+	old.digest = digest
	721	+	old.used = s.now()
	722	+	s.indexTagLocked(key, digest)
	723	+	return nil
	724	+	}
	725	+	s.addLocked(&entry{key: key, size: tagCharge, used: s.now(), digest: digest})
	726	+	return nil
	727	+	}

A internal/registryproxy/store_test.go +125 -0

@@ -0,0 +1,125 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+	package registryproxy
	3	+
	4	+	import (
	5	+	"context"
	6	+	"crypto/sha256"
	7	+	"fmt"
	8	+	"io"
	9	+	"os"
	10	+	"path/filepath"
	11	+	"sync"
	12	+	"sync/atomic"
	13	+	"testing"
	14	+	"time"
	15	+	)
	16	+
	17	+	func sum(b string) string { return fmt.Sprintf("sha256:%x", sha256.Sum256([]byte(b))) }
	18	+	func put(t testing.T, s store, b string) {
	19	+	t.Helper()
	20	+	f, _, _, err := s.ensure(context.Background(), "blobs", sum(b), func(_ context.Context, w io.Writer) (string, error) { _, err := io.WriteString(w, b); return "", err }, keepalive{})
	21	+	if err != nil {
	22	+	t.Fatal(err)
	23	+	}
	24	+	_ = f.Close()
	25	+	}
	26	+	func TestStoreIntegrityReopenEviction(t *testing.T) {
	27	+	now := time.Now()
	28	+	dir := t.TempDir()
	29	+	s, err := openStore(dir, 8, func() time.Time { return now })
	30	+	if err != nil {
	31	+	t.Fatal(err)
	32	+	}
	33	+	put(t, s, "aaaa")
	34	+	now = now.Add(time.Second)
	35	+	put(t, s, "bbbb")
	36	+	now = now.Add(time.Second)
	37	+	f, _, err := s.open("blobs", sum("aaaa"))
	38	+	if err != nil {
	39	+	t.Fatal(err)
	40	+	}
	41	+	_ = f.Close()
	42	+	now = now.Add(time.Second)
	43	+	put(t, s, "cccc")
	44	+	if f, _, err := s.open("blobs", sum("bbbb")); err == nil {
	45	+	_ = f.Close()
	46	+	t.Fatal("LRU survived eviction")
	47	+	}
	48	+	_, _, _, err = s.ensure(context.Background(), "blobs", sum("dddd"), func(_ context.Context, w io.Writer) (string, error) {
	49	+	_, err := io.WriteString(w, "evil")
	50	+	return "", err
	51	+	}, keepalive{})
	52	+	if err != errDigest {
	53	+	t.Fatalf("mismatch: %v", err)
	54	+	}
	55	+	if _, err := os.Stat(filepath.Join(dir, "blobs", sum("dddd")[7:])); !os.IsNotExist(err) {
	56	+	t.Fatalf("bad commit: %v", err)
	57	+	}
	58	+	put(t, s, "eeee")
	59	+	if err := os.WriteFile(filepath.Join(dir, "tmp", "leftover"), []byte("partial"), 0600); err != nil {
	60	+	t.Fatal(err)
	61	+	}
	62	+	s, err = openStore(dir, 8, time.Now)
	63	+	if err != nil {
	64	+	t.Fatal(err)
	65	+	}
	66	+	if f, _, err := s.open("blobs", sum("eeee")); err != nil {
	67	+	t.Fatal(err)
	68	+	} else {
	69	+	_ = f.Close()
	70	+	}
	71	+	ds, _ := os.ReadDir(filepath.Join(dir, "tmp"))
	72	+	if len(ds) != 0 {
	73	+	t.Fatal("tmp not cleared")
	74	+	}
	75	+	s.mu.Lock()
	76	+	defer s.mu.Unlock()
	77	+	if s.total+s.pending > 8 {
	78	+	t.Fatal("cap exceeded")
	79	+	}
	80	+	}
	81	+	func TestStoreConcurrentEnsure(t *testing.T) {
	82	+	s, err := openStore(t.TempDir(), 1024, time.Now)
	83	+	if err != nil {
	84	+	t.Fatal(err)
	85	+	}
	86	+	var calls atomic.Int32
	87	+	var wg sync.WaitGroup
	88	+	start := make(chan struct{})
	89	+	for range 20 {
	90	+	wg.Go(func() {
	91	+	<-start
	92	+	f, _, _, err := s.ensure(context.Background(), "blobs", sum("content"), func(_ context.Context, w io.Writer) (string, error) {
	93	+	calls.Add(1)
	94	+	_, err := io.WriteString(w, "content")
	95	+	return "", err
	96	+	}, keepalive{})
	97	+	if err != nil {
	98	+	t.Error(err)
	99	+	return
	100	+	}
	101	+	_ = f.Close()
	102	+	})
	103	+	}
	104	+	close(start)
	105	+	wg.Wait()
	106	+	if calls.Load() != 1 {
	107	+	t.Fatalf("fetches %d", calls.Load())
	108	+	}
	109	+	}
	110	+	func TestStoreOversize(t *testing.T) {
	111	+	s, err := openStore(t.TempDir(), 3, time.Now)
	112	+	if err != nil {
	113	+	t.Fatal(err)
	114	+	}
	115	+	_, _, _, err = s.ensure(context.Background(), "blobs", sum("large"), func(_ context.Context, w io.Writer) (string, error) {
	116	+	_, err := io.WriteString(w, "large")
	117	+	return "", err
	118	+	}, keepalive{})
	119	+	if err != errTooLarge {
	120	+	t.Fatal(err)
	121	+	}
	122	+	if s.pending != 0 \|\| s.total != 0 {
	123	+	t.Fatalf("accounting: %d %d", s.pending, s.total)
	124	+	}
	125	+	}

@@ -0,0 +1,125 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+	package registryproxy
	3	+
	4	+	import (
	5	+	"context"
	6	+	"crypto/sha256"
	7	+	"fmt"
	8	+	"io"
	9	+	"os"
	10	+	"path/filepath"
	11	+	"sync"
	12	+	"sync/atomic"
	13	+	"testing"
	14	+	"time"
	15	+	)
	16	+
	17	+	func sum(b string) string { return fmt.Sprintf("sha256:%x", sha256.Sum256([]byte(b))) }
	18	+	func put(t testing.T, s store, b string) {
	19	+	t.Helper()
	20	+	f, _, _, err := s.ensure(context.Background(), "blobs", sum(b), func(_ context.Context, w io.Writer) (string, error) { _, err := io.WriteString(w, b); return "", err }, keepalive{})
	21	+	if err != nil {
	22	+	t.Fatal(err)
	23	+	}
	24	+	_ = f.Close()
	25	+	}
	26	+	func TestStoreIntegrityReopenEviction(t *testing.T) {
	27	+	now := time.Now()
	28	+	dir := t.TempDir()
	29	+	s, err := openStore(dir, 8, func() time.Time { return now })
	30	+	if err != nil {
	31	+	t.Fatal(err)
	32	+	}
	33	+	put(t, s, "aaaa")
	34	+	now = now.Add(time.Second)
	35	+	put(t, s, "bbbb")
	36	+	now = now.Add(time.Second)
	37	+	f, _, err := s.open("blobs", sum("aaaa"))
	38	+	if err != nil {
	39	+	t.Fatal(err)
	40	+	}
	41	+	_ = f.Close()
	42	+	now = now.Add(time.Second)
	43	+	put(t, s, "cccc")
	44	+	if f, _, err := s.open("blobs", sum("bbbb")); err == nil {
	45	+	_ = f.Close()
	46	+	t.Fatal("LRU survived eviction")
	47	+	}
	48	+	_, _, _, err = s.ensure(context.Background(), "blobs", sum("dddd"), func(_ context.Context, w io.Writer) (string, error) {
	49	+	_, err := io.WriteString(w, "evil")
	50	+	return "", err
	51	+	}, keepalive{})
	52	+	if err != errDigest {
	53	+	t.Fatalf("mismatch: %v", err)
	54	+	}
	55	+	if _, err := os.Stat(filepath.Join(dir, "blobs", sum("dddd")[7:])); !os.IsNotExist(err) {
	56	+	t.Fatalf("bad commit: %v", err)
	57	+	}
	58	+	put(t, s, "eeee")
	59	+	if err := os.WriteFile(filepath.Join(dir, "tmp", "leftover"), []byte("partial"), 0600); err != nil {
	60	+	t.Fatal(err)
	61	+	}
	62	+	s, err = openStore(dir, 8, time.Now)
	63	+	if err != nil {
	64	+	t.Fatal(err)
	65	+	}
	66	+	if f, _, err := s.open("blobs", sum("eeee")); err != nil {
	67	+	t.Fatal(err)
	68	+	} else {
	69	+	_ = f.Close()
	70	+	}
	71	+	ds, _ := os.ReadDir(filepath.Join(dir, "tmp"))
	72	+	if len(ds) != 0 {
	73	+	t.Fatal("tmp not cleared")
	74	+	}
	75	+	s.mu.Lock()
	76	+	defer s.mu.Unlock()
	77	+	if s.total+s.pending > 8 {
	78	+	t.Fatal("cap exceeded")
	79	+	}
	80	+	}
	81	+	func TestStoreConcurrentEnsure(t *testing.T) {
	82	+	s, err := openStore(t.TempDir(), 1024, time.Now)
	83	+	if err != nil {
	84	+	t.Fatal(err)
	85	+	}
	86	+	var calls atomic.Int32
	87	+	var wg sync.WaitGroup
	88	+	start := make(chan struct{})
	89	+	for range 20 {
	90	+	wg.Go(func() {
	91	+	<-start
	92	+	f, _, _, err := s.ensure(context.Background(), "blobs", sum("content"), func(_ context.Context, w io.Writer) (string, error) {
	93	+	calls.Add(1)
	94	+	_, err := io.WriteString(w, "content")
	95	+	return "", err
	96	+	}, keepalive{})
	97	+	if err != nil {
	98	+	t.Error(err)
	99	+	return
	100	+	}
	101	+	_ = f.Close()
	102	+	})
	103	+	}
	104	+	close(start)
	105	+	wg.Wait()
	106	+	if calls.Load() != 1 {
	107	+	t.Fatalf("fetches %d", calls.Load())
	108	+	}
	109	+	}
	110	+	func TestStoreOversize(t *testing.T) {
	111	+	s, err := openStore(t.TempDir(), 3, time.Now)
	112	+	if err != nil {
	113	+	t.Fatal(err)
	114	+	}
	115	+	_, _, _, err = s.ensure(context.Background(), "blobs", sum("large"), func(_ context.Context, w io.Writer) (string, error) {
	116	+	_, err := io.WriteString(w, "large")
	117	+	return "", err
	118	+	}, keepalive{})
	119	+	if err != errTooLarge {
	120	+	t.Fatal(err)
	121	+	}
	122	+	if s.pending != 0 \|\| s.total != 0 {
	123	+	t.Fatalf("accounting: %d %d", s.pending, s.total)
	124	+	}
	125	+	}

A internal/registryproxy/upstream.go +156 -0

@@ -0,0 +1,156 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package registryproxy
	4	+
	5	+	import (
	6	+	"context"
	7	+	"encoding/json"
	8	+	"errors"
	9	+	"fmt"
	10	+	"io"
	11	+	"net"
	12	+	"net/http"
	13	+	"net/url"
	14	+	"sync"
	15	+	"time"
	16	+	)
	17	+
	18	+	type upstreamError struct{ status int }
	19	+
	20	+	func (e upstreamError) Error() string { return fmt.Sprintf("registry upstream status %d", e.status) }
	21	+
	22	+	// unavailable reports an upstream that cannot answer now (down, timing out,
	23	+	// failing or rate limiting): a known tag may then be served stale.
	24	+	func unavailable(err error) bool {
	25	+	var e upstreamError
	26	+	if errors.As(err, &e) {
	27	+	return e.status >= http.StatusInternalServerError \|\| e.status == http.StatusTooManyRequests
	28	+	}
	29	+	var network net.Error
	30	+	return errors.As(err, &network) \|\| errors.Is(err, context.DeadlineExceeded)
	31	+	}
	32	+	func proxyStatus(err error) int {
	33	+	var e upstreamError
	34	+	if errors.As(err, &e) && (e.status == http.StatusUnauthorized \|\| e.status == http.StatusForbidden \|\| e.status == http.StatusNotFound) {
	35	+	return http.StatusNotFound
	36	+	}
	37	+	return http.StatusBadGateway
	38	+	}
	39	+
	40	+	type bearer struct {
	41	+	value string
	42	+	until time.Time
	43	+	}
	44	+	type upstream struct {
	45	+	base, auth string
	46	+	client *http.Client
	47	+	blobClient *http.Client
	48	+	requestTimeout, blobIdle time.Duration
	49	+	now func() time.Time
	50	+	mu sync.Mutex
	51	+	tokens map[string]bearer
	52	+	}
	53	+
	54	+	func (u *upstream) token(ctx context.Context, name string, refresh bool) (string, error) {
	55	+	// Do not hold the token map lock over network I/O: an unavailable auth
	56	+	// endpoint must not queue unrelated repositories behind its timeout.
	57	+	// No guest credentials or challenge-controlled URLs are accepted.
	58	+	u.mu.Lock()
	59	+	t, ok := u.tokens[name]
	60	+	u.mu.Unlock()
	61	+	if ok && !refresh && u.now().Before(t.until) {
	62	+	return t.value, nil
	63	+	}
	64	+	a, err := url.Parse(u.auth)
	65	+	if err != nil {
	66	+	return "", err
	67	+	}
	68	+	q := a.Query()
	69	+	q.Set("service", "registry.docker.io")
	70	+	q.Set("scope", "repository:"+name+":pull")
	71	+	a.RawQuery = q.Encode()
	72	+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, a.String(), nil)
	73	+	if err != nil {
	74	+	return "", err
	75	+	}
	76	+	resp, err := u.client.Do(req)
	77	+	if err != nil {
	78	+	return "", err
	79	+	}
	80	+	defer func() { _ = resp.Body.Close() }()
	81	+	if resp.StatusCode != http.StatusOK {
	82	+	return "", upstreamError{resp.StatusCode}
	83	+	}
	84	+	var data struct {
	85	+	Token string `json:"token"`
	86	+	AccessToken string `json:"access_token"`
	87	+	Expires int64 `json:"expires_in"`
	88	+	}
	89	+	b, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20+1))
	90	+	if err != nil {
	91	+	return "", err
	92	+	}
	93	+	if len(b) > 1<<20 {
	94	+	return "", errors.New("registry token response too large")
	95	+	}
	96	+	if err = json.Unmarshal(b, &data); err != nil {
	97	+	return "", err
	98	+	}
	99	+	if data.Token == "" {
	100	+	data.Token = data.AccessToken
	101	+	}
	102	+	if data.Token == "" {
	103	+	return "", errors.New("empty registry token")
	104	+	}
	105	+	if data.Expires <= 0 {
	106	+	data.Expires = 60
	107	+	}
	108	+	if data.Expires > 86400 {
	109	+	data.Expires = 86400
	110	+	}
	111	+	u.mu.Lock()
	112	+	defer u.mu.Unlock()
	113	+	if len(u.tokens) >= 1024 {
	114	+	clear(u.tokens)
	115	+	}
	116	+	u.tokens[name] = bearer{data.Token, u.now().Add(time.Duration(data.Expires) * time.Second)}
	117	+	return data.Token, nil
	118	+	}
	119	+
	120	+	// request is a metadata request (manifests, blob HEAD) bounded by ctx. Manifests
	121	+	// use the client that never follows redirects; a blob HEAD uses the blob client,
	122	+	// because Docker Hub answers it with a redirect to its CDN.
	123	+	func (u upstream) request(ctx context.Context, method, name, kind, ref, accept string) (http.Response, error) {
	124	+	client := u.client
	125	+	if kind == "blobs" {
	126	+	client = u.blobClient
	127	+	}
	128	+	for attempt := 0; attempt < 2; attempt++ {
	129	+	tok, err := u.token(ctx, name, attempt > 0)
	130	+	if err != nil {
	131	+	return nil, err
	132	+	}
	133	+	req, err := http.NewRequestWithContext(ctx, method, u.base+"/v2/"+name+"/"+kind+"/"+ref, nil)
	134	+	if err != nil {
	135	+	return nil, err
	136	+	}
	137	+	req.Header.Set("Authorization", "Bearer "+tok)
	138	+	if accept != "" {
	139	+	req.Header.Set("Accept", accept)
	140	+	}
	141	+	// Do not forward guest Authorization, cookies, query parameters or Range.
	142	+	resp, err := client.Do(req)
	143	+	if err != nil {
	144	+	return nil, err
	145	+	}
	146	+	if resp.StatusCode == http.StatusOK {
	147	+	return resp, nil
	148	+	}
	149	+	_ = resp.Body.Close()
	150	+	if (resp.StatusCode == http.StatusUnauthorized \|\| resp.StatusCode == http.StatusForbidden) && attempt == 0 {
	151	+	continue
	152	+	}
	153	+	return nil, upstreamError{resp.StatusCode}
	154	+	}
	155	+	return nil, upstreamError{http.StatusUnauthorized}
	156	+	}

@@ -0,0 +1,156 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package registryproxy
	4	+
	5	+	import (
	6	+	"context"
	7	+	"encoding/json"
	8	+	"errors"
	9	+	"fmt"
	10	+	"io"
	11	+	"net"
	12	+	"net/http"
	13	+	"net/url"
	14	+	"sync"
	15	+	"time"
	16	+	)
	17	+
	18	+	type upstreamError struct{ status int }
	19	+
	20	+	func (e upstreamError) Error() string { return fmt.Sprintf("registry upstream status %d", e.status) }
	21	+
	22	+	// unavailable reports an upstream that cannot answer now (down, timing out,
	23	+	// failing or rate limiting): a known tag may then be served stale.
	24	+	func unavailable(err error) bool {
	25	+	var e upstreamError
	26	+	if errors.As(err, &e) {
	27	+	return e.status >= http.StatusInternalServerError \|\| e.status == http.StatusTooManyRequests
	28	+	}
	29	+	var network net.Error
	30	+	return errors.As(err, &network) \|\| errors.Is(err, context.DeadlineExceeded)
	31	+	}
	32	+	func proxyStatus(err error) int {
	33	+	var e upstreamError
	34	+	if errors.As(err, &e) && (e.status == http.StatusUnauthorized \|\| e.status == http.StatusForbidden \|\| e.status == http.StatusNotFound) {
	35	+	return http.StatusNotFound
	36	+	}
	37	+	return http.StatusBadGateway
	38	+	}
	39	+
	40	+	type bearer struct {
	41	+	value string
	42	+	until time.Time
	43	+	}
	44	+	type upstream struct {
	45	+	base, auth string
	46	+	client *http.Client
	47	+	blobClient *http.Client
	48	+	requestTimeout, blobIdle time.Duration
	49	+	now func() time.Time
	50	+	mu sync.Mutex
	51	+	tokens map[string]bearer
	52	+	}
	53	+
	54	+	func (u *upstream) token(ctx context.Context, name string, refresh bool) (string, error) {
	55	+	// Do not hold the token map lock over network I/O: an unavailable auth
	56	+	// endpoint must not queue unrelated repositories behind its timeout.
	57	+	// No guest credentials or challenge-controlled URLs are accepted.
	58	+	u.mu.Lock()
	59	+	t, ok := u.tokens[name]
	60	+	u.mu.Unlock()
	61	+	if ok && !refresh && u.now().Before(t.until) {
	62	+	return t.value, nil
	63	+	}
	64	+	a, err := url.Parse(u.auth)
	65	+	if err != nil {
	66	+	return "", err
	67	+	}
	68	+	q := a.Query()
	69	+	q.Set("service", "registry.docker.io")
	70	+	q.Set("scope", "repository:"+name+":pull")
	71	+	a.RawQuery = q.Encode()
	72	+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, a.String(), nil)
	73	+	if err != nil {
	74	+	return "", err
	75	+	}
	76	+	resp, err := u.client.Do(req)
	77	+	if err != nil {
	78	+	return "", err
	79	+	}
	80	+	defer func() { _ = resp.Body.Close() }()
	81	+	if resp.StatusCode != http.StatusOK {
	82	+	return "", upstreamError{resp.StatusCode}
	83	+	}
	84	+	var data struct {
	85	+	Token string `json:"token"`
	86	+	AccessToken string `json:"access_token"`
	87	+	Expires int64 `json:"expires_in"`
	88	+	}
	89	+	b, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20+1))
	90	+	if err != nil {
	91	+	return "", err
	92	+	}
	93	+	if len(b) > 1<<20 {
	94	+	return "", errors.New("registry token response too large")
	95	+	}
	96	+	if err = json.Unmarshal(b, &data); err != nil {
	97	+	return "", err
	98	+	}
	99	+	if data.Token == "" {
	100	+	data.Token = data.AccessToken
	101	+	}
	102	+	if data.Token == "" {
	103	+	return "", errors.New("empty registry token")
	104	+	}
	105	+	if data.Expires <= 0 {
	106	+	data.Expires = 60
	107	+	}
	108	+	if data.Expires > 86400 {
	109	+	data.Expires = 86400
	110	+	}
	111	+	u.mu.Lock()
	112	+	defer u.mu.Unlock()
	113	+	if len(u.tokens) >= 1024 {
	114	+	clear(u.tokens)
	115	+	}
	116	+	u.tokens[name] = bearer{data.Token, u.now().Add(time.Duration(data.Expires) * time.Second)}
	117	+	return data.Token, nil
	118	+	}
	119	+
	120	+	// request is a metadata request (manifests, blob HEAD) bounded by ctx. Manifests
	121	+	// use the client that never follows redirects; a blob HEAD uses the blob client,
	122	+	// because Docker Hub answers it with a redirect to its CDN.
	123	+	func (u upstream) request(ctx context.Context, method, name, kind, ref, accept string) (http.Response, error) {
	124	+	client := u.client
	125	+	if kind == "blobs" {
	126	+	client = u.blobClient
	127	+	}
	128	+	for attempt := 0; attempt < 2; attempt++ {
	129	+	tok, err := u.token(ctx, name, attempt > 0)
	130	+	if err != nil {
	131	+	return nil, err
	132	+	}
	133	+	req, err := http.NewRequestWithContext(ctx, method, u.base+"/v2/"+name+"/"+kind+"/"+ref, nil)
	134	+	if err != nil {
	135	+	return nil, err
	136	+	}
	137	+	req.Header.Set("Authorization", "Bearer "+tok)
	138	+	if accept != "" {
	139	+	req.Header.Set("Accept", accept)
	140	+	}
	141	+	// Do not forward guest Authorization, cookies, query parameters or Range.
	142	+	resp, err := client.Do(req)
	143	+	if err != nil {
	144	+	return nil, err
	145	+	}
	146	+	if resp.StatusCode == http.StatusOK {
	147	+	return resp, nil
	148	+	}
	149	+	_ = resp.Body.Close()
	150	+	if (resp.StatusCode == http.StatusUnauthorized \|\| resp.StatusCode == http.StatusForbidden) && attempt == 0 {
	151	+	continue
	152	+	}
	153	+	return nil, upstreamError{resp.StatusCode}
	154	+	}
	155	+	return nil, upstreamError{http.StatusUnauthorized}
	156	+	}

A registry_setup_test.go +43 -0

@@ -0,0 +1,43 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+	package main
	3	+
	4	+	import (
	5	+	"bnema.dev/ermite-runner/internal/sandbox"
	6	+	"context"
	7	+	"testing"
	8	+	)
	9	+
	10	+	func TestRegistryConfiguration(t *testing.T) {
	11	+	t.Setenv(envCache, "1")
	12	+	t.Setenv("ERMITE_RUNNER_ACTIONS_CACHE_MAX", "0")
	13	+	for _, tc := range []struct {
	14	+	registry, cache string
	15	+	on bool
	16	+	}{{"", "1M", true}, {"1M", "2M", true}, {"0", "1M", false}, {"0G", "1M", false}, {" 0M ", "1M", false}, {"00", "1M", false}, {"0GiB", "1M", false}, {"10M", "1M", true}} {
	17	+	t.Setenv(envRegistryMax, tc.registry)
	18	+	t.Setenv("ERMITE_RUNNER_CACHE_MAX", tc.cache)
	19	+	cfg := sandbox.Config{StateDir: t.TempDir()}
	20	+	svc, _, err := setupCache(context.Background(), &cfg)
	21	+	if err != nil {
	22	+	t.Fatal(err)
	23	+	}
	24	+	if svc.RegistryEnabled() != tc.on {
	25	+	t.Fatalf("%+v: registry=%v", tc, svc.RegistryEnabled())
	26	+	}
	27	+	_ = svc.CloseAll()
	28	+	}
	29	+	for _, bad := range []string{"invalid", "G", "-1G"} {
	30	+	t.Setenv(envRegistryMax, bad)
	31	+	cfg := sandbox.Config{StateDir: t.TempDir()}
	32	+	if _, _, err := setupCache(context.Background(), &cfg); err == nil {
	33	+	t.Fatalf("%q accepted", bad)
	34	+	}
	35	+	}
	36	+	t.Setenv(envCache, "0")
	37	+	t.Setenv(envRegistryMax, "invalid")
	38	+	cfg := sandbox.Config{StateDir: t.TempDir()}
	39	+	svc, _, err := setupCache(context.Background(), &cfg)
	40	+	if err != nil \|\| svc != nil {
	41	+	t.Fatalf("cache off: %v %v", svc, err)
	42	+	}
	43	+	}

@@ -0,0 +1,43 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+	package main
	3	+
	4	+	import (
	5	+	"bnema.dev/ermite-runner/internal/sandbox"
	6	+	"context"
	7	+	"testing"
	8	+	)
	9	+
	10	+	func TestRegistryConfiguration(t *testing.T) {
	11	+	t.Setenv(envCache, "1")
	12	+	t.Setenv("ERMITE_RUNNER_ACTIONS_CACHE_MAX", "0")
	13	+	for _, tc := range []struct {
	14	+	registry, cache string
	15	+	on bool
	16	+	}{{"", "1M", true}, {"1M", "2M", true}, {"0", "1M", false}, {"0G", "1M", false}, {" 0M ", "1M", false}, {"00", "1M", false}, {"0GiB", "1M", false}, {"10M", "1M", true}} {
	17	+	t.Setenv(envRegistryMax, tc.registry)
	18	+	t.Setenv("ERMITE_RUNNER_CACHE_MAX", tc.cache)
	19	+	cfg := sandbox.Config{StateDir: t.TempDir()}
	20	+	svc, _, err := setupCache(context.Background(), &cfg)
	21	+	if err != nil {
	22	+	t.Fatal(err)
	23	+	}
	24	+	if svc.RegistryEnabled() != tc.on {
	25	+	t.Fatalf("%+v: registry=%v", tc, svc.RegistryEnabled())
	26	+	}
	27	+	_ = svc.CloseAll()
	28	+	}
	29	+	for _, bad := range []string{"invalid", "G", "-1G"} {
	30	+	t.Setenv(envRegistryMax, bad)
	31	+	cfg := sandbox.Config{StateDir: t.TempDir()}
	32	+	if _, _, err := setupCache(context.Background(), &cfg); err == nil {
	33	+	t.Fatalf("%q accepted", bad)
	34	+	}
	35	+	}
	36	+	t.Setenv(envCache, "0")
	37	+	t.Setenv(envRegistryMax, "invalid")
	38	+	cfg := sandbox.Config{StateDir: t.TempDir()}
	39	+	svc, _, err := setupCache(context.Background(), &cfg)
	40	+	if err != nil \|\| svc != nil {
	41	+	t.Fatalf("cache off: %v %v", svc, err)
	42	+	}
	43	+	}