Merge pull request 'feat(externals): provide Node.js to jobs outside the image' (#10) from feat/node-externals into main

commit: 6da746a1b9202c9056c6d1d156108872c9f31717
author: bnema · 9d ago · 2026-10-13 10:23 UTC
merge of: 93b85fe + ea3c7aa · diff against the first parent
signature: SSH UnverifiedSHA256:VOCA5lYARHcbKDZ7GhTV45VmcjWnJ6ZQIBazj17LivA The key is unknown

20 files · +2586 -29

M README.md +25 -3


@@ -16,7 +16,7 @@ Both modes run the same executable, the runner's own, attached read-only to
16	16		### vm
17	17
18	18		- Firecracker is driven through its HTTP API on a unix socket with `net/http`, not through `firecracker-go-sdk` (its last release, v1.0.0 from 2022, predates Firecracker 1.17 and pulls CNI, go-openapi and netlink for a handful of PUT requests).
19		-	- Disks: the runner executable (read-only boot root, ext4 built with `mkfs.ext4 -d`, cached by the executable's hash), the shared read-only base image, and a per-job sparse scratch disk (`ERMITE_RUNNER_JOB_DISK`). The guest init builds an overlayfs root (lower = base, upper = scratch) and keeps dockerd's data root on the scratch disk. The scratch disk and `eth0` are rate limited (defaults 400 MiB/s and 20000 IOPS, 1 Gbit/s each way; `ERMITE_RUNNER_JOB_DISK_BANDWIDTH` in MiB/s, `_JOB_DISK_IOPS`, `_JOB_NET_BANDWIDTH` in Mbit/s; 0 disables).
	19	+	- Disks: the runner executable (read-only boot root, ext4 built with `mkfs.ext4 -d`, cached by the executable's hash), the shared read-only base image, a per-job sparse scratch disk (`ERMITE_RUNNER_JOB_DISK`) and, when there are externals (below), a fourth read-only disk with the host's Node.js builds, cached by their content. The guest init builds an overlayfs root (lower = base, upper = scratch), mounts the externals disk (`/dev/vdd`) at `/home/runner/externals` and keeps dockerd's data root on the scratch disk. The scratch disk and `eth0` are rate limited (defaults 400 MiB/s and 20000 IOPS, 1 Gbit/s each way; `ERMITE_RUNNER_JOB_DISK_BANDWIDTH` in MiB/s, `_JOB_DISK_IOPS`, `_JOB_NET_BANDWIDTH` in Mbit/s; 0 disables).
20	20		- The jailer runs each VM as uid/gid `ERMITE_RUNNER_UID_BASE` + slot in its own chroot under the state directory, in a new PID namespace and in a cgroup v2 with `memory.max` (guest memory + 256 MiB for the VMM) and `pids.max`. The kernel, the base image and the state directory must be on the same filesystem (files are hard-linked into the jail) and the kernel and base image must be world-readable.
21	21		- The state directory and every parent of it must be owned by root and not writable by group or others. The jailer runs as root and builds the chroots under `<state dir>/jail`; the driver checks that `jail` itself is a real directory, root-owned and not group/other-writable, but it does not walk the parents: a user who can rename or replace a parent directory can still redirect root's file operations. `/var/lib/ermite-runner` (the default, created by the unit's `StateDirectory`) is fine; do not point `ERMITE_RUNNER_STATE_DIR` into a home directory or `/tmp`.
22	22		- One TAP (`ertap<slot>`) per VM with a /30 from `ERMITE_RUNNER_NETWORK` (default `10.200.0.0/16`), vsock for the executor stream, IPv6 disabled in the guest.

@@ -31,6 +31,26 @@ Both modes run the same executable, the runner's own, attached read-only to
31	31		- Per job: a bridge network (`erbr<slot>`), the scratch disk, a Unix socket in a bind-mounted per-job directory of the state directory (the job sees only that directory, not the scratch file). Every Docker object carries the label `dev.bnema.ermite-runner.job`; leftovers are reaped by label at start. The state directory path must keep the socket path under 107 bytes.
32	32		- The Docker daemon must run on the same host as the runner (host paths are bind-mounted).
33	33
	34	+	### Externals (Node.js)
	35	+
	36	+	The host provides Node.js to the jobs outside the image, like its own executable, so that a runner upgrade or a new Node.js version never needs an image rebuild. The builds are the ones of `actions/runner` (`Misc/externals.sh`), pinned in `internal/externals/table.go` with the SHA-256 of each archive:
	37	+
	38	+	\| Name \| Node.js \| Source \| Architectures \|
	39	+	\| --- \| --- \| --- \| --- \|
	40	+	\| `node20` \| 20.20.2 \| `nodejs.org/dist/v20.20.2/node-v20.20.2-linux-<arch>.tar.gz` \| x64, arm64 \|
	41	+	\| `node24` \| 24.21.0 \| `nodejs.org/dist/v24.21.0/node-v24.21.0-linux-<arch>.tar.gz` \| x64, arm64 \|
	42	+	\| `node20_alpine` \| 20.20.2 \| `github.com/actions/alpine_nodejs` release `v20.20.2` \| x64 \|
	43	+	\| `node24_alpine` \| 24.21.0 \| `github.com/actions/alpine_nodejs` release `v24.21.0` \| x64 \|
	44	+
	45	+	- Download. `ermite-runner run` (and `sandbox-check`) fetch what is missing at start, into `$ERMITE_RUNNER_STATE_DIR/externals/<name>-<version>-<arch>/`, in the background; the start waits for them at most 2 minutes (a runner with a network normally takes its first job with everything in place, one without never waits long). The archive is the `.tar.gz` (the Go standard library has no xz); its SHA-256 is checked before anything is unpacked. Redirects must stay on https.
	46	+	- Extraction keeps the top directory out and refuses absolute names, `..`, devices and fifos, symbolic links that are absolute or climb out of the tree, entries below a link, hard links to anything but an unpacked file, and archives over 200,000 entries or 2 GiB; writes go through `os.Root`. The tree is built in a temporary directory and renamed into place, so a directory under its final name is always complete.
	47	+	- Only pinned versions are kept: at every start everything else in `externals/` (other versions, other architectures, interrupted downloads) is removed.
	48	+	- Layout in the sandbox (GitHub's): `/home/runner/externals/node20/bin/node`, `/node24/bin/node`, and the alpine ones for job containers, read-only. In `container` mode they are read-only bind mounts of the host directories; in `vm` mode one ext4 disk (`mkfs.ext4 -d`) named after the content of the set (`cache/extdisk-<key>.ext4`), so a new Node.js builds a new disk and leaves the executable's disk and the base image alone. Superseded disks are removed an hour after the last build. The set is read at every job start: a download that finishes later is used by the next job.
	49	+	- When a build cannot be provided (offline, HTTP error, wrong checksum, unsafe archive, no build for the architecture) the runner still runs: it logs why and attaches in its place a directory with the file `UNAVAILABLE` (the reason) and a `bin/node` script, `cat "$(dirname "$0")/../UNAVAILABLE" >&2; exit 1`, so a job that needs the tool fails with a message such as `node24 (Node.js 24.21.0) is not available on this runner: download …: SHA-256 mismatch …` rather than `No such file or directory`. These directories (`externals/.unavailable/<name>-<reason hash>`) are built once, complete, and renamed into place; a file is never rewritten in place, so a reader (the build of a VM's externals disk, a job running the stub) never sees a half-written one. A directory handed out to a sandbox in the last hour is kept by the pruning, since a running container may have it bind-mounted.
	50	+	- Retries. A download that failed for a reason that may pass (network error, HTTP status) is retried every 5 minutes by a timer, whether jobs come or not, until the runner stops. A verdict (wrong checksum, unsafe archive, no build for the architecture) is not retried before the next start.
	51	+	- Archive layouts. The nodejs.org archives have one top directory (`node-vX-linux-x64/`), which is dropped; the `actions/alpine_nodejs` archives start at `./bin/node` and are unpacked as they are. The alpine checksums are the digests of the GitHub release assets.
	52	+	- `ERMITE_RUNNER_NODE=0` turns all of this off: nothing is downloaded or attached.
	53	+
34	54		### One runner per host
35	55
36	56		Reaping removes every process, device, firewall rule and Docker object the runner owns, so two runners on one host would destroy each other's jobs. Both modes take an exclusive lock at a fixed host path, `/run/ermite-runner.lock` (`ERMITE_RUNNER_LOCK_FILE` overrides it, for tests), and refuse to start when it is held, whatever their state directories or isolation modes. The kernel drops the lock when the process dies. A runner in a container or in another mount namespace must share that path (or it is not protected): do not run it so.

@@ -209,8 +229,8 @@ mounts, nftables in the host namespace, Docker.
209	229
210	230		A job runs in the image named by `ERMITE_RUNNER_IMAGE` (container: an OCI
211	231		reference; vm: the directory with `vmlinux` and `base.ext4`, both built from
212		-	`images/ubuntu-24.04/Dockerfile`). The runner binary is attached to the sandbox
213		-	by the host and is never part of the image. What the executor needs from the
	232	+	`images/ubuntu-24.04/Dockerfile`). The runner binary and Node.js (see "Externals") are attached to the sandbox
	233	+	by the host and are never part of the image. What the executor needs from the
214	234		image:
215	235
216	236		- a user `runner` with uid and gid 1001 and the home `/home/runner`; the

@@ -261,6 +281,7 @@ the moment it starts.
261	281		\| `ERMITE_RUNNER_JOB_PIDS` \| `32768` \| container process limit \|
262	282		\| `ERMITE_RUNNER_DENY_PRIVILEGED` \| `0` \| `1` refuses container mode without sysbox \|
263	283		\| `ERMITE_RUNNER_GUEST_FORGE` \| `0` \| `1` lets sandboxes reach the host and port of `ERMITE_RUNNER_URL` on the host (staging only, see "Guest forge access") \|
	284	+	\| `ERMITE_RUNNER_NODE` \| `1` \| `0` turns the Node.js externals off (see "Externals") \|
264	285		\| `ERMITE_RUNNER_LOCK_FILE` \| `/run/ermite-runner.lock` \| host-wide single-instance lock (tests only) \|
265	286		\| `ERMITE_RUNNER_JOB_DISK_BANDWIDTH`, `_JOB_DISK_IOPS`, `_JOB_NET_BANDWIDTH` \| `400`, `20000`, `1000` \| VM rate limits in MiB/s, IOPS and Mbit/s; 0 disables \|
266	287		\| `ERMITE_RUNNER_UID_BASE` \| `900000` \| jailer uid pool start \|

@@ -317,6 +338,7 @@ ERMITE_RUNNER_IT_E2E=1 ERMITE_RUNNER_TEST_IMAGE=ermite-runner-ubuntu-24.04:dev G
317	338		- `TestContainerRunsAJobAsTheRunnerUser` and `TestContainerCancelStopsTheJobAndOrphans` drive a job over the stream: steps run as `runner` (1001), file ownership, `GITHUB_PATH` and `GITHUB_ENV`, `python`, cancellation by SIGINT, the `always()` step, and the orphan cleanup of "Complete job".
318	339		- `TestChainCorpus` runs the whole chain: a fake `RunnerService` (`internal/host/hosttest`, which checks the offsets, limits and tokens of `runner.proto`) hands out 22 jobs of the ermite-conformance corpus, the real host loop starts them in containers (two at a time), and what the fake stored (steps, conclusions, outputs, log lines) is compared with the recordings made on github.com. It reads `ERMITE_CONFORMANCE_DIR` (default: a sibling checkout of ermite-conformance) and never writes to it.
319	340		- `TestVMRunsAJob` (`internal/sandbox/integration`, run by `scripts/it-vm-userns.sh`) does the first of these in a micro-VM.
	341	+	- `TestVMNodeExternals` (same place) downloads the pinned `node24` (or takes it from `ERMITE_RUNNER_IT_NODE_DIR`, a directory with `node-v24.21.0-linux-x64.tar.gz`, for an offline host; the checksum and the extraction are the real ones), boots a micro-VM and runs `/home/runner/externals/node24/bin/node -e 'console.log(process.version)'` in it, checking that the mount is read-only.
320	342
321	343		Checked as root on a real KVM host (Arch Linux, kernel 6.18 LTS, Firecracker and jailer 1.17, rootful podman as the Docker API): the jailer (uid drop, chroot, `memory.max` and `pids.max` cgroups, PID namespace), the VM and container integration scenarios, the firewall policy and NAT, the nested `dockerd` on the loop-mounted scratch disk (full disk gives `ENOSPC`), and `internal/e2e` including `TestChainCorpus`. Not checked: sysbox, and a real Docker Engine (podman's compatibility API is not a drop-in, see below).
322	344

 ### vm
 - Firecracker is driven through its HTTP API on a unix socket with `net/http`, not through `firecracker-go-sdk` (its last release, v1.0.0 from 2022, predates Firecracker 1.17 and pulls CNI, go-openapi and netlink for a handful of PUT requests).
-- Disks: the runner executable (read-only boot root, ext4 built with `mkfs.ext4 -d`, cached by the executable's hash), the shared read-only base image, and a per-job sparse scratch disk (`ERMITE_RUNNER_JOB_DISK`). The guest init builds an overlayfs root (lower = base, upper = scratch) and keeps dockerd's data root on the scratch disk. The scratch disk and `eth0` are rate limited (defaults 400 MiB/s and 20000 IOPS, 1 Gbit/s each way; `ERMITE_RUNNER_JOB_DISK_BANDWIDTH` in MiB/s, `_JOB_DISK_IOPS`, `_JOB_NET_BANDWIDTH` in Mbit/s; 0 disables).
+- Disks: the runner executable (read-only boot root, ext4 built with `mkfs.ext4 -d`, cached by the executable's hash), the shared read-only base image, a per-job sparse scratch disk (`ERMITE_RUNNER_JOB_DISK`) and, when there are externals (below), a fourth read-only disk with the host's Node.js builds, cached by their content. The guest init builds an overlayfs root (lower = base, upper = scratch), mounts the externals disk (`/dev/vdd`) at `/home/runner/externals` and keeps dockerd's data root on the scratch disk. The scratch disk and `eth0` are rate limited (defaults 400 MiB/s and 20000 IOPS, 1 Gbit/s each way; `ERMITE_RUNNER_JOB_DISK_BANDWIDTH` in MiB/s, `_JOB_DISK_IOPS`, `_JOB_NET_BANDWIDTH` in Mbit/s; 0 disables).
 - The jailer runs each VM as uid/gid `ERMITE_RUNNER_UID_BASE` + slot in its own chroot under the state directory, in a new PID namespace and in a cgroup v2 with `memory.max` (guest memory + 256 MiB for the VMM) and `pids.max`. The kernel, the base image and the state directory must be on the same filesystem (files are hard-linked into the jail) and the kernel and base image must be world-readable.
 - **The state directory and every parent of it must be owned by root and not writable by group or others.** The jailer runs as root and builds the chroots under `<state dir>/jail`; the driver checks that `jail` itself is a real directory, root-owned and not group/other-writable, but it does not walk the parents: a user who can rename or replace a parent directory can still redirect root's file operations. `/var/lib/ermite-runner` (the default, created by the unit's `StateDirectory`) is fine; do not point `ERMITE_RUNNER_STATE_DIR` into a home directory or `/tmp`.
 - One TAP (`ertap<slot>`) per VM with a /30 from `ERMITE_RUNNER_NETWORK` (default `10.200.0.0/16`), vsock for the executor stream, IPv6 disabled in the guest.
 - Per job: a bridge network (`erbr<slot>`), the scratch disk, a Unix socket in a bind-mounted per-job directory of the state directory (the job sees only that directory, not the scratch file). Every Docker object carries the label `dev.bnema.ermite-runner.job`; leftovers are reaped by label at start. The state directory path must keep the socket path under 107 bytes.
 - The Docker daemon must run on the same host as the runner (host paths are bind-mounted).
+### Externals (Node.js)
+The host provides Node.js to the jobs **outside the image**, like its own executable, so that a runner upgrade or a new Node.js version never needs an image rebuild. The builds are the ones of `actions/runner` (`Misc/externals.sh`), pinned in `internal/externals/table.go` with the SHA-256 of each archive:
+| Name | Node.js | Source | Architectures |
+| --- | --- | --- | --- |
+| `node20` | 20.20.2 | `nodejs.org/dist/v20.20.2/node-v20.20.2-linux-<arch>.tar.gz` | x64, arm64 |
+| `node24` | 24.21.0 | `nodejs.org/dist/v24.21.0/node-v24.21.0-linux-<arch>.tar.gz` | x64, arm64 |
+| `node20_alpine` | 20.20.2 | `github.com/actions/alpine_nodejs` release `v20.20.2` | x64 |
+| `node24_alpine` | 24.21.0 | `github.com/actions/alpine_nodejs` release `v24.21.0` | x64 |
+- **Download.** `ermite-runner run` (and `sandbox-check`) fetch what is missing at start, into `$ERMITE_RUNNER_STATE_DIR/externals/<name>-<version>-<arch>/`, in the background; the start waits for them at most 2 minutes (a runner with a network normally takes its first job with everything in place, one without never waits long). The archive is the `.tar.gz` (the Go standard library has no xz); its SHA-256 is checked **before** anything is unpacked. Redirects must stay on https.
+- **Extraction** keeps the top directory out and refuses absolute names, `..`, devices and fifos, symbolic links that are absolute or climb out of the tree, entries below a link, hard links to anything but an unpacked file, and archives over 200,000 entries or 2 GiB; writes go through `os.Root`. The tree is built in a temporary directory and renamed into place, so a directory under its final name is always complete.
+- **Only pinned versions are kept**: at every start everything else in `externals/` (other versions, other architectures, interrupted downloads) is removed.
+- **Layout in the sandbox** (GitHub's): `/home/runner/externals/node20/bin/node`, `/node24/bin/node`, and the alpine ones for job containers, read-only. In `container` mode they are read-only bind mounts of the host directories; in `vm` mode one ext4 disk (`mkfs.ext4 -d`) named after the content of the set (`cache/extdisk-<key>.ext4`), so a new Node.js builds a new disk and leaves the executable's disk and the base image alone. Superseded disks are removed an hour after the last build. The set is read at every job start: a download that finishes later is used by the next job.
+- **When a build cannot be provided** (offline, HTTP error, wrong checksum, unsafe archive, no build for the architecture) the runner **still runs**: it logs why and attaches in its place a directory with the file `UNAVAILABLE` (the reason) and a `bin/node` script, `cat "$(dirname "$0")/../UNAVAILABLE" >&2; exit 1`, so a job that needs the tool fails with a message such as `node24 (Node.js 24.21.0) is not available on this runner: download …: SHA-256 mismatch …` rather than `No such file or directory`. These directories (`externals/.unavailable/<name>-<reason hash>`) are built once, complete, and renamed into place; a file is never rewritten in place, so a reader (the build of a VM's externals disk, a job running the stub) never sees a half-written one. A directory handed out to a sandbox in the last hour is kept by the pruning, since a running container may have it bind-mounted.
+- **Retries.** A download that failed for a reason that may pass (network error, HTTP status) is retried every 5 minutes by a timer, whether jobs come or not, until the runner stops. A verdict (wrong checksum, unsafe archive, no build for the architecture) is not retried before the next start.
+- **Archive layouts.** The nodejs.org archives have one top directory (`node-vX-linux-x64/`), which is dropped; the `actions/alpine_nodejs` archives start at `./bin/node` and are unpacked as they are. The alpine checksums are the digests of the GitHub release assets.
+- `ERMITE_RUNNER_NODE=0` turns all of this off: nothing is downloaded or attached.
 ### One runner per host
 Reaping removes every process, device, firewall rule and Docker object the runner owns, so two runners on one host would destroy each other's jobs. Both modes take an exclusive lock at a fixed host path, `/run/ermite-runner.lock` (`ERMITE_RUNNER_LOCK_FILE` overrides it, for tests), and refuse to start when it is held, whatever their state directories or isolation modes. The kernel drops the lock when the process dies. A runner in a container or in another mount namespace must share that path (or it is not protected): do not run it so.
 A job runs in the image named by `ERMITE_RUNNER_IMAGE` (container: an OCI
 reference; vm: the directory with `vmlinux` and `base.ext4`, both built from
-`images/ubuntu-24.04/Dockerfile`). The runner binary is attached to the sandbox
+`images/ubuntu-24.04/Dockerfile`). The runner binary and Node.js (see "Externals") are attached to the sandbox
-by the host and is never part of the image. What the executor needs from the
+by the host and are never part of the image. What the executor needs from the
 image:
 - a user **`runner`** with uid and gid 1001 and the home `/home/runner`; the
 | `ERMITE_RUNNER_JOB_PIDS` | `32768` | container process limit |
 | `ERMITE_RUNNER_DENY_PRIVILEGED` | `0` | `1` refuses container mode without sysbox |
 | `ERMITE_RUNNER_GUEST_FORGE` | `0` | `1` lets sandboxes reach the host and port of `ERMITE_RUNNER_URL` on the host (staging only, see "Guest forge access") |
+| `ERMITE_RUNNER_NODE` | `1` | `0` turns the Node.js externals off (see "Externals") |
 | `ERMITE_RUNNER_LOCK_FILE` | `/run/ermite-runner.lock` | host-wide single-instance lock (tests only) |
 | `ERMITE_RUNNER_JOB_DISK_BANDWIDTH`, `_JOB_DISK_IOPS`, `_JOB_NET_BANDWIDTH` | `400`, `20000`, `1000` | VM rate limits in MiB/s, IOPS and Mbit/s; 0 disables |
 | `ERMITE_RUNNER_UID_BASE` | `900000` | jailer uid pool start |
 - `TestContainerRunsAJobAsTheRunnerUser` and `TestContainerCancelStopsTheJobAndOrphans` drive a job over the stream: steps run as `runner` (1001), file ownership, `GITHUB_PATH` and `GITHUB_ENV`, `python`, cancellation by SIGINT, the `always()` step, and the orphan cleanup of "Complete job".
 - `TestChainCorpus` runs the whole chain: a fake `RunnerService` (`internal/host/hosttest`, which checks the offsets, limits and tokens of `runner.proto`) hands out 22 jobs of the ermite-conformance corpus, the real host loop starts them in containers (two at a time), and what the fake stored (steps, conclusions, outputs, log lines) is compared with the recordings made on github.com. It reads `ERMITE_CONFORMANCE_DIR` (default: a sibling checkout of ermite-conformance) and never writes to it.
 - `TestVMRunsAJob` (`internal/sandbox/integration`, run by `scripts/it-vm-userns.sh`) does the first of these in a micro-VM.
+- `TestVMNodeExternals` (same place) downloads the pinned `node24` (or takes it from `ERMITE_RUNNER_IT_NODE_DIR`, a directory with `node-v24.21.0-linux-x64.tar.gz`, for an offline host; the checksum and the extraction are the real ones), boots a micro-VM and runs `/home/runner/externals/node24/bin/node -e 'console.log(process.version)'` in it, checking that the mount is read-only.
 Checked as root on a real KVM host (Arch Linux, kernel 6.18 LTS, Firecracker and jailer 1.17, rootful podman as the Docker API): the jailer (uid drop, chroot, `memory.max` and `pids.max` cgroups, PID namespace), the VM and container integration scenarios, the firewall policy and NAT, the nested `dockerd` on the loop-mounted scratch disk (full disk gives `ENOSPC`), and `internal/e2e` including `TestChainCorpus`. Not checked: sysbox, and a real Docker Engine (podman's compatibility API is not a drop-in, see below).

M host_cmd.go +28 -0


@@ -9,11 +9,13 @@ import (
9	9		"fmt"
10	10		"net"
11	11		"os"
	12	+	"time"
12	13
13	14		"github.com/bnema/zerowrap"
14	15
15	16		"bnema.dev/ermite-api/gen/ermite/ermiteconnect"
16	17
	18	+	"bnema.dev/ermite-runner/internal/externals"
17	19		"bnema.dev/ermite-runner/internal/host"
18	20		"bnema.dev/ermite-runner/internal/sandbox"
19	21		"bnema.dev/ermite-runner/internal/sandbox/drivers"

@@ -31,6 +33,31 @@ func loadHostConfig() (sandbox.Config, host.Config, error) {
31	33		return scfg, hcfg, err
32	34		}
33	35
	36	+	// externalsWait is how long the start of the runner waits for the Node.js
	37	+	// downloads. A runner with a working network is then ready for its first job; one
	38	+	// without goes on after the first failure (nothing waits for a dead network), and
	39	+	// the downloads that are still running when the time is up finish in the
	40	+	// background.
	41	+	const externalsWait = 2 * time.Minute
	42	+
	43	+	// setupExternals downloads the Node.js externals (README, "Externals") and hands
	44	+	// them to the sandbox drivers through scfg. It never fails the start: whatever
	45	+	// cannot be provided is reported to the jobs that need it.
	46	+	func setupExternals(ctx context.Context, scfg *sandbox.Config) {
	47	+	log := zerowrap.FromCtx(ctx)
	48	+	if !scfg.NodeExternals {
	49	+	log.Info().Msg("Node.js externals are off (" + sandbox.EnvNode + "=0)")
	50	+	return
	51	+	}
	52	+	m, ok := externals.New(scfg.StateDir)
	53	+	if !ok {
	54	+	log.Warn().Msg("no Node.js builds for this architecture: jobs get no externals")
	55	+	return
	56	+	}
	57	+	m.Start(ctx, externalsWait)
	58	+	scfg.Externals = m
	59	+	}
	60	+
34	61		func buildDriver(ctx context.Context, scfg sandbox.Config) (sandbox.Driver, error) {
35	62		return sandbox.Select(ctx, scfg, drivers.Factories(), sandbox.Prober{})
36	63		}

@@ -116,6 +143,7 @@ func runCmd(ctx context.Context) int {
116	143		scfg.GuestForge = gf
117	144		hcfg.GuestForgeOrigin = gf.Origin
118	145		}
	146	+	setupExternals(ctx, &scfg)
119	147		drv, err := buildDriver(ctx, scfg)
120	148		if err != nil {
121	149		fmt.Fprintln(os.Stderr, "ermite-runner:", err)


@@ -9,11 +9,13 @@ import (
9	"fmt"	9		"fmt"
10	"net"	10		"net"
11	"os"	11		"os"
		12	+	"time"
12		13
13	"github.com/bnema/zerowrap"	14		"github.com/bnema/zerowrap"
14		15
15	"bnema.dev/ermite-api/gen/ermite/ermiteconnect"	16		"bnema.dev/ermite-api/gen/ermite/ermiteconnect"
16		17
		18	+	"bnema.dev/ermite-runner/internal/externals"
17	"bnema.dev/ermite-runner/internal/host"	19		"bnema.dev/ermite-runner/internal/host"
18	"bnema.dev/ermite-runner/internal/sandbox"	20		"bnema.dev/ermite-runner/internal/sandbox"
19	"bnema.dev/ermite-runner/internal/sandbox/drivers"	21		"bnema.dev/ermite-runner/internal/sandbox/drivers"

@@ -31,6 +33,31 @@ func loadHostConfig() (sandbox.Config, host.Config, error) {
31	return scfg, hcfg, err	33		return scfg, hcfg, err
32	}	34		}
33		35
		36	+	// externalsWait is how long the start of the runner waits for the Node.js
		37	+	// downloads. A runner with a working network is then ready for its first job; one
		38	+	// without goes on after the first failure (nothing waits for a dead network), and
		39	+	// the downloads that are still running when the time is up finish in the
		40	+	// background.
		41	+	const externalsWait = 2 * time.Minute
		42	+
		43	+	// setupExternals downloads the Node.js externals (README, "Externals") and hands
		44	+	// them to the sandbox drivers through scfg. It never fails the start: whatever
		45	+	// cannot be provided is reported to the jobs that need it.
		46	+	func setupExternals(ctx context.Context, scfg *sandbox.Config) {
		47	+	log := zerowrap.FromCtx(ctx)
		48	+	if !scfg.NodeExternals {
		49	+	log.Info().Msg("Node.js externals are off (" + sandbox.EnvNode + "=0)")
		50	+	return
		51	+	}
		52	+	m, ok := externals.New(scfg.StateDir)
		53	+	if !ok {
		54	+	log.Warn().Msg("no Node.js builds for this architecture: jobs get no externals")
		55	+	return
		56	+	}
		57	+	m.Start(ctx, externalsWait)
		58	+	scfg.Externals = m
		59	+	}
		60	+
34	func buildDriver(ctx context.Context, scfg sandbox.Config) (sandbox.Driver, error) {	61		func buildDriver(ctx context.Context, scfg sandbox.Config) (sandbox.Driver, error) {
35	return sandbox.Select(ctx, scfg, drivers.Factories(), sandbox.Prober{})	62		return sandbox.Select(ctx, scfg, drivers.Factories(), sandbox.Prober{})
36	}	63		}

@@ -116,6 +143,7 @@ func runCmd(ctx context.Context) int {
116	scfg.GuestForge = gf	143		scfg.GuestForge = gf
117	hcfg.GuestForgeOrigin = gf.Origin	144		hcfg.GuestForgeOrigin = gf.Origin
118	}	145		}
		146	+	setupExternals(ctx, &scfg)
119	drv, err := buildDriver(ctx, scfg)	147		drv, err := buildDriver(ctx, scfg)
120	if err != nil {	148		if err != nil {
121	fmt.Fprintln(os.Stderr, "ermite-runner:", err)	149		fmt.Fprintln(os.Stderr, "ermite-runner:", err)

A internal/externals/externals_test.go +949 -0

@@ -0,0 +1,949 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package externals
	4	+
	5	+	import (
	6	+	"archive/tar"
	7	+	"bytes"
	8	+	"compress/gzip"
	9	+	"context"
	10	+	"crypto/sha256"
	11	+	"encoding/hex"
	12	+	"fmt"
	13	+	"net/http"
	14	+	"net/http/httptest"
	15	+	"net/url"
	16	+	"os"
	17	+	"os/exec"
	18	+	"path/filepath"
	19	+	"regexp"
	20	+	"strings"
	21	+	"sync"
	22	+	"sync/atomic"
	23	+	"syscall"
	24	+	"testing"
	25	+	"time"
	26	+
	27	+	"github.com/bnema/zerowrap"
	28	+
	29	+	"bnema.dev/ermite-runner/internal/sandbox"
	30	+	)
	31	+
	32	+	// ---- the table
	33	+
	34	+	func TestPinnedTable(t *testing.T) {
	35	+	hex64 := regexp.MustCompile(`^[0-9a-f]{64}$`)
	36	+	seen := map[string]bool{}
	37	+	for _, a := range Pinned {
	38	+	key := a.Name + "/" + a.Arch
	39	+	if seen[key] {
	40	+	t.Errorf("%s listed twice", key)
	41	+	}
	42	+	seen[key] = true
	43	+	if !strings.HasPrefix(a.URL, "https://") \|\| !strings.HasSuffix(a.URL, ".tar.gz") \|\| !strings.Contains(a.URL, "v"+a.Version+"/") {
	44	+	t.Errorf("%s: URL %q", key, a.URL)
	45	+	}
	46	+	if !hex64.MatchString(a.SHA256) {
	47	+	t.Errorf("%s: SHA-256 %q", key, a.SHA256)
	48	+	}
	49	+	if a.StripTop != strings.HasPrefix(a.URL, nodeDist) {
	50	+	t.Errorf("%s: StripTop %v for %s", key, a.StripTop, a.URL)
	51	+	}
	52	+	}
	53	+	// actions/runner's externals.sh: node20 and node24 for both architectures,
	54	+	// the alpine builds for x64.
	55	+	for _, k := range []string{"node20/x64", "node20/arm64", "node24/x64", "node24/arm64", "node20_alpine/x64", "node24_alpine/x64"} {
	56	+	if !seen[k] {
	57	+	t.Errorf("%s missing", k)
	58	+	}
	59	+	}
	60	+	for _, a := range Pinned {
	61	+	if strings.HasPrefix(a.Name, "node20") && a.Version != "20.20.2" \|\| strings.HasPrefix(a.Name, "node24") && a.Version != "24.21.0" {
	62	+	t.Errorf("%s pins %s", a.Name, a.Version)
	63	+	}
	64	+	}
	65	+	for _, a := range Pinned {
	66	+	if strings.HasSuffix(a.Name, "_alpine") && (a.Arch != "x64" \|\| !strings.HasPrefix(a.URL, alpineDist+"v"+a.Version+"/node-v"+a.Version+"-alpine-x64.tar.gz")) {
	67	+	t.Errorf("%s: %+v", a.Name, a)
	68	+	}
	69	+	}
	70	+	if got := len(ForArch(Pinned, "arm64")); got != 2 {
	71	+	t.Errorf("arm64 has %d artifacts", got)
	72	+	}
	73	+	}
	74	+
	75	+	func TestVerifySum(t *testing.T) {
	76	+	if err := VerifySum("AB12", "ab12"); err != nil {
	77	+	t.Error(err)
	78	+	}
	79	+	if err := VerifySum("ab12", "ab13"); err == nil \|\| !strings.Contains(err.Error(), "mismatch") {
	80	+	t.Errorf("got %v", err)
	81	+	}
	82	+	}
	83	+
	84	+	// ---- archives
	85	+
	86	+	type ent struct {
	87	+	name, link string
	88	+	typ byte
	89	+	mode int64
	90	+	body string
	91	+	}
	92	+
	93	+	func targz(t *testing.T, ents ...ent) []byte {
	94	+	t.Helper()
	95	+	var buf bytes.Buffer
	96	+	gz := gzip.NewWriter(&buf)
	97	+	tw := tar.NewWriter(gz)
	98	+	for _, e := range ents {
	99	+	typ := e.typ
	100	+	if typ == 0 {
	101	+	typ = tar.TypeReg
	102	+	}
	103	+	h := &tar.Header{Name: e.name, Linkname: e.link, Typeflag: typ, Mode: e.mode, Size: int64(len(e.body))}
	104	+	if typ != tar.TypeReg {
	105	+	h.Size = 0
	106	+	}
	107	+	if err := tw.WriteHeader(h); err != nil {
	108	+	t.Fatal(err)
	109	+	}
	110	+	if typ == tar.TypeReg {
	111	+	if _, err := tw.Write([]byte(e.body)); err != nil {
	112	+	t.Fatal(err)
	113	+	}
	114	+	}
	115	+	}
	116	+	if err := tw.Close(); err != nil {
	117	+	t.Fatal(err)
	118	+	}
	119	+	if err := gz.Close(); err != nil {
	120	+	t.Fatal(err)
	121	+	}
	122	+	return buf.Bytes()
	123	+	}
	124	+
	125	+	func nodeTree() []ent {
	126	+	return []ent{
	127	+	{name: "node-v1-linux-x64/", typ: tar.TypeDir, mode: 0o755},
	128	+	{name: "node-v1-linux-x64/bin/", typ: tar.TypeDir, mode: 0o755},
	129	+	{name: "node-v1-linux-x64/bin/node", mode: 0o755, body: "#!/bin/sh\necho v1\n"},
	130	+	{name: "node-v1-linux-x64/lib/node_modules/npm/bin/npm-cli.js", mode: 0o644, body: "x"},
	131	+	{name: "node-v1-linux-x64/bin/npm", typ: tar.TypeSymlink, link: "../lib/node_modules/npm/bin/npm-cli.js", mode: 0o777},
	132	+	{name: "node-v1-linux-x64/bin/nodejs", typ: tar.TypeLink, link: "node-v1-linux-x64/bin/node", mode: 0o755},
	133	+	}
	134	+	}
	135	+
	136	+	func extractBytes(t *testing.T, data []byte) (string, error) {
	137	+	t.Helper()
	138	+	dir := t.TempDir()
	139	+	arch := filepath.Join(dir, "a.tar.gz")
	140	+	if err := os.WriteFile(arch, data, 0o600); err != nil {
	141	+	t.Fatal(err)
	142	+	}
	143	+	dst := filepath.Join(dir, "out")
	144	+	if err := os.Mkdir(dst, 0o755); err != nil {
	145	+	t.Fatal(err)
	146	+	}
	147	+	return dst, extractTarGz(arch, dst, true)
	148	+	}
	149	+
	150	+	func TestExtractLayoutAndModes(t *testing.T) {
	151	+	dst, err := extractBytes(t, targz(t, nodeTree()...))
	152	+	if err != nil {
	153	+	t.Fatal(err)
	154	+	}
	155	+	st, err := os.Stat(filepath.Join(dst, "bin", "node"))
	156	+	if err != nil \|\| st.Mode().Perm() != 0o755 {
	157	+	t.Fatalf("bin/node: %v %v", st, err)
	158	+	}
	159	+	if st, _ := os.Stat(filepath.Join(dst, "lib/node_modules/npm/bin/npm-cli.js")); st == nil \|\| st.Mode().Perm() != 0o644 {
	160	+	t.Fatalf("npm-cli.js: %v", st)
	161	+	}
	162	+	if l, err := os.Readlink(filepath.Join(dst, "bin", "npm")); err != nil \|\| l != "../lib/node_modules/npm/bin/npm-cli.js" {
	163	+	t.Fatalf("npm link: %q %v", l, err)
	164	+	}
	165	+	a, _ := os.Stat(filepath.Join(dst, "bin", "node"))
	166	+	b, _ := os.Stat(filepath.Join(dst, "bin", "nodejs"))
	167	+	if !os.SameFile(a, b) {
	168	+	t.Fatal("hard link not recreated")
	169	+	}
	170	+	if !isReady(dst) {
	171	+	t.Fatal("not ready")
	172	+	}
	173	+	}
	174	+
	175	+	func TestModesDoNotDependOnTheUmask(t *testing.T) {
	176	+	old := syscall.Umask(0o077)
	177	+	defer syscall.Umask(old)
	178	+	dst, err := extractBytes(t, targz(t, nodeTree()...))
	179	+	if err != nil {
	180	+	t.Fatal(err)
	181	+	}
	182	+	for p, want := range map[string]os.FileMode{
	183	+	dst: 0o755, filepath.Join(dst, "bin"): 0o755, filepath.Join(dst, "lib/node_modules"): 0o755,
	184	+	filepath.Join(dst, "bin/node"): 0o755, filepath.Join(dst, "lib/node_modules/npm/bin/npm-cli.js"): 0o644,
	185	+	} {
	186	+	if st, err := os.Stat(p); err != nil \|\| st.Mode().Perm() != want {
	187	+	t.Errorf("%s: %v %v, want %v", p, st, err, want)
	188	+	}
	189	+	}
	190	+	m := &Manager{Dir: filepath.Join(t.TempDir(), "e"), Arch: "x64"}
	191	+	e := m.unavailable("node24", "why", false)
	192	+	for p, want := range map[string]os.FileMode{
	193	+	filepath.Dir(e.ext.Dir): 0o755, e.ext.Dir: 0o755, filepath.Join(e.ext.Dir, "bin"): 0o755,
	194	+	filepath.Join(e.ext.Dir, "bin/node"): 0o755, filepath.Join(e.ext.Dir, "UNAVAILABLE"): 0o644,
	195	+	} {
	196	+	if st, err := os.Stat(p); err != nil \|\| st.Mode().Perm() != want {
	197	+	t.Errorf("%s: %v %v, want %v", p, st, err, want)
	198	+	}
	199	+	}
	200	+	}
	201	+
	202	+	func TestExtractRefusesUnsafeArchives(t *testing.T) {
	203	+	top := "node-v1-linux-x64/"
	204	+	for name, ents := range map[string][]ent{
	205	+	"absolute path": {{name: "/etc/passwd", body: "x"}},
	206	+	"dot dot": {{name: top + "../../evil", body: "x"}},
	207	+	"dot dot in middle": {{name: top + "a/../../evil", body: "x"}},
	208	+	"backslash": {{name: top + `a\b`, body: "x"}},
	209	+	"absolute symlink": {{name: top + "l", typ: tar.TypeSymlink, link: "/etc"}},
	210	+	"escaping symlink": {{name: top + "l", typ: tar.TypeSymlink, link: "../../.."}},
	211	+	"escaping from deep": {{name: top + "a/b/l", typ: tar.TypeSymlink, link: "../../../x"}},
	212	+	"climb after a name": {{name: top + "a/l", typ: tar.TypeSymlink, link: "b/../../../x"}},
	213	+	"empty link target": {{name: top + "l", typ: tar.TypeSymlink, link: ""}},
	214	+	"write through link": {
	215	+	{name: top + "d", typ: tar.TypeSymlink, link: "."},
	216	+	{name: top + "d/file", body: "x"},
	217	+	},
	218	+	"write through dir link": {
	219	+	{name: top + "sub/", typ: tar.TypeDir},
	220	+	{name: top + "sub/up", typ: tar.TypeSymlink, link: ".."},
	221	+	{name: top + "sub/up/file", body: "x"},
	222	+	},
	223	+	"hard link outside": {{name: top + "h", typ: tar.TypeLink, link: "/etc/passwd"}},
	224	+	"hard link dot dot": {{name: top + "h", typ: tar.TypeLink, link: top + "../x"}},
	225	+	"hard link to top": {{name: top + "h", typ: tar.TypeLink, link: top}},
	226	+	"hard link to a link": {
	227	+	{name: top + "l", typ: tar.TypeSymlink, link: "x"},
	228	+	{name: top + "h", typ: tar.TypeLink, link: top + "l"},
	229	+	},
	230	+	"device": {{name: top + "dev", typ: tar.TypeChar}},
	231	+	"fifo": {{name: top + "fifo", typ: tar.TypeFifo}},
	232	+	"duplicate": {{name: top + "f", body: "1"}, {name: top + "f", body: "2"}},
	233	+	} {
	234	+	t.Run(name, func(t *testing.T) {
	235	+	dst, err := extractBytes(t, targz(t, ents...))
	236	+	if err == nil {
	237	+	t.Fatal("accepted")
	238	+	}
	239	+	// Nothing was written outside the destination, whatever the order.
	240	+	parent := filepath.Dir(dst)
	241	+	if _, err := os.Stat(filepath.Join(parent, "evil")); err == nil {
	242	+	t.Fatal("wrote outside")
	243	+	}
	244	+	if _, err := os.Stat(filepath.Join(dst, "..", "file")); err == nil {
	245	+	t.Fatal("wrote outside through a link")
	246	+	}
	247	+	})
	248	+	}
	249	+	}
	250	+
	251	+	func TestExtractRefusesGarbageAndTooMany(t *testing.T) {
	252	+	if _, err := extractBytes(t, []byte("not gzip")); err == nil {
	253	+	t.Error("garbage accepted")
	254	+	}
	255	+	var ents []ent
	256	+	for i := 0; i < maxEntries+1; i++ {
	257	+	ents = append(ents, ent{name: "t/d/", typ: tar.TypeDir})
	258	+	}
	259	+	if _, err := extractBytes(t, targz(t, ents...)); err == nil \|\| !strings.Contains(err.Error(), "entries") {
	260	+	t.Errorf("entry limit: %v", err)
	261	+	}
	262	+	}
	263	+
	264	+	func TestLinkTargetRule(t *testing.T) {
	265	+	for _, c := range []struct {
	266	+	name, target string
	267	+	ok bool
	268	+	}{
	269	+	{"bin/npm", "../lib/x", true},
	270	+	{"bin/npm", "../../x", false}, // one above the tree
	271	+	{"lib/a/b/l", "../../../x", true},
	272	+	{"lib/a/b/l", "../../../../x", false},
	273	+	{"l", "bin/node", true},
	274	+	{"l", "../x", false},
	275	+	{"l", "/abs", false},
	276	+	{"a/l", "b/../c", false},
	277	+	} {
	278	+	if err := checkLinkTarget(c.name, c.target); (err == nil) != c.ok {
	279	+	t.Errorf("%s -> %s: %v", c.name, c.target, err)
	280	+	}
	281	+	}
	282	+	}
	283	+
	284	+	// ---- the manager
	285	+
	286	+	type fixture struct {
	287	+	srv *httptest.Server
	288	+	hits atomic.Int64
	289	+	archive []byte
	290	+	sum string
	291	+	archiveAlpine []byte // no top directory, like actions/alpine_nodejs
	292	+	sumAlpine string
	293	+	table []Artifact
	294	+	dir string
	295	+	m *Manager
	296	+	}
	297	+
	298	+	func newFixture(t testing.T) fixture {
	299	+	t.Helper()
	300	+	f := &fixture{}
	301	+	f.archive = targz(t, nodeTree()...)
	302	+	s := sha256.Sum256(f.archive)
	303	+	f.sum = hex.EncodeToString(s[:])
	304	+	f.archiveAlpine = targz(t,
	305	+	ent{name: "./bin/", typ: tar.TypeDir, mode: 0o755},
	306	+	ent{name: "./bin/node", mode: 0o755, body: "#!/bin/sh\necho alpine\n"},
	307	+	ent{name: "./LICENSE", mode: 0o644, body: "x"})
	308	+	sa := sha256.Sum256(f.archiveAlpine)
	309	+	f.sumAlpine = hex.EncodeToString(sa[:])
	310	+	f.srv = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	311	+	f.hits.Add(1)
	312	+	switch {
	313	+	case strings.HasSuffix(r.URL.Path, "/alpine.tar.gz"):
	314	+	_, _ = w.Write(f.archiveAlpine)
	315	+	case strings.HasSuffix(r.URL.Path, "-linux-x64.tar.gz"):
	316	+	_, _ = w.Write(f.archive)
	317	+	case strings.HasSuffix(r.URL.Path, "tampered.tar.gz"):
	318	+	_, _ = w.Write(append([]byte(nil), f.archive[:len(f.archive)-1]...))
	319	+	default:
	320	+	http.NotFound(w, r)
	321	+	}
	322	+	}))
	323	+	t.Cleanup(f.srv.Close)
	324	+	f.dir = filepath.Join(t.TempDir(), "externals")
	325	+	f.table = []Artifact{
	326	+	{Name: "node20", Version: "1.0.0", Arch: "x64", URL: f.srv.URL + "/v1.0.0/node-v1.0.0-linux-x64.tar.gz", SHA256: f.sum, StripTop: true},
	327	+	{Name: "node24", Version: "2.0.0", Arch: "x64", URL: f.srv.URL + "/v2.0.0/node-v2.0.0-linux-x64.tar.gz", SHA256: f.sum, StripTop: true},
	328	+	{Name: "node20_alpine", Version: "1.0.0", Arch: "x64", URL: f.srv.URL + "/v1.0.0/alpine.tar.gz", SHA256: f.sumAlpine},
	329	+	}
	330	+	f.m = &Manager{Dir: f.dir, Arch: "x64", Table: f.table, Client: f.srv.Client()}
	331	+	return f
	332	+	}
	333	+
	334	+	func byName(set []sandbox.External) map[string]sandbox.External {
	335	+	m := map[string]sandbox.External{}
	336	+	for _, e := range set {
	337	+	m[e.Name] = e
	338	+	}
	339	+	return m
	340	+	}
	341	+
	342	+	func TestSyncDownloadsVerifiesAndReuses(t *testing.T) {
	343	+	f := newFixture(t)
	344	+	if err := f.m.Sync(context.Background()); err != nil {
	345	+	t.Fatal(err)
	346	+	}
	347	+	set := byName(f.m.Externals())
	348	+	n20, n24, al := set["node20"], set["node24"], set["node20_alpine"]
	349	+	if n20.Unavailable \|\| n24.Unavailable \|\| n20.Dir != filepath.Join(f.dir, "node20-1.0.0-x64") {
	350	+	t.Fatalf("set: %+v", set)
	351	+	}
	352	+	if n20.ID == n24.ID {
	353	+	t.Error("two tools share an ID")
	354	+	}
	355	+	if b, err := os.ReadFile(filepath.Join(n24.Dir, "bin", "node")); err != nil \|\| !strings.Contains(string(b), "echo v1") {
	356	+	t.Fatalf("node24: %q %v", b, err)
	357	+	}
	358	+	// The alpine archive has no top directory: bin/node is at its root.
	359	+	if al.Unavailable {
	360	+	t.Fatalf("alpine: %+v", al)
	361	+	}
	362	+	if b, err := os.ReadFile(filepath.Join(al.Dir, "bin", "node")); err != nil \|\| !strings.Contains(string(b), "echo alpine") {
	363	+	t.Fatalf("alpine node: %q %v", b, err)
	364	+	}
	365	+	hits := f.hits.Load()
	366	+	if hits != 3 {
	367	+	t.Fatalf("downloads: %d", hits)
	368	+	}
	369	+
	370	+	// A second start finds everything in place and downloads nothing.
	371	+	m2 := &Manager{Dir: f.dir, Arch: "x64", Table: f.table, Client: f.srv.Client()}
	372	+	if err := m2.Sync(context.Background()); err != nil {
	373	+	t.Fatal(err)
	374	+	}
	375	+	if f.hits.Load() != hits {
	376	+	t.Fatal("downloaded again")
	377	+	}
	378	+	if got := byName(m2.Externals())["node24"]; got.ID != n24.ID {
	379	+	t.Fatalf("ID changed: %q -> %q", n24.ID, got.ID)
	380	+	}
	381	+	}
	382	+
	383	+	// runStub runs the bin/node of an unavailable tool: it must fail with the reason
	384	+	// on stderr, from wherever the tool is mounted.
	385	+	func runStub(t *testing.T, e sandbox.External, want string) {
	386	+	t.Helper()
	387	+	if _, err := exec.LookPath("sh"); err != nil {
	388	+	t.Skip("sh needed")
	389	+	}
	390	+	// Run it through a symlinked path too, as a guest sees it at another place.
	391	+	cmd := exec.Command(filepath.Join(e.Dir, "bin", "node"), "-e", "x")
	392	+	var stderr, stdout bytes.Buffer
	393	+	cmd.Stderr, cmd.Stdout = &stderr, &stdout
	394	+	err := cmd.Run()
	395	+	if ee, ok := err.(*exec.ExitError); !ok \|\| ee.ExitCode() != 1 {
	396	+	t.Fatalf("the stub must exit 1, got %v", err)
	397	+	}
	398	+	if !strings.Contains(stderr.String(), want) \|\| stdout.Len() != 0 {
	399	+	t.Fatalf("stub stderr %q stdout %q, want %q", stderr.String(), stdout.String(), want)
	400	+	}
	401	+	if b, _ := os.ReadFile(filepath.Join(e.Dir, "bin", "node")); strings.Contains(string(b), want) {
	402	+	t.Fatalf("the stub holds the reason: %q", b)
	403	+	}
	404	+	}
	405	+
	406	+	func TestSyncRefusesAChecksumMismatchBeforeExtracting(t *testing.T) {
	407	+	f := newFixture(t)
	408	+	f.table[0].SHA256 = strings.Repeat("0", 64)
	409	+	f.m.Table = f.table
	410	+	err := f.m.Sync(context.Background())
	411	+	if err == nil \|\| !strings.Contains(err.Error(), "SHA-256 mismatch") {
	412	+	t.Fatalf("got %v", err)
	413	+	}
	414	+	n20 := byName(f.m.Externals())["node20"]
	415	+	if !n20.Unavailable {
	416	+	t.Fatal("a tool with a wrong checksum is available")
	417	+	}
	418	+	if _, err := os.Stat(filepath.Join(f.dir, "node20-1.0.0-x64")); err == nil {
	419	+	t.Fatal("extracted although the checksum differs")
	420	+	}
	421	+	reason, _ := os.ReadFile(filepath.Join(n20.Dir, sandbox.ExternalUnavailableFile))
	422	+	if !strings.Contains(string(reason), "SHA-256 mismatch") {
	423	+	t.Fatalf("reason: %q", reason)
	424	+	}
	425	+	// The other tool is unaffected.
	426	+	if byName(f.m.Externals())["node24"].Unavailable {
	427	+	t.Fatal("node24 lost to node20's failure")
	428	+	}
	429	+	// A checksum failure is a verdict: no retry is scheduled for it.
	430	+	if f.m.state["node20"].retry {
	431	+	t.Fatal("retry scheduled for a mismatch")
	432	+	}
	433	+	runStub(t, n20, "SHA-256 mismatch")
	434	+	ents, _ := os.ReadDir(f.dir)
	435	+	for _, e := range ents {
	436	+	if strings.HasPrefix(e.Name(), ".tmp-") {
	437	+	t.Fatalf("leftover %s", e.Name())
	438	+	}
	439	+	}
	440	+	}
	441	+
	442	+	func TestSyncRefusesAnUnsafeArchiveWithTheRightChecksum(t *testing.T) {
	443	+	f := newFixture(t)
	444	+	bad := targz(t, ent{name: "node-v1-linux-x64/bin/node", mode: 0o755, body: "x"}, ent{name: "node-v1-linux-x64/l", typ: tar.TypeSymlink, link: "/etc"})
	445	+	f.archive = bad
	446	+	s := sha256.Sum256(bad)
	447	+	f.table[0].SHA256, f.table[1].SHA256 = hex.EncodeToString(s[:]), hex.EncodeToString(s[:])
	448	+	f.m.Table = f.table
	449	+	err := f.m.Sync(context.Background())
	450	+	if err == nil \|\| !strings.Contains(err.Error(), "absolute") {
	451	+	t.Fatalf("got %v", err)
	452	+	}
	453	+	if _, err := os.Stat(filepath.Join(f.dir, "node20-1.0.0-x64")); err == nil {
	454	+	t.Fatal("unsafe archive left in place")
	455	+	}
	456	+	}
	457	+
	458	+	func TestSyncOfflineKeepsTheRunnerGoing(t *testing.T) {
	459	+	f := newFixture(t)
	460	+	f.srv.Close() // offline
	461	+	err := f.m.Sync(context.Background())
	462	+	if err == nil {
	463	+	t.Fatal("no error offline")
	464	+	}
	465	+	for _, n := range []string{"node20", "node24"} {
	466	+	e := byName(f.m.Externals())[n]
	467	+	if !e.Unavailable {
	468	+	t.Fatalf("%s available offline", n)
	469	+	}
	470	+	runStub(t, e, "is not available on this runner")
	471	+	}
	472	+	if !f.m.wantsRetry() {
	473	+	t.Fatal("a network failure must be retried")
	474	+	}
	475	+	}
	476	+
	477	+	func TestBadStatusAndRedirectToHTTP(t *testing.T) {
	478	+	f := newFixture(t)
	479	+	f.table[0].URL = f.srv.URL + "/missing.tar.gz"
	480	+	f.m.Table = f.table
	481	+	if err := f.m.Sync(context.Background()); err == nil \|\| !strings.Contains(err.Error(), "404") {
	482	+	t.Fatalf("got %v", err)
	483	+	}
	484	+	if !byName(f.m.Externals())["node20"].Unavailable {
	485	+	t.Fatal("404 gives a tool")
	486	+	}
	487	+
	488	+	// The default client refuses a redirect to plain http, and too many.
	489	+	check := newClient().CheckRedirect
	490	+	if err := check(&http.Request{URL: mustURL(t, "http://example.org/x")}, nil); err == nil {
	491	+	t.Error("redirect to http followed")
	492	+	}
	493	+	if err := check(&http.Request{URL: mustURL(t, "https://example.org/x")}, make([]*http.Request, 5)); err == nil {
	494	+	t.Error("redirect loop followed")
	495	+	}
	496	+	if err := check(&http.Request{URL: mustURL(t, "https://example.org/x")}, make([]*http.Request, 1)); err != nil {
	497	+	t.Error(err)
	498	+	}
	499	+	// A plain http URL in the table is refused outright.
	500	+	m2 := &Manager{Dir: filepath.Join(t.TempDir(), "e"), Arch: "x64", Client: f.srv.Client()}
	501	+	m2.Table = []Artifact{{Name: "node20", Version: "1", Arch: "x64", URL: "http://example.invalid/a.tar.gz", SHA256: f.sum}}
	502	+	if err := m2.Sync(context.Background()); err == nil \|\| !strings.Contains(err.Error(), "not an https URL") {
	503	+	t.Fatalf("got %v", err)
	504	+	}
	505	+	}
	506	+
	507	+	func mustURL(t testing.T, s string) url.URL {
	508	+	t.Helper()
	509	+	u, err := url.Parse(s)
	510	+	if err != nil {
	511	+	t.Fatal(err)
	512	+	}
	513	+	return u
	514	+	}
	515	+
	516	+	func TestSyncKeepsOnlyPinnedVersions(t *testing.T) {
	517	+	f := newFixture(t)
	518	+	if err := f.m.Sync(context.Background()); err != nil {
	519	+	t.Fatal(err)
	520	+	}
	521	+	// Leftovers: an old version, an interrupted download, a stray file, an
	522	+	// incomplete directory under a pinned name.
	523	+	for _, d := range []string{"node20-0.9.0-x64", ".tmp-123", "node24-2.0.0-arm64", "node-old"} {
	524	+	if err := os.MkdirAll(filepath.Join(f.dir, d, "bin"), 0o755); err != nil {
	525	+	t.Fatal(err)
	526	+	}
	527	+	}
	528	+	if err := os.WriteFile(filepath.Join(f.dir, "stray"), []byte("x"), 0o644); err != nil {
	529	+	t.Fatal(err)
	530	+	}
	531	+	if err := f.m.Sync(context.Background()); err != nil {
	532	+	t.Fatal(err)
	533	+	}
	534	+	var got []string
	535	+	ents, _ := os.ReadDir(f.dir)
	536	+	for _, e := range ents {
	537	+	got = append(got, e.Name())
	538	+	}
	539	+	want := "node20-1.0.0-x64 node20_alpine-1.0.0-x64 node24-2.0.0-x64"
	540	+	if strings.Join(got, " ") != want {
	541	+	t.Fatalf("kept %v, want %s", got, want)
	542	+	}
	543	+	}
	544	+
	545	+	func TestSyncRebuildsAnIncompleteDirectory(t *testing.T) {
	546	+	f := newFixture(t)
	547	+	final := filepath.Join(f.dir, "node20-1.0.0-x64")
	548	+	if err := os.MkdirAll(filepath.Join(final, "bin"), 0o755); err != nil { // no bin/node
	549	+	t.Fatal(err)
	550	+	}
	551	+	if err := f.m.Sync(context.Background()); err != nil {
	552	+	t.Fatal(err)
	553	+	}
	554	+	if !isReady(final) {
	555	+	t.Fatal("incomplete directory kept")
	556	+	}
	557	+	}
	558	+
	559	+	func TestANewVersionChangesTheIDAndDropsTheOldDirectory(t *testing.T) {
	560	+	f := newFixture(t)
	561	+	if err := f.m.Sync(context.Background()); err != nil {
	562	+	t.Fatal(err)
	563	+	}
	564	+	old := byName(f.m.Externals())["node24"]
	565	+	f.table[1].Version = "2.1.0"
	566	+	f.table[1].URL = f.srv.URL + "/v2.1.0/node-v2.1.0-linux-x64.tar.gz"
	567	+	if err := f.m.Sync(context.Background()); err != nil {
	568	+	t.Fatal(err)
	569	+	}
	570	+	n := byName(f.m.Externals())["node24"]
	571	+	if n.ID == old.ID \|\| n.Dir == old.Dir {
	572	+	t.Fatalf("same tool after an upgrade: %+v %+v", old, n)
	573	+	}
	574	+	if _, err := os.Stat(old.Dir); err == nil {
	575	+	t.Fatal("old version kept")
	576	+	}
	577	+	}
	578	+
	579	+	func TestNoBuildForTheArchitecture(t *testing.T) {
	580	+	f := newFixture(t)
	581	+	f.m.Arch = "arm64" // the fixture table is x64 only
	582	+	if err := f.m.Sync(context.Background()); err != nil {
	583	+	t.Fatal(err)
	584	+	}
	585	+	for _, e := range f.m.Externals() {
	586	+	reason, _ := os.ReadFile(filepath.Join(e.Dir, sandbox.ExternalUnavailableFile))
	587	+	if !e.Unavailable \|\| !strings.Contains(string(reason), "no "+e.Name+" build for linux-arm64") {
	588	+	t.Errorf("%s: %+v %q", e.Name, e, reason)
	589	+	}
	590	+	}
	591	+	if f.hits.Load() != 0 {
	592	+	t.Error("downloaded for the wrong architecture")
	593	+	}
	594	+	}
	595	+
	596	+	func TestBeforeTheFirstSyncTheToolsAreBeingDownloaded(t *testing.T) {
	597	+	f := newFixture(t)
	598	+	set := f.m.Externals()
	599	+	if len(set) != 3 {
	600	+	t.Fatalf("set: %+v", set)
	601	+	}
	602	+	for _, e := range set {
	603	+	reason, _ := os.ReadFile(filepath.Join(e.Dir, sandbox.ExternalUnavailableFile))
	604	+	if !e.Unavailable \|\| !strings.Contains(string(reason), "still being downloaded") {
	605	+	t.Errorf("%s: %q", e.Name, reason)
	606	+	}
	607	+	}
	608	+	}
	609	+
	610	+	func TestFailedDownloadIsRetriedInTheBackground(t *testing.T) {
	611	+	f := newFixture(t)
	612	+	var up atomic.Bool
	613	+	srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	614	+	if !up.Load() {
	615	+	http.Error(w, "down", http.StatusServiceUnavailable)
	616	+	return
	617	+	}
	618	+	_, _ = w.Write(f.archive)
	619	+	}))
	620	+	defer srv.Close()
	621	+	m := &Manager{Dir: f.dir, Arch: "x64", Client: srv.Client(), RetryEvery: time.Millisecond}
	622	+	m.Table = []Artifact{{Name: "node24", Version: "2.0.0", Arch: "x64", URL: srv.URL + "/a-linux-x64.tar.gz", SHA256: f.sum, StripTop: true}}
	623	+	ctx, cancel := context.WithCancel(context.Background())
	624	+	defer cancel()
	625	+	m.Start(ctx, time.Minute)
	626	+	if !m.Externals()[0].Unavailable {
	627	+	t.Fatal("available while the server is down")
	628	+	}
	629	+	up.Store(true)
	630	+	deadline := time.Now().Add(10 * time.Second)
	631	+	for time.Now().Before(deadline) {
	632	+	time.Sleep(20 * time.Millisecond)
	633	+	if !m.Externals()[0].Unavailable {
	634	+	return
	635	+	}
	636	+	}
	637	+	t.Fatal("never retried")
	638	+	}
	639	+
	640	+	func TestStartDoesNotWaitLongerThanAsked(t *testing.T) {
	641	+	block := make(chan struct{})
	642	+	srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { <-block }))
	643	+	defer srv.Close()
	644	+	m := &Manager{Dir: filepath.Join(t.TempDir(), "e"), Arch: "x64", Client: srv.Client()}
	645	+	m.Table = []Artifact{{Name: "node24", Version: "2.0.0", Arch: "x64", URL: srv.URL + "/a.tar.gz", SHA256: strings.Repeat("a", 64)}}
	646	+	ctx, cancel := context.WithCancel(context.Background())
	647	+	start := time.Now()
	648	+	m.Start(ctx, 100*time.Millisecond)
	649	+	if d := time.Since(start); d > 5*time.Second {
	650	+	t.Fatalf("waited %v", d)
	651	+	}
	652	+	// The download goes on in the background; stop it before the directory goes.
	653	+	cancel()
	654	+	close(block)
	655	+	for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(10 * time.Millisecond) {
	656	+	m.mu.Lock()
	657	+	busy := m.syncing
	658	+	m.mu.Unlock()
	659	+	if !busy {
	660	+	return
	661	+	}
	662	+	}
	663	+	t.Fatal("the background sync did not stop")
	664	+	}
	665	+
	666	+	// ---- the directories of unavailable tools
	667	+
	668	+	func TestStubIsBuiltOnceAndNeverRewrittenInPlace(t *testing.T) {
	669	+	m := &Manager{Dir: filepath.Join(t.TempDir(), "e"), Arch: "x64"}
	670	+	e := m.unavailable("node24", "why", false)
	671	+	node := filepath.Join(e.ext.Dir, "bin", "node")
	672	+	file := filepath.Join(e.ext.Dir, sandbox.ExternalUnavailableFile)
	673	+	ino := func(p string) uint64 {
	674	+	st, err := os.Stat(p)
	675	+	if err != nil {
	676	+	t.Fatal(err)
	677	+	}
	678	+	return st.Sys().(*syscall.Stat_t).Ino
	679	+	}
	680	+	n0, f0 := ino(node), ino(file)
	681	+
	682	+	// Again, from this process and from a fresh one (a restart): same files.
	683	+	m.unavailable("node24", "why", false)
	684	+	(&Manager{Dir: m.Dir, Arch: "x64"}).unavailable("node24", "why", false)
	685	+	if ino(node) != n0 \|\| ino(file) != f0 {
	686	+	t.Fatal("an existing, correct stub was rewritten")
	687	+	}
	688	+
	689	+	// A damaged or missing file is replaced by a rename (a new inode, so a reader
	690	+	// of the old one never sees it change), with the right mode.
	691	+	if err := os.WriteFile(node, nil, 0o755); err != nil {
	692	+	t.Fatal(err)
	693	+	}
	694	+	if err := os.Remove(file); err != nil {
	695	+	t.Fatal(err)
	696	+	}
	697	+	(&Manager{Dir: m.Dir, Arch: "x64"}).unavailable("node24", "why", false)
	698	+	if b, _ := os.ReadFile(node); string(b) != unavailableStub {
	699	+	t.Fatalf("stub not repaired: %q", b)
	700	+	}
	701	+	if b, _ := os.ReadFile(file); string(b) != "why\n" {
	702	+	t.Fatalf("reason not repaired: %q", b)
	703	+	}
	704	+	if st, _ := os.Stat(node); st.Mode().Perm() != 0o755 \|\| ino(node) == n0 {
	705	+	t.Fatalf("repair in place: %v", st.Mode())
	706	+	}
	707	+	runStub(t, e.ext, "why")
	708	+	left, _ := filepath.Glob(filepath.Join(e.ext.Dir, "", ".tmp-"))
	709	+	left2, _ := filepath.Glob(filepath.Join(e.ext.Dir, ".tmp-*"))
	710	+	if len(left)+len(left2) != 0 {
	711	+	t.Fatalf("temporary files left: %v %v", left, left2)
	712	+	}
	713	+	}
	714	+
	715	+	// While Externals() and Sync() run in several goroutines, a reader that walks
	716	+	// the directories of the unavailable tools (as the build of a VM's externals
	717	+	// disk does) never sees a missing, empty or partial file.
	718	+	func TestConcurrentExternalsAndSyncNeverShowAHalfWrittenStub(t *testing.T) {
	719	+	f := newFixture(t)
	720	+	f.srv.Close() // offline: every tool is unavailable
	721	+	ctx := context.Background()
	722	+	stop := make(chan struct{})
	723	+	var wg sync.WaitGroup
	724	+	var bad atomic.Value
	725	+	check := func(dir string) {
	726	+	for _, c := range []struct{ name, want string }{{"bin/node", unavailableStub}} {
	727	+	b, err := os.ReadFile(filepath.Join(dir, c.name))
	728	+	if err != nil {
	729	+	bad.Store(fmt.Sprintf("%s/%s: %v", dir, c.name, err))
	730	+	} else if string(b) != c.want {
	731	+	bad.Store(fmt.Sprintf("%s/%s: %q", dir, c.name, b))
	732	+	}
	733	+	}
	734	+	b, err := os.ReadFile(filepath.Join(dir, sandbox.ExternalUnavailableFile))
	735	+	if err != nil \|\| len(b) == 0 \|\| !strings.HasSuffix(string(b), "\n") {
	736	+	bad.Store(fmt.Sprintf("%s/UNAVAILABLE: %q %v", dir, b, err))
	737	+	}
	738	+	}
	739	+	for i := 0; i < 4; i++ {
	740	+	wg.Add(1)
	741	+	go func() {
	742	+	defer wg.Done()
	743	+	for {
	744	+	select {
	745	+	case <-stop:
	746	+	return
	747	+	default:
	748	+	}
	749	+	for _, e := range f.m.Externals() {
	750	+	check(e.Dir) // what a driver would hand to the disk build
	751	+	}
	752	+	}
	753	+	}()
	754	+	}
	755	+	wg.Add(1)
	756	+	go func() {
	757	+	defer wg.Done()
	758	+	for {
	759	+	select {
	760	+	case <-stop:
	761	+	return
	762	+	default:
	763	+	}
	764	+	dirs, _ := filepath.Glob(filepath.Join(f.dir, unavailableDir, "*"))
	765	+	for _, d := range dirs {
	766	+	if !strings.Contains(filepath.Base(d), ".tmp-") {
	767	+	check(d)
	768	+	}
	769	+	}
	770	+	}
	771	+	}()
	772	+	for i := 0; i < 5; i++ {
	773	+	_ = f.m.Sync(ctx)
	774	+	}
	775	+	close(stop)
	776	+	wg.Wait()
	777	+	if v := bad.Load(); v != nil {
	778	+	t.Fatalf("a reader saw an incomplete stub: %v", v)
	779	+	}
	780	+	}
	781	+
	782	+	func TestPruneKeepsStubDirectoriesHandedOutInTheLastHour(t *testing.T) {
	783	+	f := newFixture(t)
	784	+	f.srv.Close()
	785	+	_ = f.m.Sync(context.Background()) // offline: stubs for the three tools
	786	+	root := filepath.Join(f.dir, unavailableDir)
	787	+	mk := func(name string, age time.Duration) string {
	788	+	p := filepath.Join(root, name)
	789	+	if err := os.MkdirAll(filepath.Join(p, "bin"), 0o755); err != nil {
	790	+	t.Fatal(err)
	791	+	}
	792	+	old := time.Now().Add(-age)
	793	+	if err := os.Chtimes(p, old, old); err != nil {
	794	+	t.Fatal(err)
	795	+	}
	796	+	return p
	797	+	}
	798	+	fresh := mk("node24-unavailable-aaaaaaaaaaaaaaaa", 5*time.Minute) // handed to a running job
	799	+	stale := mk("node24-unavailable-bbbbbbbbbbbbbbbb", 2*time.Hour)
	800	+
	801	+	// A reason that is not current any more but was handed out again just now
	802	+	// (Externals touches what it returns) survives; one that was not, does not.
	803	+	cur := f.m.Externals()[0]
	804	+	old := time.Now().Add(-3 * time.Hour)
	805	+	if err := os.Chtimes(cur.Dir, old, old); err != nil {
	806	+	t.Fatal(err)
	807	+	}
	808	+	f.m.Externals() // handed out: touched
	809	+	if st, _ := os.Stat(cur.Dir); time.Since(st.ModTime()) > time.Minute {
	810	+	t.Fatalf("Externals did not touch %s", cur.Dir)
	811	+	}
	812	+	// The set changes (the reasons are not the same any more), the touched
	813	+	// directory is no longer current.
	814	+	f.m.mu.Lock()
	815	+	f.m.state = map[string]entry{}
	816	+	f.m.mu.Unlock()
	817	+	f.m.prune(zerowrap.Logger{}, map[string]bool{})
	818	+	for _, p := range []string{fresh, cur.Dir} {
	819	+	if _, err := os.Stat(p); err != nil {
	820	+	t.Errorf("%s was pruned although handed out in the last hour", p)
	821	+	}
	822	+	}
	823	+	if _, err := os.Stat(stale); err == nil {
	824	+	t.Error("a stale stub directory was kept")
	825	+	}
	826	+	}
	827	+
	828	+	// ---- retries
	829	+
	830	+	func TestRetriesRunOnATickerWithoutAnyJob(t *testing.T) {
	831	+	f := newFixture(t)
	832	+	var up atomic.Bool
	833	+	var calls atomic.Int64
	834	+	srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	835	+	calls.Add(1)
	836	+	if !up.Load() {
	837	+	http.Error(w, "down", http.StatusServiceUnavailable)
	838	+	return
	839	+	}
	840	+	_, _ = w.Write(f.archive)
	841	+	}))
	842	+	defer srv.Close()
	843	+	m := &Manager{Dir: f.dir, Arch: "x64", Client: srv.Client(), RetryEvery: 20 * time.Millisecond}
	844	+	m.Table = []Artifact{{Name: "node24", Version: "2.0.0", Arch: "x64", URL: srv.URL + "/a-linux-x64.tar.gz", SHA256: f.sum, StripTop: true}}
	845	+	ctx, cancel := context.WithCancel(context.Background())
	846	+	m.Start(ctx, time.Minute)
	847	+
	848	+	// Nobody calls Externals(): the ticker alone retries.
	849	+	deadline := time.Now().Add(10 * time.Second)
	850	+	for calls.Load() < 3 && time.Now().Before(deadline) {
	851	+	time.Sleep(10 * time.Millisecond)
	852	+	}
	853	+	if calls.Load() < 3 {
	854	+	t.Fatalf("%d attempts without any job", calls.Load())
	855	+	}
	856	+	up.Store(true)
	857	+	for time.Now().Before(deadline) {
	858	+	m.mu.Lock()
	859	+	ok := !m.state["node24"].ext.Unavailable
	860	+	m.mu.Unlock()
	861	+	if ok {
	862	+	break
	863	+	}
	864	+	time.Sleep(10 * time.Millisecond)
	865	+	}
	866	+	if m.Externals()[0].Unavailable {
	867	+	t.Fatal("never recovered")
	868	+	}
	869	+
	870	+	// Once everything is there the ticker stops downloading, and with the context
	871	+	// it stops altogether.
	872	+	before := calls.Load()
	873	+	time.Sleep(150 * time.Millisecond)
	874	+	if calls.Load() != before {
	875	+	t.Fatal("the ticker keeps requesting what it has")
	876	+	}
	877	+	cancel()
	878	+	m.mu.Lock()
	879	+	m.state["node24"] = entry{ext: sandbox.External{Name: "node24", Unavailable: true}, retry: true}
	880	+	m.mu.Unlock()
	881	+	time.Sleep(150 * time.Millisecond)
	882	+	if calls.Load() != before {
	883	+	t.Fatal("a retry ran after the context ended")
	884	+	}
	885	+	}
	886	+
	887	+	// A verdict (wrong checksum) is not retried by the ticker.
	888	+	func TestNoRetryForAVerdict(t *testing.T) {
	889	+	f := newFixture(t)
	890	+	f.table[0].SHA256 = strings.Repeat("0", 64)
	891	+	m := &Manager{Dir: f.dir, Arch: "x64", Table: f.table[:1], Client: f.srv.Client(), RetryEvery: 10 * time.Millisecond}
	892	+	ctx, cancel := context.WithCancel(context.Background())
	893	+	defer cancel()
	894	+	m.Start(ctx, time.Minute)
	895	+	hits := f.hits.Load()
	896	+	time.Sleep(150 * time.Millisecond)
	897	+	if f.hits.Load() != hits {
	898	+	t.Fatal("a checksum mismatch was retried")
	899	+	}
	900	+	}
	901	+
	902	+	// ---- the real archives, when they are at hand
	903	+
	904	+	// The actions/alpine_nodejs archives have no top directory (./bin/node). Set
	905	+	// ERMITE_RUNNER_TEST_ALPINE_DIR, or leave the files in /tmp/alpine-node.
	906	+	func TestRealAlpineArchives(t *testing.T) {
	907	+	dir := os.Getenv("ERMITE_RUNNER_TEST_ALPINE_DIR")
	908	+	if dir == "" {
	909	+	dir = "/tmp/alpine-node"
	910	+	}
	911	+	ran := false
	912	+	for _, a := range Pinned {
	913	+	if !strings.HasSuffix(a.Name, "_alpine") {
	914	+	continue
	915	+	}
	916	+	archive := filepath.Join(dir, filepath.Base(a.URL))
	917	+	b, err := os.ReadFile(archive)
	918	+	if err != nil {
	919	+	t.Logf("%s: %v", a.Name, err)
	920	+	continue
	921	+	}
	922	+	ran = true
	923	+	t.Run(a.Name, func(t *testing.T) {
	924	+	sum := sha256.Sum256(b)
	925	+	if err := VerifySum(hex.EncodeToString(sum[:]), a.SHA256); err != nil {
	926	+	t.Fatalf("the pinned checksum is not the file's: %v", err)
	927	+	}
	928	+	dst := t.TempDir()
	929	+	if err := extractTarGz(archive, dst, a.StripTop); err != nil {
	930	+	t.Fatal(err)
	931	+	}
	932	+	if !isReady(dst) {
	933	+	t.Fatal("no executable bin/node")
	934	+	}
	935	+	if st, err := os.Stat(filepath.Join(dst, "LICENSE")); err != nil \|\| st.Mode().Perm() != 0o644 {
	936	+	t.Fatalf("LICENSE: %v %v", st, err)
	937	+	}
	938	+	// Taking the first component off would lose bin/node.
	939	+	wrong := t.TempDir()
	940	+	_ = extractTarGz(archive, wrong, true)
	941	+	if isReady(wrong) {
	942	+	t.Fatal("the alpine archive must not be unpacked with the top directory stripped")
	943	+	}
	944	+	})
	945	+	}
	946	+	if !ran {
	947	+	t.Skipf("no alpine archive in %s", dir)
	948	+	}
	949	+	}

@@ -0,0 +1,949 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package externals
	4	+
	5	+	import (
	6	+	"archive/tar"
	7	+	"bytes"
	8	+	"compress/gzip"
	9	+	"context"
	10	+	"crypto/sha256"
	11	+	"encoding/hex"
	12	+	"fmt"
	13	+	"net/http"
	14	+	"net/http/httptest"
	15	+	"net/url"
	16	+	"os"
	17	+	"os/exec"
	18	+	"path/filepath"
	19	+	"regexp"
	20	+	"strings"
	21	+	"sync"
	22	+	"sync/atomic"
	23	+	"syscall"
	24	+	"testing"
	25	+	"time"
	26	+
	27	+	"github.com/bnema/zerowrap"
	28	+
	29	+	"bnema.dev/ermite-runner/internal/sandbox"
	30	+	)
	31	+
	32	+	// ---- the table
	33	+
	34	+	func TestPinnedTable(t *testing.T) {
	35	+	hex64 := regexp.MustCompile(`^[0-9a-f]{64}$`)
	36	+	seen := map[string]bool{}
	37	+	for _, a := range Pinned {
	38	+	key := a.Name + "/" + a.Arch
	39	+	if seen[key] {
	40	+	t.Errorf("%s listed twice", key)
	41	+	}
	42	+	seen[key] = true
	43	+	if !strings.HasPrefix(a.URL, "https://") \|\| !strings.HasSuffix(a.URL, ".tar.gz") \|\| !strings.Contains(a.URL, "v"+a.Version+"/") {
	44	+	t.Errorf("%s: URL %q", key, a.URL)
	45	+	}
	46	+	if !hex64.MatchString(a.SHA256) {
	47	+	t.Errorf("%s: SHA-256 %q", key, a.SHA256)
	48	+	}
	49	+	if a.StripTop != strings.HasPrefix(a.URL, nodeDist) {
	50	+	t.Errorf("%s: StripTop %v for %s", key, a.StripTop, a.URL)
	51	+	}
	52	+	}
	53	+	// actions/runner's externals.sh: node20 and node24 for both architectures,
	54	+	// the alpine builds for x64.
	55	+	for _, k := range []string{"node20/x64", "node20/arm64", "node24/x64", "node24/arm64", "node20_alpine/x64", "node24_alpine/x64"} {
	56	+	if !seen[k] {
	57	+	t.Errorf("%s missing", k)
	58	+	}
	59	+	}
	60	+	for _, a := range Pinned {
	61	+	if strings.HasPrefix(a.Name, "node20") && a.Version != "20.20.2" \|\| strings.HasPrefix(a.Name, "node24") && a.Version != "24.21.0" {
	62	+	t.Errorf("%s pins %s", a.Name, a.Version)
	63	+	}
	64	+	}
	65	+	for _, a := range Pinned {
	66	+	if strings.HasSuffix(a.Name, "_alpine") && (a.Arch != "x64" \|\| !strings.HasPrefix(a.URL, alpineDist+"v"+a.Version+"/node-v"+a.Version+"-alpine-x64.tar.gz")) {
	67	+	t.Errorf("%s: %+v", a.Name, a)
	68	+	}
	69	+	}
	70	+	if got := len(ForArch(Pinned, "arm64")); got != 2 {
	71	+	t.Errorf("arm64 has %d artifacts", got)
	72	+	}
	73	+	}
	74	+
	75	+	func TestVerifySum(t *testing.T) {
	76	+	if err := VerifySum("AB12", "ab12"); err != nil {
	77	+	t.Error(err)
	78	+	}
	79	+	if err := VerifySum("ab12", "ab13"); err == nil \|\| !strings.Contains(err.Error(), "mismatch") {
	80	+	t.Errorf("got %v", err)
	81	+	}
	82	+	}
	83	+
	84	+	// ---- archives
	85	+
	86	+	type ent struct {
	87	+	name, link string
	88	+	typ byte
	89	+	mode int64
	90	+	body string
	91	+	}
	92	+
	93	+	func targz(t *testing.T, ents ...ent) []byte {
	94	+	t.Helper()
	95	+	var buf bytes.Buffer
	96	+	gz := gzip.NewWriter(&buf)
	97	+	tw := tar.NewWriter(gz)
	98	+	for _, e := range ents {
	99	+	typ := e.typ
	100	+	if typ == 0 {
	101	+	typ = tar.TypeReg
	102	+	}
	103	+	h := &tar.Header{Name: e.name, Linkname: e.link, Typeflag: typ, Mode: e.mode, Size: int64(len(e.body))}
	104	+	if typ != tar.TypeReg {
	105	+	h.Size = 0
	106	+	}
	107	+	if err := tw.WriteHeader(h); err != nil {
	108	+	t.Fatal(err)
	109	+	}
	110	+	if typ == tar.TypeReg {
	111	+	if _, err := tw.Write([]byte(e.body)); err != nil {
	112	+	t.Fatal(err)
	113	+	}
	114	+	}
	115	+	}
	116	+	if err := tw.Close(); err != nil {
	117	+	t.Fatal(err)
	118	+	}
	119	+	if err := gz.Close(); err != nil {
	120	+	t.Fatal(err)
	121	+	}
	122	+	return buf.Bytes()
	123	+	}
	124	+
	125	+	func nodeTree() []ent {
	126	+	return []ent{
	127	+	{name: "node-v1-linux-x64/", typ: tar.TypeDir, mode: 0o755},
	128	+	{name: "node-v1-linux-x64/bin/", typ: tar.TypeDir, mode: 0o755},
	129	+	{name: "node-v1-linux-x64/bin/node", mode: 0o755, body: "#!/bin/sh\necho v1\n"},
	130	+	{name: "node-v1-linux-x64/lib/node_modules/npm/bin/npm-cli.js", mode: 0o644, body: "x"},
	131	+	{name: "node-v1-linux-x64/bin/npm", typ: tar.TypeSymlink, link: "../lib/node_modules/npm/bin/npm-cli.js", mode: 0o777},
	132	+	{name: "node-v1-linux-x64/bin/nodejs", typ: tar.TypeLink, link: "node-v1-linux-x64/bin/node", mode: 0o755},
	133	+	}
	134	+	}
	135	+
	136	+	func extractBytes(t *testing.T, data []byte) (string, error) {
	137	+	t.Helper()
	138	+	dir := t.TempDir()
	139	+	arch := filepath.Join(dir, "a.tar.gz")
	140	+	if err := os.WriteFile(arch, data, 0o600); err != nil {
	141	+	t.Fatal(err)
	142	+	}
	143	+	dst := filepath.Join(dir, "out")
	144	+	if err := os.Mkdir(dst, 0o755); err != nil {
	145	+	t.Fatal(err)
	146	+	}
	147	+	return dst, extractTarGz(arch, dst, true)
	148	+	}
	149	+
	150	+	func TestExtractLayoutAndModes(t *testing.T) {
	151	+	dst, err := extractBytes(t, targz(t, nodeTree()...))
	152	+	if err != nil {
	153	+	t.Fatal(err)
	154	+	}
	155	+	st, err := os.Stat(filepath.Join(dst, "bin", "node"))
	156	+	if err != nil \|\| st.Mode().Perm() != 0o755 {
	157	+	t.Fatalf("bin/node: %v %v", st, err)
	158	+	}
	159	+	if st, _ := os.Stat(filepath.Join(dst, "lib/node_modules/npm/bin/npm-cli.js")); st == nil \|\| st.Mode().Perm() != 0o644 {
	160	+	t.Fatalf("npm-cli.js: %v", st)
	161	+	}
	162	+	if l, err := os.Readlink(filepath.Join(dst, "bin", "npm")); err != nil \|\| l != "../lib/node_modules/npm/bin/npm-cli.js" {
	163	+	t.Fatalf("npm link: %q %v", l, err)
	164	+	}
	165	+	a, _ := os.Stat(filepath.Join(dst, "bin", "node"))
	166	+	b, _ := os.Stat(filepath.Join(dst, "bin", "nodejs"))
	167	+	if !os.SameFile(a, b) {
	168	+	t.Fatal("hard link not recreated")
	169	+	}
	170	+	if !isReady(dst) {
	171	+	t.Fatal("not ready")
	172	+	}
	173	+	}
	174	+
	175	+	func TestModesDoNotDependOnTheUmask(t *testing.T) {
	176	+	old := syscall.Umask(0o077)
	177	+	defer syscall.Umask(old)
	178	+	dst, err := extractBytes(t, targz(t, nodeTree()...))
	179	+	if err != nil {
	180	+	t.Fatal(err)
	181	+	}
	182	+	for p, want := range map[string]os.FileMode{
	183	+	dst: 0o755, filepath.Join(dst, "bin"): 0o755, filepath.Join(dst, "lib/node_modules"): 0o755,
	184	+	filepath.Join(dst, "bin/node"): 0o755, filepath.Join(dst, "lib/node_modules/npm/bin/npm-cli.js"): 0o644,
	185	+	} {
	186	+	if st, err := os.Stat(p); err != nil \|\| st.Mode().Perm() != want {
	187	+	t.Errorf("%s: %v %v, want %v", p, st, err, want)
	188	+	}
	189	+	}
	190	+	m := &Manager{Dir: filepath.Join(t.TempDir(), "e"), Arch: "x64"}
	191	+	e := m.unavailable("node24", "why", false)
	192	+	for p, want := range map[string]os.FileMode{
	193	+	filepath.Dir(e.ext.Dir): 0o755, e.ext.Dir: 0o755, filepath.Join(e.ext.Dir, "bin"): 0o755,
	194	+	filepath.Join(e.ext.Dir, "bin/node"): 0o755, filepath.Join(e.ext.Dir, "UNAVAILABLE"): 0o644,
	195	+	} {
	196	+	if st, err := os.Stat(p); err != nil \|\| st.Mode().Perm() != want {
	197	+	t.Errorf("%s: %v %v, want %v", p, st, err, want)
	198	+	}
	199	+	}
	200	+	}
	201	+
	202	+	func TestExtractRefusesUnsafeArchives(t *testing.T) {
	203	+	top := "node-v1-linux-x64/"
	204	+	for name, ents := range map[string][]ent{
	205	+	"absolute path": {{name: "/etc/passwd", body: "x"}},
	206	+	"dot dot": {{name: top + "../../evil", body: "x"}},
	207	+	"dot dot in middle": {{name: top + "a/../../evil", body: "x"}},
	208	+	"backslash": {{name: top + `a\b`, body: "x"}},
	209	+	"absolute symlink": {{name: top + "l", typ: tar.TypeSymlink, link: "/etc"}},
	210	+	"escaping symlink": {{name: top + "l", typ: tar.TypeSymlink, link: "../../.."}},
	211	+	"escaping from deep": {{name: top + "a/b/l", typ: tar.TypeSymlink, link: "../../../x"}},
	212	+	"climb after a name": {{name: top + "a/l", typ: tar.TypeSymlink, link: "b/../../../x"}},
	213	+	"empty link target": {{name: top + "l", typ: tar.TypeSymlink, link: ""}},
	214	+	"write through link": {
	215	+	{name: top + "d", typ: tar.TypeSymlink, link: "."},
	216	+	{name: top + "d/file", body: "x"},
	217	+	},
	218	+	"write through dir link": {
	219	+	{name: top + "sub/", typ: tar.TypeDir},
	220	+	{name: top + "sub/up", typ: tar.TypeSymlink, link: ".."},
	221	+	{name: top + "sub/up/file", body: "x"},
	222	+	},
	223	+	"hard link outside": {{name: top + "h", typ: tar.TypeLink, link: "/etc/passwd"}},
	224	+	"hard link dot dot": {{name: top + "h", typ: tar.TypeLink, link: top + "../x"}},
	225	+	"hard link to top": {{name: top + "h", typ: tar.TypeLink, link: top}},
	226	+	"hard link to a link": {
	227	+	{name: top + "l", typ: tar.TypeSymlink, link: "x"},
	228	+	{name: top + "h", typ: tar.TypeLink, link: top + "l"},
	229	+	},
	230	+	"device": {{name: top + "dev", typ: tar.TypeChar}},
	231	+	"fifo": {{name: top + "fifo", typ: tar.TypeFifo}},
	232	+	"duplicate": {{name: top + "f", body: "1"}, {name: top + "f", body: "2"}},
	233	+	} {
	234	+	t.Run(name, func(t *testing.T) {
	235	+	dst, err := extractBytes(t, targz(t, ents...))
	236	+	if err == nil {
	237	+	t.Fatal("accepted")
	238	+	}
	239	+	// Nothing was written outside the destination, whatever the order.
	240	+	parent := filepath.Dir(dst)
	241	+	if _, err := os.Stat(filepath.Join(parent, "evil")); err == nil {
	242	+	t.Fatal("wrote outside")
	243	+	}
	244	+	if _, err := os.Stat(filepath.Join(dst, "..", "file")); err == nil {
	245	+	t.Fatal("wrote outside through a link")
	246	+	}
	247	+	})
	248	+	}
	249	+	}
	250	+
	251	+	func TestExtractRefusesGarbageAndTooMany(t *testing.T) {
	252	+	if _, err := extractBytes(t, []byte("not gzip")); err == nil {
	253	+	t.Error("garbage accepted")
	254	+	}
	255	+	var ents []ent
	256	+	for i := 0; i < maxEntries+1; i++ {
	257	+	ents = append(ents, ent{name: "t/d/", typ: tar.TypeDir})
	258	+	}
	259	+	if _, err := extractBytes(t, targz(t, ents...)); err == nil \|\| !strings.Contains(err.Error(), "entries") {
	260	+	t.Errorf("entry limit: %v", err)
	261	+	}
	262	+	}
	263	+
	264	+	func TestLinkTargetRule(t *testing.T) {
	265	+	for _, c := range []struct {
	266	+	name, target string
	267	+	ok bool
	268	+	}{
	269	+	{"bin/npm", "../lib/x", true},
	270	+	{"bin/npm", "../../x", false}, // one above the tree
	271	+	{"lib/a/b/l", "../../../x", true},
	272	+	{"lib/a/b/l", "../../../../x", false},
	273	+	{"l", "bin/node", true},
	274	+	{"l", "../x", false},
	275	+	{"l", "/abs", false},
	276	+	{"a/l", "b/../c", false},
	277	+	} {
	278	+	if err := checkLinkTarget(c.name, c.target); (err == nil) != c.ok {
	279	+	t.Errorf("%s -> %s: %v", c.name, c.target, err)
	280	+	}
	281	+	}
	282	+	}
	283	+
	284	+	// ---- the manager
	285	+
	286	+	type fixture struct {
	287	+	srv *httptest.Server
	288	+	hits atomic.Int64
	289	+	archive []byte
	290	+	sum string
	291	+	archiveAlpine []byte // no top directory, like actions/alpine_nodejs
	292	+	sumAlpine string
	293	+	table []Artifact
	294	+	dir string
	295	+	m *Manager
	296	+	}
	297	+
	298	+	func newFixture(t testing.T) fixture {
	299	+	t.Helper()
	300	+	f := &fixture{}
	301	+	f.archive = targz(t, nodeTree()...)
	302	+	s := sha256.Sum256(f.archive)
	303	+	f.sum = hex.EncodeToString(s[:])
	304	+	f.archiveAlpine = targz(t,
	305	+	ent{name: "./bin/", typ: tar.TypeDir, mode: 0o755},
	306	+	ent{name: "./bin/node", mode: 0o755, body: "#!/bin/sh\necho alpine\n"},
	307	+	ent{name: "./LICENSE", mode: 0o644, body: "x"})
	308	+	sa := sha256.Sum256(f.archiveAlpine)
	309	+	f.sumAlpine = hex.EncodeToString(sa[:])
	310	+	f.srv = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	311	+	f.hits.Add(1)
	312	+	switch {
	313	+	case strings.HasSuffix(r.URL.Path, "/alpine.tar.gz"):
	314	+	_, _ = w.Write(f.archiveAlpine)
	315	+	case strings.HasSuffix(r.URL.Path, "-linux-x64.tar.gz"):
	316	+	_, _ = w.Write(f.archive)
	317	+	case strings.HasSuffix(r.URL.Path, "tampered.tar.gz"):
	318	+	_, _ = w.Write(append([]byte(nil), f.archive[:len(f.archive)-1]...))
	319	+	default:
	320	+	http.NotFound(w, r)
	321	+	}
	322	+	}))
	323	+	t.Cleanup(f.srv.Close)
	324	+	f.dir = filepath.Join(t.TempDir(), "externals")
	325	+	f.table = []Artifact{
	326	+	{Name: "node20", Version: "1.0.0", Arch: "x64", URL: f.srv.URL + "/v1.0.0/node-v1.0.0-linux-x64.tar.gz", SHA256: f.sum, StripTop: true},
	327	+	{Name: "node24", Version: "2.0.0", Arch: "x64", URL: f.srv.URL + "/v2.0.0/node-v2.0.0-linux-x64.tar.gz", SHA256: f.sum, StripTop: true},
	328	+	{Name: "node20_alpine", Version: "1.0.0", Arch: "x64", URL: f.srv.URL + "/v1.0.0/alpine.tar.gz", SHA256: f.sumAlpine},
	329	+	}
	330	+	f.m = &Manager{Dir: f.dir, Arch: "x64", Table: f.table, Client: f.srv.Client()}
	331	+	return f
	332	+	}
	333	+
	334	+	func byName(set []sandbox.External) map[string]sandbox.External {
	335	+	m := map[string]sandbox.External{}
	336	+	for _, e := range set {
	337	+	m[e.Name] = e
	338	+	}
	339	+	return m
	340	+	}
	341	+
	342	+	func TestSyncDownloadsVerifiesAndReuses(t *testing.T) {
	343	+	f := newFixture(t)
	344	+	if err := f.m.Sync(context.Background()); err != nil {
	345	+	t.Fatal(err)
	346	+	}
	347	+	set := byName(f.m.Externals())
	348	+	n20, n24, al := set["node20"], set["node24"], set["node20_alpine"]
	349	+	if n20.Unavailable \|\| n24.Unavailable \|\| n20.Dir != filepath.Join(f.dir, "node20-1.0.0-x64") {
	350	+	t.Fatalf("set: %+v", set)
	351	+	}
	352	+	if n20.ID == n24.ID {
	353	+	t.Error("two tools share an ID")
	354	+	}
	355	+	if b, err := os.ReadFile(filepath.Join(n24.Dir, "bin", "node")); err != nil \|\| !strings.Contains(string(b), "echo v1") {
	356	+	t.Fatalf("node24: %q %v", b, err)
	357	+	}
	358	+	// The alpine archive has no top directory: bin/node is at its root.
	359	+	if al.Unavailable {
	360	+	t.Fatalf("alpine: %+v", al)
	361	+	}
	362	+	if b, err := os.ReadFile(filepath.Join(al.Dir, "bin", "node")); err != nil \|\| !strings.Contains(string(b), "echo alpine") {
	363	+	t.Fatalf("alpine node: %q %v", b, err)
	364	+	}
	365	+	hits := f.hits.Load()
	366	+	if hits != 3 {
	367	+	t.Fatalf("downloads: %d", hits)
	368	+	}
	369	+
	370	+	// A second start finds everything in place and downloads nothing.
	371	+	m2 := &Manager{Dir: f.dir, Arch: "x64", Table: f.table, Client: f.srv.Client()}
	372	+	if err := m2.Sync(context.Background()); err != nil {
	373	+	t.Fatal(err)
	374	+	}
	375	+	if f.hits.Load() != hits {
	376	+	t.Fatal("downloaded again")
	377	+	}
	378	+	if got := byName(m2.Externals())["node24"]; got.ID != n24.ID {
	379	+	t.Fatalf("ID changed: %q -> %q", n24.ID, got.ID)
	380	+	}
	381	+	}
	382	+
	383	+	// runStub runs the bin/node of an unavailable tool: it must fail with the reason
	384	+	// on stderr, from wherever the tool is mounted.
	385	+	func runStub(t *testing.T, e sandbox.External, want string) {
	386	+	t.Helper()
	387	+	if _, err := exec.LookPath("sh"); err != nil {
	388	+	t.Skip("sh needed")
	389	+	}
	390	+	// Run it through a symlinked path too, as a guest sees it at another place.
	391	+	cmd := exec.Command(filepath.Join(e.Dir, "bin", "node"), "-e", "x")
	392	+	var stderr, stdout bytes.Buffer
	393	+	cmd.Stderr, cmd.Stdout = &stderr, &stdout
	394	+	err := cmd.Run()
	395	+	if ee, ok := err.(*exec.ExitError); !ok \|\| ee.ExitCode() != 1 {
	396	+	t.Fatalf("the stub must exit 1, got %v", err)
	397	+	}
	398	+	if !strings.Contains(stderr.String(), want) \|\| stdout.Len() != 0 {
	399	+	t.Fatalf("stub stderr %q stdout %q, want %q", stderr.String(), stdout.String(), want)
	400	+	}
	401	+	if b, _ := os.ReadFile(filepath.Join(e.Dir, "bin", "node")); strings.Contains(string(b), want) {
	402	+	t.Fatalf("the stub holds the reason: %q", b)
	403	+	}
	404	+	}
	405	+
	406	+	func TestSyncRefusesAChecksumMismatchBeforeExtracting(t *testing.T) {
	407	+	f := newFixture(t)
	408	+	f.table[0].SHA256 = strings.Repeat("0", 64)
	409	+	f.m.Table = f.table
	410	+	err := f.m.Sync(context.Background())
	411	+	if err == nil \|\| !strings.Contains(err.Error(), "SHA-256 mismatch") {
	412	+	t.Fatalf("got %v", err)
	413	+	}
	414	+	n20 := byName(f.m.Externals())["node20"]
	415	+	if !n20.Unavailable {
	416	+	t.Fatal("a tool with a wrong checksum is available")
	417	+	}
	418	+	if _, err := os.Stat(filepath.Join(f.dir, "node20-1.0.0-x64")); err == nil {
	419	+	t.Fatal("extracted although the checksum differs")
	420	+	}
	421	+	reason, _ := os.ReadFile(filepath.Join(n20.Dir, sandbox.ExternalUnavailableFile))
	422	+	if !strings.Contains(string(reason), "SHA-256 mismatch") {
	423	+	t.Fatalf("reason: %q", reason)
	424	+	}
	425	+	// The other tool is unaffected.
	426	+	if byName(f.m.Externals())["node24"].Unavailable {
	427	+	t.Fatal("node24 lost to node20's failure")
	428	+	}
	429	+	// A checksum failure is a verdict: no retry is scheduled for it.
	430	+	if f.m.state["node20"].retry {
	431	+	t.Fatal("retry scheduled for a mismatch")
	432	+	}
	433	+	runStub(t, n20, "SHA-256 mismatch")
	434	+	ents, _ := os.ReadDir(f.dir)
	435	+	for _, e := range ents {
	436	+	if strings.HasPrefix(e.Name(), ".tmp-") {
	437	+	t.Fatalf("leftover %s", e.Name())
	438	+	}
	439	+	}
	440	+	}
	441	+
	442	+	func TestSyncRefusesAnUnsafeArchiveWithTheRightChecksum(t *testing.T) {
	443	+	f := newFixture(t)
	444	+	bad := targz(t, ent{name: "node-v1-linux-x64/bin/node", mode: 0o755, body: "x"}, ent{name: "node-v1-linux-x64/l", typ: tar.TypeSymlink, link: "/etc"})
	445	+	f.archive = bad
	446	+	s := sha256.Sum256(bad)
	447	+	f.table[0].SHA256, f.table[1].SHA256 = hex.EncodeToString(s[:]), hex.EncodeToString(s[:])
	448	+	f.m.Table = f.table
	449	+	err := f.m.Sync(context.Background())
	450	+	if err == nil \|\| !strings.Contains(err.Error(), "absolute") {
	451	+	t.Fatalf("got %v", err)
	452	+	}
	453	+	if _, err := os.Stat(filepath.Join(f.dir, "node20-1.0.0-x64")); err == nil {
	454	+	t.Fatal("unsafe archive left in place")
	455	+	}
	456	+	}
	457	+
	458	+	func TestSyncOfflineKeepsTheRunnerGoing(t *testing.T) {
	459	+	f := newFixture(t)
	460	+	f.srv.Close() // offline
	461	+	err := f.m.Sync(context.Background())
	462	+	if err == nil {
	463	+	t.Fatal("no error offline")
	464	+	}
	465	+	for _, n := range []string{"node20", "node24"} {
	466	+	e := byName(f.m.Externals())[n]
	467	+	if !e.Unavailable {
	468	+	t.Fatalf("%s available offline", n)
	469	+	}
	470	+	runStub(t, e, "is not available on this runner")
	471	+	}
	472	+	if !f.m.wantsRetry() {
	473	+	t.Fatal("a network failure must be retried")
	474	+	}
	475	+	}
	476	+
	477	+	func TestBadStatusAndRedirectToHTTP(t *testing.T) {
	478	+	f := newFixture(t)
	479	+	f.table[0].URL = f.srv.URL + "/missing.tar.gz"
	480	+	f.m.Table = f.table
	481	+	if err := f.m.Sync(context.Background()); err == nil \|\| !strings.Contains(err.Error(), "404") {
	482	+	t.Fatalf("got %v", err)
	483	+	}
	484	+	if !byName(f.m.Externals())["node20"].Unavailable {
	485	+	t.Fatal("404 gives a tool")
	486	+	}
	487	+
	488	+	// The default client refuses a redirect to plain http, and too many.
	489	+	check := newClient().CheckRedirect
	490	+	if err := check(&http.Request{URL: mustURL(t, "http://example.org/x")}, nil); err == nil {
	491	+	t.Error("redirect to http followed")
	492	+	}
	493	+	if err := check(&http.Request{URL: mustURL(t, "https://example.org/x")}, make([]*http.Request, 5)); err == nil {
	494	+	t.Error("redirect loop followed")
	495	+	}
	496	+	if err := check(&http.Request{URL: mustURL(t, "https://example.org/x")}, make([]*http.Request, 1)); err != nil {
	497	+	t.Error(err)
	498	+	}
	499	+	// A plain http URL in the table is refused outright.
	500	+	m2 := &Manager{Dir: filepath.Join(t.TempDir(), "e"), Arch: "x64", Client: f.srv.Client()}
	501	+	m2.Table = []Artifact{{Name: "node20", Version: "1", Arch: "x64", URL: "http://example.invalid/a.tar.gz", SHA256: f.sum}}
	502	+	if err := m2.Sync(context.Background()); err == nil \|\| !strings.Contains(err.Error(), "not an https URL") {
	503	+	t.Fatalf("got %v", err)
	504	+	}
	505	+	}
	506	+
	507	+	func mustURL(t testing.T, s string) url.URL {
	508	+	t.Helper()
	509	+	u, err := url.Parse(s)
	510	+	if err != nil {
	511	+	t.Fatal(err)
	512	+	}
	513	+	return u
	514	+	}
	515	+
	516	+	func TestSyncKeepsOnlyPinnedVersions(t *testing.T) {
	517	+	f := newFixture(t)
	518	+	if err := f.m.Sync(context.Background()); err != nil {
	519	+	t.Fatal(err)
	520	+	}
	521	+	// Leftovers: an old version, an interrupted download, a stray file, an
	522	+	// incomplete directory under a pinned name.
	523	+	for _, d := range []string{"node20-0.9.0-x64", ".tmp-123", "node24-2.0.0-arm64", "node-old"} {
	524	+	if err := os.MkdirAll(filepath.Join(f.dir, d, "bin"), 0o755); err != nil {
	525	+	t.Fatal(err)
	526	+	}
	527	+	}
	528	+	if err := os.WriteFile(filepath.Join(f.dir, "stray"), []byte("x"), 0o644); err != nil {
	529	+	t.Fatal(err)
	530	+	}
	531	+	if err := f.m.Sync(context.Background()); err != nil {
	532	+	t.Fatal(err)
	533	+	}
	534	+	var got []string
	535	+	ents, _ := os.ReadDir(f.dir)
	536	+	for _, e := range ents {
	537	+	got = append(got, e.Name())
	538	+	}
	539	+	want := "node20-1.0.0-x64 node20_alpine-1.0.0-x64 node24-2.0.0-x64"
	540	+	if strings.Join(got, " ") != want {
	541	+	t.Fatalf("kept %v, want %s", got, want)
	542	+	}
	543	+	}
	544	+
	545	+	func TestSyncRebuildsAnIncompleteDirectory(t *testing.T) {
	546	+	f := newFixture(t)
	547	+	final := filepath.Join(f.dir, "node20-1.0.0-x64")
	548	+	if err := os.MkdirAll(filepath.Join(final, "bin"), 0o755); err != nil { // no bin/node
	549	+	t.Fatal(err)
	550	+	}
	551	+	if err := f.m.Sync(context.Background()); err != nil {
	552	+	t.Fatal(err)
	553	+	}
	554	+	if !isReady(final) {
	555	+	t.Fatal("incomplete directory kept")
	556	+	}
	557	+	}
	558	+
	559	+	func TestANewVersionChangesTheIDAndDropsTheOldDirectory(t *testing.T) {
	560	+	f := newFixture(t)
	561	+	if err := f.m.Sync(context.Background()); err != nil {
	562	+	t.Fatal(err)
	563	+	}
	564	+	old := byName(f.m.Externals())["node24"]
	565	+	f.table[1].Version = "2.1.0"
	566	+	f.table[1].URL = f.srv.URL + "/v2.1.0/node-v2.1.0-linux-x64.tar.gz"
	567	+	if err := f.m.Sync(context.Background()); err != nil {
	568	+	t.Fatal(err)
	569	+	}
	570	+	n := byName(f.m.Externals())["node24"]
	571	+	if n.ID == old.ID \|\| n.Dir == old.Dir {
	572	+	t.Fatalf("same tool after an upgrade: %+v %+v", old, n)
	573	+	}
	574	+	if _, err := os.Stat(old.Dir); err == nil {
	575	+	t.Fatal("old version kept")
	576	+	}
	577	+	}
	578	+
	579	+	func TestNoBuildForTheArchitecture(t *testing.T) {
	580	+	f := newFixture(t)
	581	+	f.m.Arch = "arm64" // the fixture table is x64 only
	582	+	if err := f.m.Sync(context.Background()); err != nil {
	583	+	t.Fatal(err)
	584	+	}
	585	+	for _, e := range f.m.Externals() {
	586	+	reason, _ := os.ReadFile(filepath.Join(e.Dir, sandbox.ExternalUnavailableFile))
	587	+	if !e.Unavailable \|\| !strings.Contains(string(reason), "no "+e.Name+" build for linux-arm64") {
	588	+	t.Errorf("%s: %+v %q", e.Name, e, reason)
	589	+	}
	590	+	}
	591	+	if f.hits.Load() != 0 {
	592	+	t.Error("downloaded for the wrong architecture")
	593	+	}
	594	+	}
	595	+
	596	+	func TestBeforeTheFirstSyncTheToolsAreBeingDownloaded(t *testing.T) {
	597	+	f := newFixture(t)
	598	+	set := f.m.Externals()
	599	+	if len(set) != 3 {
	600	+	t.Fatalf("set: %+v", set)
	601	+	}
	602	+	for _, e := range set {
	603	+	reason, _ := os.ReadFile(filepath.Join(e.Dir, sandbox.ExternalUnavailableFile))
	604	+	if !e.Unavailable \|\| !strings.Contains(string(reason), "still being downloaded") {
	605	+	t.Errorf("%s: %q", e.Name, reason)
	606	+	}
	607	+	}
	608	+	}
	609	+
	610	+	func TestFailedDownloadIsRetriedInTheBackground(t *testing.T) {
	611	+	f := newFixture(t)
	612	+	var up atomic.Bool
	613	+	srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	614	+	if !up.Load() {
	615	+	http.Error(w, "down", http.StatusServiceUnavailable)
	616	+	return
	617	+	}
	618	+	_, _ = w.Write(f.archive)
	619	+	}))
	620	+	defer srv.Close()
	621	+	m := &Manager{Dir: f.dir, Arch: "x64", Client: srv.Client(), RetryEvery: time.Millisecond}
	622	+	m.Table = []Artifact{{Name: "node24", Version: "2.0.0", Arch: "x64", URL: srv.URL + "/a-linux-x64.tar.gz", SHA256: f.sum, StripTop: true}}
	623	+	ctx, cancel := context.WithCancel(context.Background())
	624	+	defer cancel()
	625	+	m.Start(ctx, time.Minute)
	626	+	if !m.Externals()[0].Unavailable {
	627	+	t.Fatal("available while the server is down")
	628	+	}
	629	+	up.Store(true)
	630	+	deadline := time.Now().Add(10 * time.Second)
	631	+	for time.Now().Before(deadline) {
	632	+	time.Sleep(20 * time.Millisecond)
	633	+	if !m.Externals()[0].Unavailable {
	634	+	return
	635	+	}
	636	+	}
	637	+	t.Fatal("never retried")
	638	+	}
	639	+
	640	+	func TestStartDoesNotWaitLongerThanAsked(t *testing.T) {
	641	+	block := make(chan struct{})
	642	+	srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { <-block }))
	643	+	defer srv.Close()
	644	+	m := &Manager{Dir: filepath.Join(t.TempDir(), "e"), Arch: "x64", Client: srv.Client()}
	645	+	m.Table = []Artifact{{Name: "node24", Version: "2.0.0", Arch: "x64", URL: srv.URL + "/a.tar.gz", SHA256: strings.Repeat("a", 64)}}
	646	+	ctx, cancel := context.WithCancel(context.Background())
	647	+	start := time.Now()
	648	+	m.Start(ctx, 100*time.Millisecond)
	649	+	if d := time.Since(start); d > 5*time.Second {
	650	+	t.Fatalf("waited %v", d)
	651	+	}
	652	+	// The download goes on in the background; stop it before the directory goes.
	653	+	cancel()
	654	+	close(block)
	655	+	for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(10 * time.Millisecond) {
	656	+	m.mu.Lock()
	657	+	busy := m.syncing
	658	+	m.mu.Unlock()
	659	+	if !busy {
	660	+	return
	661	+	}
	662	+	}
	663	+	t.Fatal("the background sync did not stop")
	664	+	}
	665	+
	666	+	// ---- the directories of unavailable tools
	667	+
	668	+	func TestStubIsBuiltOnceAndNeverRewrittenInPlace(t *testing.T) {
	669	+	m := &Manager{Dir: filepath.Join(t.TempDir(), "e"), Arch: "x64"}
	670	+	e := m.unavailable("node24", "why", false)
	671	+	node := filepath.Join(e.ext.Dir, "bin", "node")
	672	+	file := filepath.Join(e.ext.Dir, sandbox.ExternalUnavailableFile)
	673	+	ino := func(p string) uint64 {
	674	+	st, err := os.Stat(p)
	675	+	if err != nil {
	676	+	t.Fatal(err)
	677	+	}
	678	+	return st.Sys().(*syscall.Stat_t).Ino
	679	+	}
	680	+	n0, f0 := ino(node), ino(file)
	681	+
	682	+	// Again, from this process and from a fresh one (a restart): same files.
	683	+	m.unavailable("node24", "why", false)
	684	+	(&Manager{Dir: m.Dir, Arch: "x64"}).unavailable("node24", "why", false)
	685	+	if ino(node) != n0 \|\| ino(file) != f0 {
	686	+	t.Fatal("an existing, correct stub was rewritten")
	687	+	}
	688	+
	689	+	// A damaged or missing file is replaced by a rename (a new inode, so a reader
	690	+	// of the old one never sees it change), with the right mode.
	691	+	if err := os.WriteFile(node, nil, 0o755); err != nil {
	692	+	t.Fatal(err)
	693	+	}
	694	+	if err := os.Remove(file); err != nil {
	695	+	t.Fatal(err)
	696	+	}
	697	+	(&Manager{Dir: m.Dir, Arch: "x64"}).unavailable("node24", "why", false)
	698	+	if b, _ := os.ReadFile(node); string(b) != unavailableStub {
	699	+	t.Fatalf("stub not repaired: %q", b)
	700	+	}
	701	+	if b, _ := os.ReadFile(file); string(b) != "why\n" {
	702	+	t.Fatalf("reason not repaired: %q", b)
	703	+	}
	704	+	if st, _ := os.Stat(node); st.Mode().Perm() != 0o755 \|\| ino(node) == n0 {
	705	+	t.Fatalf("repair in place: %v", st.Mode())
	706	+	}
	707	+	runStub(t, e.ext, "why")
	708	+	left, _ := filepath.Glob(filepath.Join(e.ext.Dir, "", ".tmp-"))
	709	+	left2, _ := filepath.Glob(filepath.Join(e.ext.Dir, ".tmp-*"))
	710	+	if len(left)+len(left2) != 0 {
	711	+	t.Fatalf("temporary files left: %v %v", left, left2)
	712	+	}
	713	+	}
	714	+
	715	+	// While Externals() and Sync() run in several goroutines, a reader that walks
	716	+	// the directories of the unavailable tools (as the build of a VM's externals
	717	+	// disk does) never sees a missing, empty or partial file.
	718	+	func TestConcurrentExternalsAndSyncNeverShowAHalfWrittenStub(t *testing.T) {
	719	+	f := newFixture(t)
	720	+	f.srv.Close() // offline: every tool is unavailable
	721	+	ctx := context.Background()
	722	+	stop := make(chan struct{})
	723	+	var wg sync.WaitGroup
	724	+	var bad atomic.Value
	725	+	check := func(dir string) {
	726	+	for _, c := range []struct{ name, want string }{{"bin/node", unavailableStub}} {
	727	+	b, err := os.ReadFile(filepath.Join(dir, c.name))
	728	+	if err != nil {
	729	+	bad.Store(fmt.Sprintf("%s/%s: %v", dir, c.name, err))
	730	+	} else if string(b) != c.want {
	731	+	bad.Store(fmt.Sprintf("%s/%s: %q", dir, c.name, b))
	732	+	}
	733	+	}
	734	+	b, err := os.ReadFile(filepath.Join(dir, sandbox.ExternalUnavailableFile))
	735	+	if err != nil \|\| len(b) == 0 \|\| !strings.HasSuffix(string(b), "\n") {
	736	+	bad.Store(fmt.Sprintf("%s/UNAVAILABLE: %q %v", dir, b, err))
	737	+	}
	738	+	}
	739	+	for i := 0; i < 4; i++ {
	740	+	wg.Add(1)
	741	+	go func() {
	742	+	defer wg.Done()
	743	+	for {
	744	+	select {
	745	+	case <-stop:
	746	+	return
	747	+	default:
	748	+	}
	749	+	for _, e := range f.m.Externals() {
	750	+	check(e.Dir) // what a driver would hand to the disk build
	751	+	}
	752	+	}
	753	+	}()
	754	+	}
	755	+	wg.Add(1)
	756	+	go func() {
	757	+	defer wg.Done()
	758	+	for {
	759	+	select {
	760	+	case <-stop:
	761	+	return
	762	+	default:
	763	+	}
	764	+	dirs, _ := filepath.Glob(filepath.Join(f.dir, unavailableDir, "*"))
	765	+	for _, d := range dirs {
	766	+	if !strings.Contains(filepath.Base(d), ".tmp-") {
	767	+	check(d)
	768	+	}
	769	+	}
	770	+	}
	771	+	}()
	772	+	for i := 0; i < 5; i++ {
	773	+	_ = f.m.Sync(ctx)
	774	+	}
	775	+	close(stop)
	776	+	wg.Wait()
	777	+	if v := bad.Load(); v != nil {
	778	+	t.Fatalf("a reader saw an incomplete stub: %v", v)
	779	+	}
	780	+	}
	781	+
	782	+	func TestPruneKeepsStubDirectoriesHandedOutInTheLastHour(t *testing.T) {
	783	+	f := newFixture(t)
	784	+	f.srv.Close()
	785	+	_ = f.m.Sync(context.Background()) // offline: stubs for the three tools
	786	+	root := filepath.Join(f.dir, unavailableDir)
	787	+	mk := func(name string, age time.Duration) string {
	788	+	p := filepath.Join(root, name)
	789	+	if err := os.MkdirAll(filepath.Join(p, "bin"), 0o755); err != nil {
	790	+	t.Fatal(err)
	791	+	}
	792	+	old := time.Now().Add(-age)
	793	+	if err := os.Chtimes(p, old, old); err != nil {
	794	+	t.Fatal(err)
	795	+	}
	796	+	return p
	797	+	}
	798	+	fresh := mk("node24-unavailable-aaaaaaaaaaaaaaaa", 5*time.Minute) // handed to a running job
	799	+	stale := mk("node24-unavailable-bbbbbbbbbbbbbbbb", 2*time.Hour)
	800	+
	801	+	// A reason that is not current any more but was handed out again just now
	802	+	// (Externals touches what it returns) survives; one that was not, does not.
	803	+	cur := f.m.Externals()[0]
	804	+	old := time.Now().Add(-3 * time.Hour)
	805	+	if err := os.Chtimes(cur.Dir, old, old); err != nil {
	806	+	t.Fatal(err)
	807	+	}
	808	+	f.m.Externals() // handed out: touched
	809	+	if st, _ := os.Stat(cur.Dir); time.Since(st.ModTime()) > time.Minute {
	810	+	t.Fatalf("Externals did not touch %s", cur.Dir)
	811	+	}
	812	+	// The set changes (the reasons are not the same any more), the touched
	813	+	// directory is no longer current.
	814	+	f.m.mu.Lock()
	815	+	f.m.state = map[string]entry{}
	816	+	f.m.mu.Unlock()
	817	+	f.m.prune(zerowrap.Logger{}, map[string]bool{})
	818	+	for _, p := range []string{fresh, cur.Dir} {
	819	+	if _, err := os.Stat(p); err != nil {
	820	+	t.Errorf("%s was pruned although handed out in the last hour", p)
	821	+	}
	822	+	}
	823	+	if _, err := os.Stat(stale); err == nil {
	824	+	t.Error("a stale stub directory was kept")
	825	+	}
	826	+	}
	827	+
	828	+	// ---- retries
	829	+
	830	+	func TestRetriesRunOnATickerWithoutAnyJob(t *testing.T) {
	831	+	f := newFixture(t)
	832	+	var up atomic.Bool
	833	+	var calls atomic.Int64
	834	+	srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	835	+	calls.Add(1)
	836	+	if !up.Load() {
	837	+	http.Error(w, "down", http.StatusServiceUnavailable)
	838	+	return
	839	+	}
	840	+	_, _ = w.Write(f.archive)
	841	+	}))
	842	+	defer srv.Close()
	843	+	m := &Manager{Dir: f.dir, Arch: "x64", Client: srv.Client(), RetryEvery: 20 * time.Millisecond}
	844	+	m.Table = []Artifact{{Name: "node24", Version: "2.0.0", Arch: "x64", URL: srv.URL + "/a-linux-x64.tar.gz", SHA256: f.sum, StripTop: true}}
	845	+	ctx, cancel := context.WithCancel(context.Background())
	846	+	m.Start(ctx, time.Minute)
	847	+
	848	+	// Nobody calls Externals(): the ticker alone retries.
	849	+	deadline := time.Now().Add(10 * time.Second)
	850	+	for calls.Load() < 3 && time.Now().Before(deadline) {
	851	+	time.Sleep(10 * time.Millisecond)
	852	+	}
	853	+	if calls.Load() < 3 {
	854	+	t.Fatalf("%d attempts without any job", calls.Load())
	855	+	}
	856	+	up.Store(true)
	857	+	for time.Now().Before(deadline) {
	858	+	m.mu.Lock()
	859	+	ok := !m.state["node24"].ext.Unavailable
	860	+	m.mu.Unlock()
	861	+	if ok {
	862	+	break
	863	+	}
	864	+	time.Sleep(10 * time.Millisecond)
	865	+	}
	866	+	if m.Externals()[0].Unavailable {
	867	+	t.Fatal("never recovered")
	868	+	}
	869	+
	870	+	// Once everything is there the ticker stops downloading, and with the context
	871	+	// it stops altogether.
	872	+	before := calls.Load()
	873	+	time.Sleep(150 * time.Millisecond)
	874	+	if calls.Load() != before {
	875	+	t.Fatal("the ticker keeps requesting what it has")
	876	+	}
	877	+	cancel()
	878	+	m.mu.Lock()
	879	+	m.state["node24"] = entry{ext: sandbox.External{Name: "node24", Unavailable: true}, retry: true}
	880	+	m.mu.Unlock()
	881	+	time.Sleep(150 * time.Millisecond)
	882	+	if calls.Load() != before {
	883	+	t.Fatal("a retry ran after the context ended")
	884	+	}
	885	+	}
	886	+
	887	+	// A verdict (wrong checksum) is not retried by the ticker.
	888	+	func TestNoRetryForAVerdict(t *testing.T) {
	889	+	f := newFixture(t)
	890	+	f.table[0].SHA256 = strings.Repeat("0", 64)
	891	+	m := &Manager{Dir: f.dir, Arch: "x64", Table: f.table[:1], Client: f.srv.Client(), RetryEvery: 10 * time.Millisecond}
	892	+	ctx, cancel := context.WithCancel(context.Background())
	893	+	defer cancel()
	894	+	m.Start(ctx, time.Minute)
	895	+	hits := f.hits.Load()
	896	+	time.Sleep(150 * time.Millisecond)
	897	+	if f.hits.Load() != hits {
	898	+	t.Fatal("a checksum mismatch was retried")
	899	+	}
	900	+	}
	901	+
	902	+	// ---- the real archives, when they are at hand
	903	+
	904	+	// The actions/alpine_nodejs archives have no top directory (./bin/node). Set
	905	+	// ERMITE_RUNNER_TEST_ALPINE_DIR, or leave the files in /tmp/alpine-node.
	906	+	func TestRealAlpineArchives(t *testing.T) {
	907	+	dir := os.Getenv("ERMITE_RUNNER_TEST_ALPINE_DIR")
	908	+	if dir == "" {
	909	+	dir = "/tmp/alpine-node"
	910	+	}
	911	+	ran := false
	912	+	for _, a := range Pinned {
	913	+	if !strings.HasSuffix(a.Name, "_alpine") {
	914	+	continue
	915	+	}
	916	+	archive := filepath.Join(dir, filepath.Base(a.URL))
	917	+	b, err := os.ReadFile(archive)
	918	+	if err != nil {
	919	+	t.Logf("%s: %v", a.Name, err)
	920	+	continue
	921	+	}
	922	+	ran = true
	923	+	t.Run(a.Name, func(t *testing.T) {
	924	+	sum := sha256.Sum256(b)
	925	+	if err := VerifySum(hex.EncodeToString(sum[:]), a.SHA256); err != nil {
	926	+	t.Fatalf("the pinned checksum is not the file's: %v", err)
	927	+	}
	928	+	dst := t.TempDir()
	929	+	if err := extractTarGz(archive, dst, a.StripTop); err != nil {
	930	+	t.Fatal(err)
	931	+	}
	932	+	if !isReady(dst) {
	933	+	t.Fatal("no executable bin/node")
	934	+	}
	935	+	if st, err := os.Stat(filepath.Join(dst, "LICENSE")); err != nil \|\| st.Mode().Perm() != 0o644 {
	936	+	t.Fatalf("LICENSE: %v %v", st, err)
	937	+	}
	938	+	// Taking the first component off would lose bin/node.
	939	+	wrong := t.TempDir()
	940	+	_ = extractTarGz(archive, wrong, true)
	941	+	if isReady(wrong) {
	942	+	t.Fatal("the alpine archive must not be unpacked with the top directory stripped")
	943	+	}
	944	+	})
	945	+	}
	946	+	if !ran {
	947	+	t.Skipf("no alpine archive in %s", dir)
	948	+	}
	949	+	}

A internal/externals/extract.go +223 -0

@@ -0,0 +1,223 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package externals
	4	+
	5	+	import (
	6	+	"archive/tar"
	7	+	"compress/gzip"
	8	+	"errors"
	9	+	"fmt"
	10	+	"io"
	11	+	"io/fs"
	12	+	"os"
	13	+	"path"
	14	+	"path/filepath"
	15	+	"strings"
	16	+	)
	17	+
	18	+	// Limits on what an archive may unpack to. They are far above a Node.js
	19	+	// release (about 6,500 entries and 220 MiB) and exist to bound a corrupt or
	20	+	// hostile archive that still has the right checksum.
	21	+	const (
	22	+	maxEntries = 200_000
	23	+	maxUnpack = 2 << 30
	24	+	)
	25	+
	26	+	// extractTarGz unpacks the .tar.gz at archive into the existing directory dest.
	27	+	// With stripTop the first path component is dropped (the nodejs.org archives
	28	+	// hold one top directory, node-vX-linux-x64); without it the names are taken as
	29	+	// they are (the actions/alpine_nodejs archives start at ./bin/node). It refuses what could write or point outside dest:
	30	+	//
	31	+	// - absolute names and names with ".." components;
	32	+	// - anything but directories, regular files, symbolic links and hard links;
	33	+	// - symbolic links that are absolute or whose target climbs with ".." anywhere
	34	+	// but at the start (so that the target cannot walk back out through another
	35	+	// link), and entries below a symbolic link;
	36	+	// - hard links to something that is not already unpacked.
	37	+	//
	38	+	// Writes go through os.Root, which also refuses to follow a link out of dest.
	39	+	// File modes are reduced to 0755 / 0644: the owner of the files is whoever runs
	40	+	// the runner, and the sandboxes only read and execute them.
	41	+	func extractTarGz(archive, dest string, stripTop bool) error {
	42	+	f, err := os.Open(archive)
	43	+	if err != nil {
	44	+	return err
	45	+	}
	46	+	defer func() { _ = f.Close() }()
	47	+	gz, err := gzip.NewReader(f)
	48	+	if err != nil {
	49	+	return fmt.Errorf("not a gzip file: %w", err)
	50	+	}
	51	+	defer func() { _ = gz.Close() }()
	52	+	root, err := os.OpenRoot(dest)
	53	+	if err != nil {
	54	+	return err
	55	+	}
	56	+	defer func() { _ = root.Close() }()
	57	+
	58	+	tr := tar.NewReader(gz)
	59	+	links := map[string]bool{} // unpacked symbolic links
	60	+	var entries int
	61	+	var total int64
	62	+	for {
	63	+	h, err := tr.Next()
	64	+	if errors.Is(err, io.EOF) {
	65	+	return openDirs(dest)
	66	+	}
	67	+	if err != nil {
	68	+	return fmt.Errorf("read archive: %w", err)
	69	+	}
	70	+	if entries++; entries > maxEntries {
	71	+	return fmt.Errorf("archive has more than %d entries", maxEntries)
	72	+	}
	73	+	if h.Typeflag == tar.TypeXGlobalHeader {
	74	+	continue
	75	+	}
	76	+	name, err := cleanName(h.Name, stripTop)
	77	+	if err != nil {
	78	+	return err
	79	+	}
	80	+	if name == "" { // the top directory itself, or "./"
	81	+	continue
	82	+	}
	83	+	if under(links, name) {
	84	+	return fmt.Errorf("entry %q is below a symbolic link", h.Name)
	85	+	}
	86	+	switch h.Typeflag {
	87	+	case tar.TypeDir:
	88	+	err = root.MkdirAll(name, 0o755)
	89	+	case tar.TypeReg:
	90	+	if total += h.Size; total > maxUnpack {
	91	+	return fmt.Errorf("archive unpacks to more than %d bytes", int64(maxUnpack))
	92	+	}
	93	+	err = writeFile(root, name, tr, h.Mode)
	94	+	case tar.TypeSymlink:
	95	+	if err = checkLinkTarget(name, h.Linkname); err == nil {
	96	+	if err = root.MkdirAll(path.Dir(name), 0o755); err == nil {
	97	+	if err = root.Symlink(h.Linkname, name); err == nil {
	98	+	links[name] = true
	99	+	}
	100	+	}
	101	+	}
	102	+	case tar.TypeLink:
	103	+	var old string
	104	+	if old, err = cleanName(h.Linkname, stripTop); err == nil {
	105	+	if old == "" \|\| under(links, old) {
	106	+	err = fmt.Errorf("hard link %q points at %q", h.Name, h.Linkname)
	107	+	} else if err = root.MkdirAll(path.Dir(name), 0o755); err == nil {
	108	+	err = root.Link(old, name)
	109	+	}
	110	+	}
	111	+	default:
	112	+	err = fmt.Errorf("entry %q has the unsupported type %q", h.Name, string(h.Typeflag))
	113	+	}
	114	+	if err != nil {
	115	+	return fmt.Errorf("unpack %q: %w", h.Name, err)
	116	+	}
	117	+	}
	118	+	}
	119	+
	120	+	func writeFile(root *os.Root, name string, r io.Reader, mode int64) error {
	121	+	if err := root.MkdirAll(path.Dir(name), 0o755); err != nil {
	122	+	return err
	123	+	}
	124	+	perm := os.FileMode(0o644)
	125	+	if mode&0o111 != 0 {
	126	+	perm = 0o755
	127	+	}
	128	+	f, err := root.OpenFile(name, os.O_WRONLY\|os.O_CREATE\|os.O_EXCL, perm)
	129	+	if err != nil {
	130	+	return err
	131	+	}
	132	+	if _, err := io.Copy(f, r); err != nil {
	133	+	_ = f.Close()
	134	+	return err
	135	+	}
	136	+	if err := f.Chmod(perm); err != nil { // the umask may have removed bits
	137	+	_ = f.Close()
	138	+	return err
	139	+	}
	140	+	return f.Close()
	141	+	}
	142	+
	143	+	// cleanName validates an archive path and returns it relative to the tree: "./"
	144	+	// prefixes and empty or "." components are dropped, and with stripTop so is the
	145	+	// first component. It returns "" for the top of the tree itself.
	146	+	func cleanName(name string, stripTop bool) (string, error) {
	147	+	if name == "" \|\| strings.ContainsRune(name, 0) \|\| strings.ContainsRune(name, '\\') {
	148	+	return "", fmt.Errorf("invalid entry name %q", name)
	149	+	}
	150	+	if strings.HasPrefix(name, "/") {
	151	+	return "", fmt.Errorf("entry %q has an absolute path", name)
	152	+	}
	153	+	var parts []string
	154	+	for _, p := range strings.Split(name, "/") {
	155	+	switch p {
	156	+	case "", ".":
	157	+	case "..":
	158	+	return "", fmt.Errorf("entry %q has a .. component", name)
	159	+	default:
	160	+	parts = append(parts, p)
	161	+	}
	162	+	}
	163	+	if stripTop {
	164	+	if len(parts) <= 1 {
	165	+	return "", nil
	166	+	}
	167	+	parts = parts[1:]
	168	+	}
	169	+	return strings.Join(parts, "/"), nil
	170	+	}
	171	+
	172	+	// checkLinkTarget accepts a relative target whose ".." components all come
	173	+	// first and that stays inside the tree. name is the cleaned path of the link.
	174	+	// The lexical rule is exact because no entry is ever unpacked below a symbolic
	175	+	// link.
	176	+	func checkLinkTarget(name, target string) error {
	177	+	if target == "" \|\| strings.ContainsRune(target, 0) \|\| strings.HasPrefix(target, "/") {
	178	+	return fmt.Errorf("symbolic link %q has the target %q: absolute or empty", name, target)
	179	+	}
	180	+	// depth is how many directories lie between the tree's top and the link.
	181	+	depth := 0
	182	+	if dir := path.Dir(name); dir != "." {
	183	+	depth = strings.Count(dir, "/") + 1
	184	+	}
	185	+	up, seenName := 0, false
	186	+	for _, p := range strings.Split(target, "/") {
	187	+	switch p {
	188	+	case "", ".":
	189	+	case "..":
	190	+	if seenName {
	191	+	return fmt.Errorf("symbolic link %q: target %q climbs after a name", name, target)
	192	+	}
	193	+	up++
	194	+	default:
	195	+	seenName = true
	196	+	}
	197	+	}
	198	+	if up > depth {
	199	+	return fmt.Errorf("symbolic link %q: target %q leaves the archive", name, target)
	200	+	}
	201	+	return nil
	202	+	}
	203	+
	204	+	// under reports whether name, or one of its parents, is an unpacked symbolic link.
	205	+	func under(links map[string]bool, name string) bool {
	206	+	for p := name; p != "." && p != "/" && p != ""; p = path.Dir(p) {
	207	+	if links[p] {
	208	+	return true
	209	+	}
	210	+	}
	211	+	return false
	212	+	}
	213	+
	214	+	// openDirs gives every directory the mode 0755, whatever the umask of the
	215	+	// runner was when it created them: the sandboxes read the tree as another user.
	216	+	func openDirs(dest string) error {
	217	+	return filepath.WalkDir(dest, func(p string, d fs.DirEntry, err error) error {
	218	+	if err != nil \|\| !d.IsDir() {
	219	+	return err
	220	+	}
	221	+	return os.Chmod(p, 0o755)
	222	+	})
	223	+	}

@@ -0,0 +1,223 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package externals
	4	+
	5	+	import (
	6	+	"archive/tar"
	7	+	"compress/gzip"
	8	+	"errors"
	9	+	"fmt"
	10	+	"io"
	11	+	"io/fs"
	12	+	"os"
	13	+	"path"
	14	+	"path/filepath"
	15	+	"strings"
	16	+	)
	17	+
	18	+	// Limits on what an archive may unpack to. They are far above a Node.js
	19	+	// release (about 6,500 entries and 220 MiB) and exist to bound a corrupt or
	20	+	// hostile archive that still has the right checksum.
	21	+	const (
	22	+	maxEntries = 200_000
	23	+	maxUnpack = 2 << 30
	24	+	)
	25	+
	26	+	// extractTarGz unpacks the .tar.gz at archive into the existing directory dest.
	27	+	// With stripTop the first path component is dropped (the nodejs.org archives
	28	+	// hold one top directory, node-vX-linux-x64); without it the names are taken as
	29	+	// they are (the actions/alpine_nodejs archives start at ./bin/node). It refuses what could write or point outside dest:
	30	+	//
	31	+	// - absolute names and names with ".." components;
	32	+	// - anything but directories, regular files, symbolic links and hard links;
	33	+	// - symbolic links that are absolute or whose target climbs with ".." anywhere
	34	+	// but at the start (so that the target cannot walk back out through another
	35	+	// link), and entries below a symbolic link;
	36	+	// - hard links to something that is not already unpacked.
	37	+	//
	38	+	// Writes go through os.Root, which also refuses to follow a link out of dest.
	39	+	// File modes are reduced to 0755 / 0644: the owner of the files is whoever runs
	40	+	// the runner, and the sandboxes only read and execute them.
	41	+	func extractTarGz(archive, dest string, stripTop bool) error {
	42	+	f, err := os.Open(archive)
	43	+	if err != nil {
	44	+	return err
	45	+	}
	46	+	defer func() { _ = f.Close() }()
	47	+	gz, err := gzip.NewReader(f)
	48	+	if err != nil {
	49	+	return fmt.Errorf("not a gzip file: %w", err)
	50	+	}
	51	+	defer func() { _ = gz.Close() }()
	52	+	root, err := os.OpenRoot(dest)
	53	+	if err != nil {
	54	+	return err
	55	+	}
	56	+	defer func() { _ = root.Close() }()
	57	+
	58	+	tr := tar.NewReader(gz)
	59	+	links := map[string]bool{} // unpacked symbolic links
	60	+	var entries int
	61	+	var total int64
	62	+	for {
	63	+	h, err := tr.Next()
	64	+	if errors.Is(err, io.EOF) {
	65	+	return openDirs(dest)
	66	+	}
	67	+	if err != nil {
	68	+	return fmt.Errorf("read archive: %w", err)
	69	+	}
	70	+	if entries++; entries > maxEntries {
	71	+	return fmt.Errorf("archive has more than %d entries", maxEntries)
	72	+	}
	73	+	if h.Typeflag == tar.TypeXGlobalHeader {
	74	+	continue
	75	+	}
	76	+	name, err := cleanName(h.Name, stripTop)
	77	+	if err != nil {
	78	+	return err
	79	+	}
	80	+	if name == "" { // the top directory itself, or "./"
	81	+	continue
	82	+	}
	83	+	if under(links, name) {
	84	+	return fmt.Errorf("entry %q is below a symbolic link", h.Name)
	85	+	}
	86	+	switch h.Typeflag {
	87	+	case tar.TypeDir:
	88	+	err = root.MkdirAll(name, 0o755)
	89	+	case tar.TypeReg:
	90	+	if total += h.Size; total > maxUnpack {
	91	+	return fmt.Errorf("archive unpacks to more than %d bytes", int64(maxUnpack))
	92	+	}
	93	+	err = writeFile(root, name, tr, h.Mode)
	94	+	case tar.TypeSymlink:
	95	+	if err = checkLinkTarget(name, h.Linkname); err == nil {
	96	+	if err = root.MkdirAll(path.Dir(name), 0o755); err == nil {
	97	+	if err = root.Symlink(h.Linkname, name); err == nil {
	98	+	links[name] = true
	99	+	}
	100	+	}
	101	+	}
	102	+	case tar.TypeLink:
	103	+	var old string
	104	+	if old, err = cleanName(h.Linkname, stripTop); err == nil {
	105	+	if old == "" \|\| under(links, old) {
	106	+	err = fmt.Errorf("hard link %q points at %q", h.Name, h.Linkname)
	107	+	} else if err = root.MkdirAll(path.Dir(name), 0o755); err == nil {
	108	+	err = root.Link(old, name)
	109	+	}
	110	+	}
	111	+	default:
	112	+	err = fmt.Errorf("entry %q has the unsupported type %q", h.Name, string(h.Typeflag))
	113	+	}
	114	+	if err != nil {
	115	+	return fmt.Errorf("unpack %q: %w", h.Name, err)
	116	+	}
	117	+	}
	118	+	}
	119	+
	120	+	func writeFile(root *os.Root, name string, r io.Reader, mode int64) error {
	121	+	if err := root.MkdirAll(path.Dir(name), 0o755); err != nil {
	122	+	return err
	123	+	}
	124	+	perm := os.FileMode(0o644)
	125	+	if mode&0o111 != 0 {
	126	+	perm = 0o755
	127	+	}
	128	+	f, err := root.OpenFile(name, os.O_WRONLY\|os.O_CREATE\|os.O_EXCL, perm)
	129	+	if err != nil {
	130	+	return err
	131	+	}
	132	+	if _, err := io.Copy(f, r); err != nil {
	133	+	_ = f.Close()
	134	+	return err
	135	+	}
	136	+	if err := f.Chmod(perm); err != nil { // the umask may have removed bits
	137	+	_ = f.Close()
	138	+	return err
	139	+	}
	140	+	return f.Close()
	141	+	}
	142	+
	143	+	// cleanName validates an archive path and returns it relative to the tree: "./"
	144	+	// prefixes and empty or "." components are dropped, and with stripTop so is the
	145	+	// first component. It returns "" for the top of the tree itself.
	146	+	func cleanName(name string, stripTop bool) (string, error) {
	147	+	if name == "" \|\| strings.ContainsRune(name, 0) \|\| strings.ContainsRune(name, '\\') {
	148	+	return "", fmt.Errorf("invalid entry name %q", name)
	149	+	}
	150	+	if strings.HasPrefix(name, "/") {
	151	+	return "", fmt.Errorf("entry %q has an absolute path", name)
	152	+	}
	153	+	var parts []string
	154	+	for _, p := range strings.Split(name, "/") {
	155	+	switch p {
	156	+	case "", ".":
	157	+	case "..":
	158	+	return "", fmt.Errorf("entry %q has a .. component", name)
	159	+	default:
	160	+	parts = append(parts, p)
	161	+	}
	162	+	}
	163	+	if stripTop {
	164	+	if len(parts) <= 1 {
	165	+	return "", nil
	166	+	}
	167	+	parts = parts[1:]
	168	+	}
	169	+	return strings.Join(parts, "/"), nil
	170	+	}
	171	+
	172	+	// checkLinkTarget accepts a relative target whose ".." components all come
	173	+	// first and that stays inside the tree. name is the cleaned path of the link.
	174	+	// The lexical rule is exact because no entry is ever unpacked below a symbolic
	175	+	// link.
	176	+	func checkLinkTarget(name, target string) error {
	177	+	if target == "" \|\| strings.ContainsRune(target, 0) \|\| strings.HasPrefix(target, "/") {
	178	+	return fmt.Errorf("symbolic link %q has the target %q: absolute or empty", name, target)
	179	+	}
	180	+	// depth is how many directories lie between the tree's top and the link.
	181	+	depth := 0
	182	+	if dir := path.Dir(name); dir != "." {
	183	+	depth = strings.Count(dir, "/") + 1
	184	+	}
	185	+	up, seenName := 0, false
	186	+	for _, p := range strings.Split(target, "/") {
	187	+	switch p {
	188	+	case "", ".":
	189	+	case "..":
	190	+	if seenName {
	191	+	return fmt.Errorf("symbolic link %q: target %q climbs after a name", name, target)
	192	+	}
	193	+	up++
	194	+	default:
	195	+	seenName = true
	196	+	}
	197	+	}
	198	+	if up > depth {
	199	+	return fmt.Errorf("symbolic link %q: target %q leaves the archive", name, target)
	200	+	}
	201	+	return nil
	202	+	}
	203	+
	204	+	// under reports whether name, or one of its parents, is an unpacked symbolic link.
	205	+	func under(links map[string]bool, name string) bool {
	206	+	for p := name; p != "." && p != "/" && p != ""; p = path.Dir(p) {
	207	+	if links[p] {
	208	+	return true
	209	+	}
	210	+	}
	211	+	return false
	212	+	}
	213	+
	214	+	// openDirs gives every directory the mode 0755, whatever the umask of the
	215	+	// runner was when it created them: the sandboxes read the tree as another user.
	216	+	func openDirs(dest string) error {
	217	+	return filepath.WalkDir(dest, func(p string, d fs.DirEntry, err error) error {
	218	+	if err != nil \|\| !d.IsDir() {
	219	+	return err
	220	+	}
	221	+	return os.Chmod(p, 0o755)
	222	+	})
	223	+	}

A internal/externals/manager.go +546 -0

@@ -0,0 +1,546 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	// Package externals provides the tools that jobs expect to find outside the
	4	+	// image, today the Node.js builds of actions/runner (node20, node24 and the
	5	+	// alpine variants for job containers), so that a runner upgrade or a new Node.js
	6	+	// version never needs an image rebuild.
	7	+	//
	8	+	// The host downloads the pinned builds (table.go) into
	9	+	// <state dir>/externals/<name>-<version>-<arch>/ at start, verifying the
	10	+	// SHA-256 of each archive before it is unpacked, and the sandbox drivers attach
	11	+	// them read-only at /home/runner/externals/<name> (GitHub's layout:
	12	+	// externals/node24/bin/node). Only the pinned versions are kept.
	13	+	//
	14	+	// A tool that cannot be provided (offline, checksum mismatch, no build for this
	15	+	// architecture, checksum not recorded) never stops the runner: the sandboxes get
	16	+	// a directory with the file UNAVAILABLE and a bin/node stub that print why, and
	17	+	// a job that needs the tool fails with that message. Failed downloads are retried in the
	18	+	// background.
	19	+	package externals
	20	+
	21	+	import (
	22	+	"context"
	23	+	"crypto/sha256"
	24	+	"encoding/hex"
	25	+	"errors"
	26	+	"fmt"
	27	+	"io"
	28	+	"net/http"
	29	+	"net/url"
	30	+	"os"
	31	+	"path/filepath"
	32	+	"strings"
	33	+	"sync"
	34	+	"time"
	35	+
	36	+	"github.com/bnema/zerowrap"
	37	+
	38	+	"bnema.dev/ermite-runner/internal/sandbox"
	39	+	)
	40	+
	41	+	// maxDownload bounds one archive (the largest build is about 60 MiB).
	42	+	const maxDownload = 512 << 20
	43	+
	44	+	// Manager downloads, verifies and keeps the externals of one architecture.
	45	+	type Manager struct {
	46	+	// Dir is <state dir>/externals.
	47	+	Dir string
	48	+	// Arch is GitHub's architecture name of the host (x64, arm64).
	49	+	Arch string
	50	+	// Table is the set of known artifacts (Pinned in production).
	51	+	Table []Artifact
	52	+	// Client downloads the archives; nil means a client with a 15 minute limit
	53	+	// that follows redirects to https locations only.
	54	+	Client *http.Client
	55	+	// RetryEvery is the interval of the background retries of the downloads that
	56	+	// failed for a reason that may pass (default 5 minutes).
	57	+	RetryEvery time.Duration
	58	+
	59	+	mu sync.Mutex
	60	+	state map[string]entry // by artifact name
	61	+	syncing bool
	62	+
	63	+	// stubMu serialises the creation of the directories of unavailable tools;
	64	+	// stubs remembers the ones built (or checked) by this process.
	65	+	stubMu sync.Mutex
	66	+	stubs map[string]bool
	67	+	}
	68	+
	69	+	type entry struct {
	70	+	ext sandbox.External
	71	+	retry bool // a failure worth trying again (network), not a verdict
	72	+	}
	73	+
	74	+	// New returns a Manager for the host architecture; ok is false when the runner
	75	+	// has no builds for it.
	76	+	func New(stateDir string) (m *Manager, ok bool) {
	77	+	arch := HostArch()
	78	+	return &Manager{Dir: filepath.Join(stateDir, "externals"), Arch: arch, Table: Pinned}, arch != ""
	79	+	}
	80	+
	81	+	// Start runs Sync in the background and waits for it at most wait, so that a
	82	+	// runner with a working network normally takes its first job with everything in
	83	+	// place, and one without never waits long. When the wait ends the download goes
	84	+	// on; jobs that start meanwhile see the tools that are not done yet as
	85	+	// unavailable ("still being downloaded"). Downloads that failed for a reason
	86	+	// that may pass (network, HTTP status) are retried every RetryEvery, jobs or
	87	+	// not, until ctx ends.
	88	+	func (m *Manager) Start(ctx context.Context, wait time.Duration) {
	89	+	log := zerowrap.FromCtx(ctx)
	90	+	done := make(chan struct{})
	91	+	go func() {
	92	+	defer close(done)
	93	+	if err := m.Sync(ctx); err != nil && ctx.Err() == nil {
	94	+	log.Warn().Err(err).Msg("some external tools are not available: jobs that need them fail with the reason; the download is retried")
	95	+	}
	96	+	}()
	97	+	go m.retryLoop(ctx)
	98	+	select {
	99	+	case <-done:
	100	+	case <-time.After(wait):
	101	+	log.Warn().Dur("waited", wait).Msg("external tools are still downloading: the runner takes jobs, those that need them fail until they are ready")
	102	+	case <-ctx.Done():
	103	+	}
	104	+	}
	105	+
	106	+	// retryLoop calls Sync every RetryEvery while a tool is missing for a reason
	107	+	// that may pass.
	108	+	func (m *Manager) retryLoop(ctx context.Context) {
	109	+	every := m.RetryEvery
	110	+	if every == 0 {
	111	+	every = 5 * time.Minute
	112	+	}
	113	+	t := time.NewTicker(every)
	114	+	defer t.Stop()
	115	+	for {
	116	+	select {
	117	+	case <-ctx.Done():
	118	+	return
	119	+	case <-t.C:
	120	+	}
	121	+	if m.wantsRetry() {
	122	+	_ = m.Sync(ctx)
	123	+	}
	124	+	}
	125	+	}
	126	+
	127	+	func (m *Manager) wantsRetry() bool {
	128	+	m.mu.Lock()
	129	+	defer m.mu.Unlock()
	130	+	for _, e := range m.state {
	131	+	if e.retry {
	132	+	return true
	133	+	}
	134	+	}
	135	+	return false
	136	+	}
	137	+
	138	+	// Sync makes the externals match the table: it downloads what is missing,
	139	+	// removes everything else from Dir, and records the outcome per tool as it goes.
	140	+	// It returns an error describing the artifacts that could not be provided; the
	141	+	// runner goes on with them marked unavailable.
	142	+	func (m *Manager) Sync(ctx context.Context) error {
	143	+	m.mu.Lock()
	144	+	if m.syncing {
	145	+	m.mu.Unlock()
	146	+	return nil
	147	+	}
	148	+	m.syncing = true
	149	+	if m.state == nil {
	150	+	m.state = map[string]entry{}
	151	+	}
	152	+	m.mu.Unlock()
	153	+	defer func() { m.mu.Lock(); m.syncing = false; m.mu.Unlock() }()
	154	+
	155	+	log := zerowrap.FromCtx(ctx)
	156	+	var errs []error
	157	+	keep := map[string]bool{}
	158	+	if err := os.MkdirAll(m.Dir, 0o755); err != nil {
	159	+	return err
	160	+	}
	161	+	for _, name := range m.names() {
	162	+	a, ok := m.artifact(name)
	163	+	var e entry
	164	+	switch {
	165	+	case !ok:
	166	+	e = m.unavailable(name, fmt.Sprintf("there is no %s build for linux-%s", name, m.Arch), false)
	167	+	default:
	168	+	var err error
	169	+	if e, err = m.ensure(ctx, a); err != nil {
	170	+	errs = append(errs, fmt.Errorf("%s %s: %w", a.Name, a.Version, err))
	171	+	log.Warn().Err(err).Str("external", a.Name).Str("version", a.Version).Msg("external tool not available")
	172	+	} else {
	173	+	log.Info().Str("external", a.Name).Str("version", a.Version).Str("arch", a.Arch).Msg("external tool ready")
	174	+	}
	175	+	}
	176	+	m.mu.Lock()
	177	+	m.state[name] = e
	178	+	m.mu.Unlock()
	179	+	keep[filepath.Base(e.ext.Dir)] = true
	180	+	if e.ext.Unavailable {
	181	+	keep[unavailableDir] = true
	182	+	}
	183	+	}
	184	+	m.prune(log, keep)
	185	+	return errors.Join(errs...)
	186	+	}
	187	+
	188	+	// Externals implements sandbox.ExternalsSource. A tool that has no outcome yet
	189	+	// is reported as being downloaded. It has no side effect on the downloads (the
	190	+	// retries run on a timer, see Start); the directories of unavailable tools are
	191	+	// built at most once and only touched here.
	192	+	func (m *Manager) Externals() []sandbox.External {
	193	+	m.mu.Lock()
	194	+	names := m.names()
	195	+	got := make([]entry, len(names))
	196	+	have := make([]bool, len(names))
	197	+	for i, n := range names {
	198	+	got[i], have[i] = m.state[n]
	199	+	}
	200	+	m.mu.Unlock()
	201	+
	202	+	// Outside the lock: the first call for a reason writes files.
	203	+	out := make([]sandbox.External, len(names))
	204	+	for i, n := range names {
	205	+	switch {
	206	+	case !have[i]:
	207	+	got[i] = m.unavailable(n, n+" is still being downloaded on this runner: run the job again in a few minutes", false)
	208	+	case got[i].ext.Unavailable:
	209	+	touch(got[i].ext.Dir) // handed out again: not stale
	210	+	}
	211	+	out[i] = got[i].ext
	212	+	}
	213	+	return out
	214	+	}
	215	+
	216	+	// names is every artifact name in the table, in table order, once.
	217	+	func (m *Manager) names() []string {
	218	+	seen := map[string]bool{}
	219	+	var out []string
	220	+	for _, a := range m.Table {
	221	+	if !seen[a.Name] {
	222	+	seen[a.Name] = true
	223	+	out = append(out, a.Name)
	224	+	}
	225	+	}
	226	+	return out
	227	+	}
	228	+
	229	+	func (m *Manager) artifact(name string) (Artifact, bool) {
	230	+	for _, a := range m.Table {
	231	+	if a.Name == name && a.Arch == m.Arch {
	232	+	return a, true
	233	+	}
	234	+	}
	235	+	return Artifact{}, false
	236	+	}
	237	+
	238	+	const unavailableDir = ".unavailable"
	239	+
	240	+	// unavailableStub is the bin/node of a tool that is missing: it prints the
	241	+	// reason (the file UNAVAILABLE next to bin/) on stderr and fails, so that a job
	242	+	// which runs the tool anyway (a node action, a run step) gets the reason rather
	243	+	// than "No such file or directory". It does not contain the reason.
	244	+	const unavailableStub = "#!/bin/sh\ncat \"$(dirname \"$0\")/../" + sandbox.ExternalUnavailableFile + "\" >&2\nexit 1\n"
	245	+
	246	+	// unavailable returns the entry of a tool that is missing: a directory with the
	247	+	// reason in ExternalUnavailableFile and the bin/node stub. Its name carries a
	248	+	// hash of the reason, so that the ID (and so a VM's disk) changes only when the
	249	+	// reason does. The directory is built once per process, complete, under a
	250	+	// temporary name and renamed into place; files that exist with the right
	251	+	// content are never rewritten, and a missing or different one is replaced
	252	+	// through a rename, so a reader (the build of a VM's externals disk, a job that
	253	+	// runs the stub) never sees a file half written. Each call touches the
	254	+	// directory: the pruning keeps what was handed out in the last hour.
	255	+	func (m *Manager) unavailable(name, reason string, retry bool) entry {
	256	+	sum := sha256.Sum256([]byte(name + "\x00" + reason))
	257	+	id := "unavailable-" + hex.EncodeToString(sum[:8])
	258	+	dir := filepath.Join(m.Dir, unavailableDir, name+"-"+id)
	259	+	e := entry{retry: retry, ext: sandbox.External{Name: name, ID: id, Dir: dir, Unavailable: true}}
	260	+
	261	+	m.stubMu.Lock()
	262	+	defer m.stubMu.Unlock()
	263	+	if _, err := os.Lstat(filepath.Join(dir, "bin", "node")); err != nil \|\| !m.stubs[dir] {
	264	+	// Best effort: on failure the entry stays marked unavailable and the
	265	+	// sandbox sees whatever there is.
	266	+	if err := buildStub(dir, reason); err == nil {
	267	+	if m.stubs == nil {
	268	+	m.stubs = map[string]bool{}
	269	+	}
	270	+	m.stubs[dir] = true
	271	+	}
	272	+	}
	273	+	touch(dir)
	274	+	return e
	275	+	}
	276	+
	277	+	func touch(dir string) {
	278	+	now := time.Now()
	279	+	_ = os.Chtimes(dir, now, now)
	280	+	}
	281	+
	282	+	// buildStub makes dir hold the UNAVAILABLE file and the bin/node stub, with
	283	+	// explicit modes (the umask of the runner must not hide them from the user of
	284	+	// the sandbox).
	285	+	func buildStub(dir, reason string) error {
	286	+	parent := filepath.Dir(dir)
	287	+	if err := os.MkdirAll(parent, 0o755); err != nil {
	288	+	return err
	289	+	}
	290	+	if err := os.Chmod(parent, 0o755); err != nil {
	291	+	return err
	292	+	}
	293	+	files := []stubFile{
	294	+	{sandbox.ExternalUnavailableFile, reason + "\n", 0o644},
	295	+	{"bin/node", unavailableStub, 0o755},
	296	+	}
	297	+	if _, err := os.Lstat(dir); errors.Is(err, os.ErrNotExist) {
	298	+	tmp, err := os.MkdirTemp(parent, ".tmp-")
	299	+	if err != nil {
	300	+	return err
	301	+	}
	302	+	defer func() { _ = os.RemoveAll(tmp) }()
	303	+	if err := os.Mkdir(filepath.Join(tmp, "bin"), 0o755); err != nil {
	304	+	return err
	305	+	}
	306	+	for _, d := range []string{tmp, filepath.Join(tmp, "bin")} {
	307	+	if err := os.Chmod(d, 0o755); err != nil {
	308	+	return err
	309	+	}
	310	+	}
	311	+	for _, f := range files {
	312	+	if err := f.write(tmp); err != nil {
	313	+	return err
	314	+	}
	315	+	}
	316	+	if err := os.Rename(tmp, dir); err == nil {
	317	+	return nil
	318	+	}
	319	+	// Lost a race with another process, or the directory appeared: repair it.
	320	+	}
	321	+	if err := os.MkdirAll(filepath.Join(dir, "bin"), 0o755); err != nil {
	322	+	return err
	323	+	}
	324	+	for _, f := range files {
	325	+	if f.current(dir) {
	326	+	continue
	327	+	}
	328	+	if err := f.write(dir); err != nil {
	329	+	return err
	330	+	}
	331	+	}
	332	+	return nil
	333	+	}
	334	+
	335	+	type stubFile struct {
	336	+	name, content string
	337	+	mode os.FileMode
	338	+	}
	339	+
	340	+	// current reports whether the file exists with this content and mode.
	341	+	func (f stubFile) current(dir string) bool {
	342	+	p := filepath.Join(dir, f.name)
	343	+	st, err := os.Lstat(p)
	344	+	if err != nil \|\| !st.Mode().IsRegular() \|\| st.Mode().Perm() != f.mode {
	345	+	return false
	346	+	}
	347	+	b, err := os.ReadFile(p)
	348	+	return err == nil && string(b) == f.content
	349	+	}
	350	+
	351	+	// write creates the file through a temporary file in the same directory and a
	352	+	// rename, never in place.
	353	+	func (f stubFile) write(dir string) error {
	354	+	p := filepath.Join(dir, f.name)
	355	+	tmp, err := os.CreateTemp(filepath.Dir(p), ".tmp-")
	356	+	if err != nil {
	357	+	return err
	358	+	}
	359	+	ok := false
	360	+	defer func() {
	361	+	if !ok {
	362	+	_ = os.Remove(tmp.Name())
	363	+	}
	364	+	}()
	365	+	if _, err := tmp.WriteString(f.content); err != nil {
	366	+	_ = tmp.Close()
	367	+	return err
	368	+	}
	369	+	if err := tmp.Chmod(f.mode); err != nil {
	370	+	_ = tmp.Close()
	371	+	return err
	372	+	}
	373	+	if err := tmp.Close(); err != nil {
	374	+	return err
	375	+	}
	376	+	if err := os.Rename(tmp.Name(), p); err != nil {
	377	+	return err
	378	+	}
	379	+	ok = true
	380	+	return nil
	381	+	}
	382	+
	383	+	// ensure returns the ready entry of a, downloading it when needed. On failure it
	384	+	// returns the unavailable entry together with the error.
	385	+	func (m *Manager) ensure(ctx context.Context, a Artifact) (entry, error) {
	386	+	final := filepath.Join(m.Dir, a.dirName())
	387	+	if isReady(final) {
	388	+	return readyEntry(a, final), nil
	389	+	}
	390	+	err := m.fetch(ctx, a, final)
	391	+	if err != nil {
	392	+	reason := fmt.Sprintf("%s (Node.js %s) is not available on this runner: %v", a.Name, a.Version, err)
	393	+	var v *verdictError
	394	+	return m.unavailable(a.Name, reason, !errors.As(err, &v)), err
	395	+	}
	396	+	return readyEntry(a, final), nil
	397	+	}
	398	+
	399	+	func readyEntry(a Artifact, dir string) entry {
	400	+	return entry{ext: sandbox.External{Name: a.Name, ID: a.dirName() + "-" + a.SHA256[:16], Dir: dir}}
	401	+	}
	402	+
	403	+	// isReady reports whether dir holds a complete tool: a regular executable
	404	+	// bin/node. Directories only appear under their final name by atomic rename
	405	+	// after a verified extraction.
	406	+	func isReady(dir string) bool {
	407	+	st, err := os.Stat(filepath.Join(dir, "bin", "node"))
	408	+	return err == nil && st.Mode().IsRegular() && st.Mode().Perm()&0o111 != 0
	409	+	}
	410	+
	411	+	// verdictError is a failure that another attempt will not change (the checksum
	412	+	// differs, the archive is unsafe).
	413	+	type verdictError struct{ error }
	414	+
	415	+	func (e *verdictError) Unwrap() error { return e.error }
	416	+
	417	+	// fetch downloads a, verifies its checksum, unpacks it next to final and
	418	+	// renames it into place.
	419	+	func (m *Manager) fetch(ctx context.Context, a Artifact, final string) error {
	420	+	u, err := url.Parse(a.URL)
	421	+	if err != nil \|\| u.Scheme != "https" {
	422	+	return &verdictError{fmt.Errorf("%q is not an https URL", a.URL)}
	423	+	}
	424	+	tmp, err := os.MkdirTemp(m.Dir, ".tmp-")
	425	+	if err != nil {
	426	+	return err
	427	+	}
	428	+	defer func() { _ = os.RemoveAll(tmp) }()
	429	+
	430	+	archive := filepath.Join(tmp, "archive.tar.gz")
	431	+	if err := m.download(ctx, a, archive); err != nil {
	432	+	return err
	433	+	}
	434	+	tree := filepath.Join(tmp, "tree")
	435	+	if err := os.Mkdir(tree, 0o755); err != nil {
	436	+	return err
	437	+	}
	438	+	if err := extractTarGz(archive, tree, a.StripTop); err != nil {
	439	+	return &verdictError{fmt.Errorf("unpack: %w", err)}
	440	+	}
	441	+	if !isReady(tree) {
	442	+	return &verdictError{errors.New("the archive has no executable bin/node")}
	443	+	}
	444	+	if err := os.RemoveAll(final); err != nil { // an incomplete directory of an older run
	445	+	return err
	446	+	}
	447	+	return os.Rename(tree, final)
	448	+	}
	449	+
	450	+	// download fetches a.URL to dest and verifies its SHA-256 as it arrives.
	451	+	func (m *Manager) download(ctx context.Context, a Artifact, dest string) error {
	452	+	client := m.Client
	453	+	if client == nil {
	454	+	client = newClient()
	455	+	}
	456	+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, a.URL, nil)
	457	+	if err != nil {
	458	+	return err
	459	+	}
	460	+	resp, err := client.Do(req)
	461	+	if err != nil {
	462	+	return fmt.Errorf("download %s: %w", a.URL, err)
	463	+	}
	464	+	defer func() { _ = resp.Body.Close() }()
	465	+	if resp.StatusCode != http.StatusOK {
	466	+	return fmt.Errorf("download %s: %s", a.URL, resp.Status)
	467	+	}
	468	+	f, err := os.OpenFile(dest, os.O_WRONLY\|os.O_CREATE\|os.O_EXCL, 0o600)
	469	+	if err != nil {
	470	+	return err
	471	+	}
	472	+	h := sha256.New()
	473	+	n, err := io.Copy(io.MultiWriter(f, h), io.LimitReader(resp.Body, maxDownload+1))
	474	+	if cerr := f.Close(); err == nil {
	475	+	err = cerr
	476	+	}
	477	+	if err != nil {
	478	+	return fmt.Errorf("download %s: %w", a.URL, err)
	479	+	}
	480	+	if n > maxDownload {
	481	+	return &verdictError{fmt.Errorf("download %s: more than %d bytes", a.URL, int64(maxDownload))}
	482	+	}
	483	+	if err := VerifySum(hex.EncodeToString(h.Sum(nil)), a.SHA256); err != nil {
	484	+	return &verdictError{fmt.Errorf("download %s: %w", a.URL, err)}
	485	+	}
	486	+	return nil
	487	+	}
	488	+
	489	+	// newClient is the default downloader: a time limit, and redirects only to
	490	+	// https locations (nodejs.org does not redirect; GitHub releases do).
	491	+	func newClient() *http.Client {
	492	+	return &http.Client{
	493	+	Timeout: 15 * time.Minute,
	494	+	CheckRedirect: func(req http.Request, via []http.Request) error {
	495	+	if req.URL.Scheme != "https" {
	496	+	return fmt.Errorf("redirect to a non-https URL %s", req.URL.Redacted())
	497	+	}
	498	+	if len(via) >= 5 {
	499	+	return errors.New("too many redirects")
	500	+	}
	501	+	return nil
	502	+	},
	503	+	}
	504	+	}
	505	+
	506	+	// VerifySum compares two hex SHA-256 values.
	507	+	func VerifySum(got, want string) error {
	508	+	if !strings.EqualFold(got, want) {
	509	+	return fmt.Errorf("SHA-256 mismatch: got %s, want %s", got, want)
	510	+	}
	511	+	return nil
	512	+	}
	513	+
	514	+	// prune removes everything in Dir that is not pinned: other versions, other
	515	+	// architectures, leftovers of an interrupted download. The reasons of
	516	+	// unavailable tools (a few bytes each) may be bind-mounted into a running
	517	+	// container, so those are only removed once they are an hour old.
	518	+	func (m *Manager) prune(log zerowrap.Logger, keep map[string]bool) {
	519	+	ents, err := os.ReadDir(m.Dir)
	520	+	if err != nil {
	521	+	return
	522	+	}
	523	+	for _, e := range ents {
	524	+	switch name := e.Name(); {
	525	+	case keep[name]:
	526	+	case name == unavailableDir:
	527	+	m.pruneUnavailable(keep)
	528	+	default:
	529	+	if err := os.RemoveAll(filepath.Join(m.Dir, name)); err != nil {
	530	+	log.Warn().Err(err).Str("path", name).Msg("removing an unpinned external tool")
	531	+	}
	532	+	}
	533	+	}
	534	+	}
	535	+
	536	+	func (m *Manager) pruneUnavailable(keep map[string]bool) {
	537	+	dir := filepath.Join(m.Dir, unavailableDir)
	538	+	ents, _ := os.ReadDir(dir)
	539	+	for _, e := range ents {
	540	+	fi, err := e.Info()
	541	+	if keep[e.Name()] \|\| err != nil \|\| time.Since(fi.ModTime()) < time.Hour {
	542	+	continue
	543	+	}
	544	+	_ = os.RemoveAll(filepath.Join(dir, e.Name()))
	545	+	}
	546	+	}

@@ -0,0 +1,546 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	// Package externals provides the tools that jobs expect to find outside the
	4	+	// image, today the Node.js builds of actions/runner (node20, node24 and the
	5	+	// alpine variants for job containers), so that a runner upgrade or a new Node.js
	6	+	// version never needs an image rebuild.
	7	+	//
	8	+	// The host downloads the pinned builds (table.go) into
	9	+	// <state dir>/externals/<name>-<version>-<arch>/ at start, verifying the
	10	+	// SHA-256 of each archive before it is unpacked, and the sandbox drivers attach
	11	+	// them read-only at /home/runner/externals/<name> (GitHub's layout:
	12	+	// externals/node24/bin/node). Only the pinned versions are kept.
	13	+	//
	14	+	// A tool that cannot be provided (offline, checksum mismatch, no build for this
	15	+	// architecture, checksum not recorded) never stops the runner: the sandboxes get
	16	+	// a directory with the file UNAVAILABLE and a bin/node stub that print why, and
	17	+	// a job that needs the tool fails with that message. Failed downloads are retried in the
	18	+	// background.
	19	+	package externals
	20	+
	21	+	import (
	22	+	"context"
	23	+	"crypto/sha256"
	24	+	"encoding/hex"
	25	+	"errors"
	26	+	"fmt"
	27	+	"io"
	28	+	"net/http"
	29	+	"net/url"
	30	+	"os"
	31	+	"path/filepath"
	32	+	"strings"
	33	+	"sync"
	34	+	"time"
	35	+
	36	+	"github.com/bnema/zerowrap"
	37	+
	38	+	"bnema.dev/ermite-runner/internal/sandbox"
	39	+	)
	40	+
	41	+	// maxDownload bounds one archive (the largest build is about 60 MiB).
	42	+	const maxDownload = 512 << 20
	43	+
	44	+	// Manager downloads, verifies and keeps the externals of one architecture.
	45	+	type Manager struct {
	46	+	// Dir is <state dir>/externals.
	47	+	Dir string
	48	+	// Arch is GitHub's architecture name of the host (x64, arm64).
	49	+	Arch string
	50	+	// Table is the set of known artifacts (Pinned in production).
	51	+	Table []Artifact
	52	+	// Client downloads the archives; nil means a client with a 15 minute limit
	53	+	// that follows redirects to https locations only.
	54	+	Client *http.Client
	55	+	// RetryEvery is the interval of the background retries of the downloads that
	56	+	// failed for a reason that may pass (default 5 minutes).
	57	+	RetryEvery time.Duration
	58	+
	59	+	mu sync.Mutex
	60	+	state map[string]entry // by artifact name
	61	+	syncing bool
	62	+
	63	+	// stubMu serialises the creation of the directories of unavailable tools;
	64	+	// stubs remembers the ones built (or checked) by this process.
	65	+	stubMu sync.Mutex
	66	+	stubs map[string]bool
	67	+	}
	68	+
	69	+	type entry struct {
	70	+	ext sandbox.External
	71	+	retry bool // a failure worth trying again (network), not a verdict
	72	+	}
	73	+
	74	+	// New returns a Manager for the host architecture; ok is false when the runner
	75	+	// has no builds for it.
	76	+	func New(stateDir string) (m *Manager, ok bool) {
	77	+	arch := HostArch()
	78	+	return &Manager{Dir: filepath.Join(stateDir, "externals"), Arch: arch, Table: Pinned}, arch != ""
	79	+	}
	80	+
	81	+	// Start runs Sync in the background and waits for it at most wait, so that a
	82	+	// runner with a working network normally takes its first job with everything in
	83	+	// place, and one without never waits long. When the wait ends the download goes
	84	+	// on; jobs that start meanwhile see the tools that are not done yet as
	85	+	// unavailable ("still being downloaded"). Downloads that failed for a reason
	86	+	// that may pass (network, HTTP status) are retried every RetryEvery, jobs or
	87	+	// not, until ctx ends.
	88	+	func (m *Manager) Start(ctx context.Context, wait time.Duration) {
	89	+	log := zerowrap.FromCtx(ctx)
	90	+	done := make(chan struct{})
	91	+	go func() {
	92	+	defer close(done)
	93	+	if err := m.Sync(ctx); err != nil && ctx.Err() == nil {
	94	+	log.Warn().Err(err).Msg("some external tools are not available: jobs that need them fail with the reason; the download is retried")
	95	+	}
	96	+	}()
	97	+	go m.retryLoop(ctx)
	98	+	select {
	99	+	case <-done:
	100	+	case <-time.After(wait):
	101	+	log.Warn().Dur("waited", wait).Msg("external tools are still downloading: the runner takes jobs, those that need them fail until they are ready")
	102	+	case <-ctx.Done():
	103	+	}
	104	+	}
	105	+
	106	+	// retryLoop calls Sync every RetryEvery while a tool is missing for a reason
	107	+	// that may pass.
	108	+	func (m *Manager) retryLoop(ctx context.Context) {
	109	+	every := m.RetryEvery
	110	+	if every == 0 {
	111	+	every = 5 * time.Minute
	112	+	}
	113	+	t := time.NewTicker(every)
	114	+	defer t.Stop()
	115	+	for {
	116	+	select {
	117	+	case <-ctx.Done():
	118	+	return
	119	+	case <-t.C:
	120	+	}
	121	+	if m.wantsRetry() {
	122	+	_ = m.Sync(ctx)
	123	+	}
	124	+	}
	125	+	}
	126	+
	127	+	func (m *Manager) wantsRetry() bool {
	128	+	m.mu.Lock()
	129	+	defer m.mu.Unlock()
	130	+	for _, e := range m.state {
	131	+	if e.retry {
	132	+	return true
	133	+	}
	134	+	}
	135	+	return false
	136	+	}
	137	+
	138	+	// Sync makes the externals match the table: it downloads what is missing,
	139	+	// removes everything else from Dir, and records the outcome per tool as it goes.
	140	+	// It returns an error describing the artifacts that could not be provided; the
	141	+	// runner goes on with them marked unavailable.
	142	+	func (m *Manager) Sync(ctx context.Context) error {
	143	+	m.mu.Lock()
	144	+	if m.syncing {
	145	+	m.mu.Unlock()
	146	+	return nil
	147	+	}
	148	+	m.syncing = true
	149	+	if m.state == nil {
	150	+	m.state = map[string]entry{}
	151	+	}
	152	+	m.mu.Unlock()
	153	+	defer func() { m.mu.Lock(); m.syncing = false; m.mu.Unlock() }()
	154	+
	155	+	log := zerowrap.FromCtx(ctx)
	156	+	var errs []error
	157	+	keep := map[string]bool{}
	158	+	if err := os.MkdirAll(m.Dir, 0o755); err != nil {
	159	+	return err
	160	+	}
	161	+	for _, name := range m.names() {
	162	+	a, ok := m.artifact(name)
	163	+	var e entry
	164	+	switch {
	165	+	case !ok:
	166	+	e = m.unavailable(name, fmt.Sprintf("there is no %s build for linux-%s", name, m.Arch), false)
	167	+	default:
	168	+	var err error
	169	+	if e, err = m.ensure(ctx, a); err != nil {
	170	+	errs = append(errs, fmt.Errorf("%s %s: %w", a.Name, a.Version, err))
	171	+	log.Warn().Err(err).Str("external", a.Name).Str("version", a.Version).Msg("external tool not available")
	172	+	} else {
	173	+	log.Info().Str("external", a.Name).Str("version", a.Version).Str("arch", a.Arch).Msg("external tool ready")
	174	+	}
	175	+	}
	176	+	m.mu.Lock()
	177	+	m.state[name] = e
	178	+	m.mu.Unlock()
	179	+	keep[filepath.Base(e.ext.Dir)] = true
	180	+	if e.ext.Unavailable {
	181	+	keep[unavailableDir] = true
	182	+	}
	183	+	}
	184	+	m.prune(log, keep)
	185	+	return errors.Join(errs...)
	186	+	}
	187	+
	188	+	// Externals implements sandbox.ExternalsSource. A tool that has no outcome yet
	189	+	// is reported as being downloaded. It has no side effect on the downloads (the
	190	+	// retries run on a timer, see Start); the directories of unavailable tools are
	191	+	// built at most once and only touched here.
	192	+	func (m *Manager) Externals() []sandbox.External {
	193	+	m.mu.Lock()
	194	+	names := m.names()
	195	+	got := make([]entry, len(names))
	196	+	have := make([]bool, len(names))
	197	+	for i, n := range names {
	198	+	got[i], have[i] = m.state[n]
	199	+	}
	200	+	m.mu.Unlock()
	201	+
	202	+	// Outside the lock: the first call for a reason writes files.
	203	+	out := make([]sandbox.External, len(names))
	204	+	for i, n := range names {
	205	+	switch {
	206	+	case !have[i]:
	207	+	got[i] = m.unavailable(n, n+" is still being downloaded on this runner: run the job again in a few minutes", false)
	208	+	case got[i].ext.Unavailable:
	209	+	touch(got[i].ext.Dir) // handed out again: not stale
	210	+	}
	211	+	out[i] = got[i].ext
	212	+	}
	213	+	return out
	214	+	}
	215	+
	216	+	// names is every artifact name in the table, in table order, once.
	217	+	func (m *Manager) names() []string {
	218	+	seen := map[string]bool{}
	219	+	var out []string
	220	+	for _, a := range m.Table {
	221	+	if !seen[a.Name] {
	222	+	seen[a.Name] = true
	223	+	out = append(out, a.Name)
	224	+	}
	225	+	}
	226	+	return out
	227	+	}
	228	+
	229	+	func (m *Manager) artifact(name string) (Artifact, bool) {
	230	+	for _, a := range m.Table {
	231	+	if a.Name == name && a.Arch == m.Arch {
	232	+	return a, true
	233	+	}
	234	+	}
	235	+	return Artifact{}, false
	236	+	}
	237	+
	238	+	const unavailableDir = ".unavailable"
	239	+
	240	+	// unavailableStub is the bin/node of a tool that is missing: it prints the
	241	+	// reason (the file UNAVAILABLE next to bin/) on stderr and fails, so that a job
	242	+	// which runs the tool anyway (a node action, a run step) gets the reason rather
	243	+	// than "No such file or directory". It does not contain the reason.
	244	+	const unavailableStub = "#!/bin/sh\ncat \"$(dirname \"$0\")/../" + sandbox.ExternalUnavailableFile + "\" >&2\nexit 1\n"
	245	+
	246	+	// unavailable returns the entry of a tool that is missing: a directory with the
	247	+	// reason in ExternalUnavailableFile and the bin/node stub. Its name carries a
	248	+	// hash of the reason, so that the ID (and so a VM's disk) changes only when the
	249	+	// reason does. The directory is built once per process, complete, under a
	250	+	// temporary name and renamed into place; files that exist with the right
	251	+	// content are never rewritten, and a missing or different one is replaced
	252	+	// through a rename, so a reader (the build of a VM's externals disk, a job that
	253	+	// runs the stub) never sees a file half written. Each call touches the
	254	+	// directory: the pruning keeps what was handed out in the last hour.
	255	+	func (m *Manager) unavailable(name, reason string, retry bool) entry {
	256	+	sum := sha256.Sum256([]byte(name + "\x00" + reason))
	257	+	id := "unavailable-" + hex.EncodeToString(sum[:8])
	258	+	dir := filepath.Join(m.Dir, unavailableDir, name+"-"+id)
	259	+	e := entry{retry: retry, ext: sandbox.External{Name: name, ID: id, Dir: dir, Unavailable: true}}
	260	+
	261	+	m.stubMu.Lock()
	262	+	defer m.stubMu.Unlock()
	263	+	if _, err := os.Lstat(filepath.Join(dir, "bin", "node")); err != nil \|\| !m.stubs[dir] {
	264	+	// Best effort: on failure the entry stays marked unavailable and the
	265	+	// sandbox sees whatever there is.
	266	+	if err := buildStub(dir, reason); err == nil {
	267	+	if m.stubs == nil {
	268	+	m.stubs = map[string]bool{}
	269	+	}
	270	+	m.stubs[dir] = true
	271	+	}
	272	+	}
	273	+	touch(dir)
	274	+	return e
	275	+	}
	276	+
	277	+	func touch(dir string) {
	278	+	now := time.Now()
	279	+	_ = os.Chtimes(dir, now, now)
	280	+	}
	281	+
	282	+	// buildStub makes dir hold the UNAVAILABLE file and the bin/node stub, with
	283	+	// explicit modes (the umask of the runner must not hide them from the user of
	284	+	// the sandbox).
	285	+	func buildStub(dir, reason string) error {
	286	+	parent := filepath.Dir(dir)
	287	+	if err := os.MkdirAll(parent, 0o755); err != nil {
	288	+	return err
	289	+	}
	290	+	if err := os.Chmod(parent, 0o755); err != nil {
	291	+	return err
	292	+	}
	293	+	files := []stubFile{
	294	+	{sandbox.ExternalUnavailableFile, reason + "\n", 0o644},
	295	+	{"bin/node", unavailableStub, 0o755},
	296	+	}
	297	+	if _, err := os.Lstat(dir); errors.Is(err, os.ErrNotExist) {
	298	+	tmp, err := os.MkdirTemp(parent, ".tmp-")
	299	+	if err != nil {
	300	+	return err
	301	+	}
	302	+	defer func() { _ = os.RemoveAll(tmp) }()
	303	+	if err := os.Mkdir(filepath.Join(tmp, "bin"), 0o755); err != nil {
	304	+	return err
	305	+	}
	306	+	for _, d := range []string{tmp, filepath.Join(tmp, "bin")} {
	307	+	if err := os.Chmod(d, 0o755); err != nil {
	308	+	return err
	309	+	}
	310	+	}
	311	+	for _, f := range files {
	312	+	if err := f.write(tmp); err != nil {
	313	+	return err
	314	+	}
	315	+	}
	316	+	if err := os.Rename(tmp, dir); err == nil {
	317	+	return nil
	318	+	}
	319	+	// Lost a race with another process, or the directory appeared: repair it.
	320	+	}
	321	+	if err := os.MkdirAll(filepath.Join(dir, "bin"), 0o755); err != nil {
	322	+	return err
	323	+	}
	324	+	for _, f := range files {
	325	+	if f.current(dir) {
	326	+	continue
	327	+	}
	328	+	if err := f.write(dir); err != nil {
	329	+	return err
	330	+	}
	331	+	}
	332	+	return nil
	333	+	}
	334	+
	335	+	type stubFile struct {
	336	+	name, content string
	337	+	mode os.FileMode
	338	+	}
	339	+
	340	+	// current reports whether the file exists with this content and mode.
	341	+	func (f stubFile) current(dir string) bool {
	342	+	p := filepath.Join(dir, f.name)
	343	+	st, err := os.Lstat(p)
	344	+	if err != nil \|\| !st.Mode().IsRegular() \|\| st.Mode().Perm() != f.mode {
	345	+	return false
	346	+	}
	347	+	b, err := os.ReadFile(p)
	348	+	return err == nil && string(b) == f.content
	349	+	}
	350	+
	351	+	// write creates the file through a temporary file in the same directory and a
	352	+	// rename, never in place.
	353	+	func (f stubFile) write(dir string) error {
	354	+	p := filepath.Join(dir, f.name)
	355	+	tmp, err := os.CreateTemp(filepath.Dir(p), ".tmp-")
	356	+	if err != nil {
	357	+	return err
	358	+	}
	359	+	ok := false
	360	+	defer func() {
	361	+	if !ok {
	362	+	_ = os.Remove(tmp.Name())
	363	+	}
	364	+	}()
	365	+	if _, err := tmp.WriteString(f.content); err != nil {
	366	+	_ = tmp.Close()
	367	+	return err
	368	+	}
	369	+	if err := tmp.Chmod(f.mode); err != nil {
	370	+	_ = tmp.Close()
	371	+	return err
	372	+	}
	373	+	if err := tmp.Close(); err != nil {
	374	+	return err
	375	+	}
	376	+	if err := os.Rename(tmp.Name(), p); err != nil {
	377	+	return err
	378	+	}
	379	+	ok = true
	380	+	return nil
	381	+	}
	382	+
	383	+	// ensure returns the ready entry of a, downloading it when needed. On failure it
	384	+	// returns the unavailable entry together with the error.
	385	+	func (m *Manager) ensure(ctx context.Context, a Artifact) (entry, error) {
	386	+	final := filepath.Join(m.Dir, a.dirName())
	387	+	if isReady(final) {
	388	+	return readyEntry(a, final), nil
	389	+	}
	390	+	err := m.fetch(ctx, a, final)
	391	+	if err != nil {
	392	+	reason := fmt.Sprintf("%s (Node.js %s) is not available on this runner: %v", a.Name, a.Version, err)
	393	+	var v *verdictError
	394	+	return m.unavailable(a.Name, reason, !errors.As(err, &v)), err
	395	+	}
	396	+	return readyEntry(a, final), nil
	397	+	}
	398	+
	399	+	func readyEntry(a Artifact, dir string) entry {
	400	+	return entry{ext: sandbox.External{Name: a.Name, ID: a.dirName() + "-" + a.SHA256[:16], Dir: dir}}
	401	+	}
	402	+
	403	+	// isReady reports whether dir holds a complete tool: a regular executable
	404	+	// bin/node. Directories only appear under their final name by atomic rename
	405	+	// after a verified extraction.
	406	+	func isReady(dir string) bool {
	407	+	st, err := os.Stat(filepath.Join(dir, "bin", "node"))
	408	+	return err == nil && st.Mode().IsRegular() && st.Mode().Perm()&0o111 != 0
	409	+	}
	410	+
	411	+	// verdictError is a failure that another attempt will not change (the checksum
	412	+	// differs, the archive is unsafe).
	413	+	type verdictError struct{ error }
	414	+
	415	+	func (e *verdictError) Unwrap() error { return e.error }
	416	+
	417	+	// fetch downloads a, verifies its checksum, unpacks it next to final and
	418	+	// renames it into place.
	419	+	func (m *Manager) fetch(ctx context.Context, a Artifact, final string) error {
	420	+	u, err := url.Parse(a.URL)
	421	+	if err != nil \|\| u.Scheme != "https" {
	422	+	return &verdictError{fmt.Errorf("%q is not an https URL", a.URL)}
	423	+	}
	424	+	tmp, err := os.MkdirTemp(m.Dir, ".tmp-")
	425	+	if err != nil {
	426	+	return err
	427	+	}
	428	+	defer func() { _ = os.RemoveAll(tmp) }()
	429	+
	430	+	archive := filepath.Join(tmp, "archive.tar.gz")
	431	+	if err := m.download(ctx, a, archive); err != nil {
	432	+	return err
	433	+	}
	434	+	tree := filepath.Join(tmp, "tree")
	435	+	if err := os.Mkdir(tree, 0o755); err != nil {
	436	+	return err
	437	+	}
	438	+	if err := extractTarGz(archive, tree, a.StripTop); err != nil {
	439	+	return &verdictError{fmt.Errorf("unpack: %w", err)}
	440	+	}
	441	+	if !isReady(tree) {
	442	+	return &verdictError{errors.New("the archive has no executable bin/node")}
	443	+	}
	444	+	if err := os.RemoveAll(final); err != nil { // an incomplete directory of an older run
	445	+	return err
	446	+	}
	447	+	return os.Rename(tree, final)
	448	+	}
	449	+
	450	+	// download fetches a.URL to dest and verifies its SHA-256 as it arrives.
	451	+	func (m *Manager) download(ctx context.Context, a Artifact, dest string) error {
	452	+	client := m.Client
	453	+	if client == nil {
	454	+	client = newClient()
	455	+	}
	456	+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, a.URL, nil)
	457	+	if err != nil {
	458	+	return err
	459	+	}
	460	+	resp, err := client.Do(req)
	461	+	if err != nil {
	462	+	return fmt.Errorf("download %s: %w", a.URL, err)
	463	+	}
	464	+	defer func() { _ = resp.Body.Close() }()
	465	+	if resp.StatusCode != http.StatusOK {
	466	+	return fmt.Errorf("download %s: %s", a.URL, resp.Status)
	467	+	}
	468	+	f, err := os.OpenFile(dest, os.O_WRONLY\|os.O_CREATE\|os.O_EXCL, 0o600)
	469	+	if err != nil {
	470	+	return err
	471	+	}
	472	+	h := sha256.New()
	473	+	n, err := io.Copy(io.MultiWriter(f, h), io.LimitReader(resp.Body, maxDownload+1))
	474	+	if cerr := f.Close(); err == nil {
	475	+	err = cerr
	476	+	}
	477	+	if err != nil {
	478	+	return fmt.Errorf("download %s: %w", a.URL, err)
	479	+	}
	480	+	if n > maxDownload {
	481	+	return &verdictError{fmt.Errorf("download %s: more than %d bytes", a.URL, int64(maxDownload))}
	482	+	}
	483	+	if err := VerifySum(hex.EncodeToString(h.Sum(nil)), a.SHA256); err != nil {
	484	+	return &verdictError{fmt.Errorf("download %s: %w", a.URL, err)}
	485	+	}
	486	+	return nil
	487	+	}
	488	+
	489	+	// newClient is the default downloader: a time limit, and redirects only to
	490	+	// https locations (nodejs.org does not redirect; GitHub releases do).
	491	+	func newClient() *http.Client {
	492	+	return &http.Client{
	493	+	Timeout: 15 * time.Minute,
	494	+	CheckRedirect: func(req http.Request, via []http.Request) error {
	495	+	if req.URL.Scheme != "https" {
	496	+	return fmt.Errorf("redirect to a non-https URL %s", req.URL.Redacted())
	497	+	}
	498	+	if len(via) >= 5 {
	499	+	return errors.New("too many redirects")
	500	+	}
	501	+	return nil
	502	+	},
	503	+	}
	504	+	}
	505	+
	506	+	// VerifySum compares two hex SHA-256 values.
	507	+	func VerifySum(got, want string) error {
	508	+	if !strings.EqualFold(got, want) {
	509	+	return fmt.Errorf("SHA-256 mismatch: got %s, want %s", got, want)
	510	+	}
	511	+	return nil
	512	+	}
	513	+
	514	+	// prune removes everything in Dir that is not pinned: other versions, other
	515	+	// architectures, leftovers of an interrupted download. The reasons of
	516	+	// unavailable tools (a few bytes each) may be bind-mounted into a running
	517	+	// container, so those are only removed once they are an hour old.
	518	+	func (m *Manager) prune(log zerowrap.Logger, keep map[string]bool) {
	519	+	ents, err := os.ReadDir(m.Dir)
	520	+	if err != nil {
	521	+	return
	522	+	}
	523	+	for _, e := range ents {
	524	+	switch name := e.Name(); {
	525	+	case keep[name]:
	526	+	case name == unavailableDir:
	527	+	m.pruneUnavailable(keep)
	528	+	default:
	529	+	if err := os.RemoveAll(filepath.Join(m.Dir, name)); err != nil {
	530	+	log.Warn().Err(err).Str("path", name).Msg("removing an unpinned external tool")
	531	+	}
	532	+	}
	533	+	}
	534	+	}
	535	+
	536	+	func (m *Manager) pruneUnavailable(keep map[string]bool) {
	537	+	dir := filepath.Join(m.Dir, unavailableDir)
	538	+	ents, _ := os.ReadDir(dir)
	539	+	for _, e := range ents {
	540	+	fi, err := e.Info()
	541	+	if keep[e.Name()] \|\| err != nil \|\| time.Since(fi.ModTime()) < time.Hour {
	542	+	continue
	543	+	}
	544	+	_ = os.RemoveAll(filepath.Join(dir, e.Name()))
	545	+	}
	546	+	}

A internal/externals/table.go +94 -0

@@ -0,0 +1,94 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package externals
	4	+
	5	+	import "runtime"
	6	+
	7	+	// Artifact is one pinned external tool: where it comes from and the SHA-256 of
	8	+	// the archive. The pins mirror actions/runner's Misc/externals.sh (node20
	9	+	// 20.20.2, node24 24.21.0), so a workflow sees the Node.js it would see on a
	10	+	// GitHub-hosted runner. The checksums of the nodejs.org builds come from the
	11	+	// official SHASUMS256.txt of each release (those of the alpine builds are the
	12	+	// digests of the GitHub release assets), and the archives are the .tar.gz ones
	13	+	// (the Go standard library has no xz).
	14	+	type Artifact struct {
	15	+	// Name is the directory of the GitHub layout, below /home/runner/externals:
	16	+	// node20, node24, node20_alpine, node24_alpine.
	17	+	Name string
	18	+	// Version is the Node.js version without the leading "v".
	19	+	Version string
	20	+	// Arch is GitHub's architecture name: x64 or arm64.
	21	+	Arch string
	22	+	// URL is the https location of the .tar.gz.
	23	+	URL string
	24	+	// SHA256 is the lower-case hex SHA-256 of the archive.
	25	+	SHA256 string
	26	+	// StripTop: the archive has one top directory to drop (nodejs.org); the
	27	+	// actions/alpine_nodejs archives start at ./bin/node.
	28	+	StripTop bool
	29	+	}
	30	+
	31	+	// dirName is the directory of the artifact below the externals directory.
	32	+	func (a Artifact) dirName() string { return a.Name + "-" + a.Version + "-" + a.Arch }
	33	+
	34	+	// The versions of actions/runner's externals.sh.
	35	+	const (
	36	+	node20Version = "20.20.2"
	37	+	node24Version = "24.21.0"
	38	+	)
	39	+
	40	+	const (
	41	+	nodeDist = "https://nodejs.org/dist/"
	42	+	alpineDist = "https://github.com/actions/alpine_nodejs/releases/download/"
	43	+	)
	44	+
	45	+	func nodejs(name, version, arch, sum string) Artifact {
	46	+	return Artifact{
	47	+	Name: name, Version: version, Arch: arch, SHA256: sum, StripTop: true,
	48	+	URL: nodeDist + "v" + version + "/node-v" + version + "-linux-" + arch + ".tar.gz",
	49	+	}
	50	+	}
	51	+
	52	+	// alpine is a build of actions/alpine_nodejs (musl, x64 only), for job
	53	+	// containers based on Alpine.
	54	+	func alpine(name, version, sum string) Artifact {
	55	+	return Artifact{
	56	+	Name: name, Version: version, Arch: "x64", SHA256: sum,
	57	+	URL: alpineDist + "v" + version + "/node-v" + version + "-alpine-x64.tar.gz",
	58	+	}
	59	+	}
	60	+
	61	+	// Pinned is every artifact the runner knows, for all architectures.
	62	+	var Pinned = []Artifact{
	63	+	nodejs("node20", node20Version, "x64", "19e56f0825510207dd904f087fe52faa0a4eb6b2aab5f0ea7a33830d04888b8b"),
	64	+	nodejs("node20", node20Version, "arm64", "47ef73d543ecf6eb19435f6c03a0ac4809b3bf0dd6b26c7c571efc2a6572a74d"),
	65	+	nodejs("node24", node24Version, "x64", "6e1db87ef58b8819e5d5402eff1536491b18edd8eb7bee5ef7897876e88dc5ff"),
	66	+	nodejs("node24", node24Version, "arm64", "724282c3b43aec998aa9527380465b45d229e021b58035f5f4f63095eabfe5d5"),
	67	+	// actions/runner ships the alpine builds for x64 only. The checksums are
	68	+	// those of the GitHub release assets.
	69	+	alpine("node20_alpine", node20Version, "f21a2253025a5d1a14332a0b1ed48871689c5ca9aa37a6141428944b75de7d91"),
	70	+	alpine("node24_alpine", node24Version, "2a4a9c6af9b6f1dd67e4c460474a6fbb64fafffe49e1166af80715bfd4e50775"),
	71	+	}
	72	+
	73	+	// HostArch is GitHub's name for the architecture of this host ("" when the
	74	+	// runner has no build for it).
	75	+	func HostArch() string {
	76	+	switch runtime.GOARCH {
	77	+	case "amd64":
	78	+	return "x64"
	79	+	case "arm64":
	80	+	return "arm64"
	81	+	}
	82	+	return ""
	83	+	}
	84	+
	85	+	// ForArch selects the artifacts of one architecture, in table order.
	86	+	func ForArch(all []Artifact, arch string) []Artifact {
	87	+	var out []Artifact
	88	+	for _, a := range all {
	89	+	if a.Arch == arch {
	90	+	out = append(out, a)
	91	+	}
	92	+	}
	93	+	return out
	94	+	}

@@ -0,0 +1,94 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package externals
	4	+
	5	+	import "runtime"
	6	+
	7	+	// Artifact is one pinned external tool: where it comes from and the SHA-256 of
	8	+	// the archive. The pins mirror actions/runner's Misc/externals.sh (node20
	9	+	// 20.20.2, node24 24.21.0), so a workflow sees the Node.js it would see on a
	10	+	// GitHub-hosted runner. The checksums of the nodejs.org builds come from the
	11	+	// official SHASUMS256.txt of each release (those of the alpine builds are the
	12	+	// digests of the GitHub release assets), and the archives are the .tar.gz ones
	13	+	// (the Go standard library has no xz).
	14	+	type Artifact struct {
	15	+	// Name is the directory of the GitHub layout, below /home/runner/externals:
	16	+	// node20, node24, node20_alpine, node24_alpine.
	17	+	Name string
	18	+	// Version is the Node.js version without the leading "v".
	19	+	Version string
	20	+	// Arch is GitHub's architecture name: x64 or arm64.
	21	+	Arch string
	22	+	// URL is the https location of the .tar.gz.
	23	+	URL string
	24	+	// SHA256 is the lower-case hex SHA-256 of the archive.
	25	+	SHA256 string
	26	+	// StripTop: the archive has one top directory to drop (nodejs.org); the
	27	+	// actions/alpine_nodejs archives start at ./bin/node.
	28	+	StripTop bool
	29	+	}
	30	+
	31	+	// dirName is the directory of the artifact below the externals directory.
	32	+	func (a Artifact) dirName() string { return a.Name + "-" + a.Version + "-" + a.Arch }
	33	+
	34	+	// The versions of actions/runner's externals.sh.
	35	+	const (
	36	+	node20Version = "20.20.2"
	37	+	node24Version = "24.21.0"
	38	+	)
	39	+
	40	+	const (
	41	+	nodeDist = "https://nodejs.org/dist/"
	42	+	alpineDist = "https://github.com/actions/alpine_nodejs/releases/download/"
	43	+	)
	44	+
	45	+	func nodejs(name, version, arch, sum string) Artifact {
	46	+	return Artifact{
	47	+	Name: name, Version: version, Arch: arch, SHA256: sum, StripTop: true,
	48	+	URL: nodeDist + "v" + version + "/node-v" + version + "-linux-" + arch + ".tar.gz",
	49	+	}
	50	+	}
	51	+
	52	+	// alpine is a build of actions/alpine_nodejs (musl, x64 only), for job
	53	+	// containers based on Alpine.
	54	+	func alpine(name, version, sum string) Artifact {
	55	+	return Artifact{
	56	+	Name: name, Version: version, Arch: "x64", SHA256: sum,
	57	+	URL: alpineDist + "v" + version + "/node-v" + version + "-alpine-x64.tar.gz",
	58	+	}
	59	+	}
	60	+
	61	+	// Pinned is every artifact the runner knows, for all architectures.
	62	+	var Pinned = []Artifact{
	63	+	nodejs("node20", node20Version, "x64", "19e56f0825510207dd904f087fe52faa0a4eb6b2aab5f0ea7a33830d04888b8b"),
	64	+	nodejs("node20", node20Version, "arm64", "47ef73d543ecf6eb19435f6c03a0ac4809b3bf0dd6b26c7c571efc2a6572a74d"),
	65	+	nodejs("node24", node24Version, "x64", "6e1db87ef58b8819e5d5402eff1536491b18edd8eb7bee5ef7897876e88dc5ff"),
	66	+	nodejs("node24", node24Version, "arm64", "724282c3b43aec998aa9527380465b45d229e021b58035f5f4f63095eabfe5d5"),
	67	+	// actions/runner ships the alpine builds for x64 only. The checksums are
	68	+	// those of the GitHub release assets.
	69	+	alpine("node20_alpine", node20Version, "f21a2253025a5d1a14332a0b1ed48871689c5ca9aa37a6141428944b75de7d91"),
	70	+	alpine("node24_alpine", node24Version, "2a4a9c6af9b6f1dd67e4c460474a6fbb64fafffe49e1166af80715bfd4e50775"),
	71	+	}
	72	+
	73	+	// HostArch is GitHub's name for the architecture of this host ("" when the
	74	+	// runner has no build for it).
	75	+	func HostArch() string {
	76	+	switch runtime.GOARCH {
	77	+	case "amd64":
	78	+	return "x64"
	79	+	case "arm64":
	80	+	return "arm64"
	81	+	}
	82	+	return ""
	83	+	}
	84	+
	85	+	// ForArch selects the artifacts of one architecture, in table order.
	86	+	func ForArch(all []Artifact, arch string) []Artifact {
	87	+	var out []Artifact
	88	+	for _, a := range all {
	89	+	if a.Arch == arch {
	90	+	out = append(out, a)
	91	+	}
	92	+	}
	93	+	return out
	94	+	}

M internal/guest/vmsetup.go +13 -0


@@ -18,6 +18,9 @@ import (
18	18		const (
19	19		devBase = "/dev/vdb" // shared read-only base image (ext4)
20	20		devScratch = "/dev/vdc" // per-job scratch disk (ext4)
	21	+	// devExternals is the host's external tools (Node.js), attached last and only
	22	+	// when the command line says ermite.externals=1.
	23	+	devExternals = "/dev/vdd"
21	24		)
22	25
23	26		// KernelArg names of the clock and DNS settings (not part of the contract

@@ -79,6 +82,16 @@ func setupVM(log zerowrap.Logger, args map[string]string) error {
79	82		return err
80	83		}
81	84		}
	85	+	// The host's external tools, read-only, in GitHub's layout. A failure is not
	86	+	// fatal: only the jobs that need Node.js are affected, and they say so.
	87	+	if args[sandbox.CmdlineExternals] == "1" {
	88	+	dst := "/mnt/newroot" + sandbox.GuestExternalsDir
	89	+	if err := os.MkdirAll(dst, 0o755); err != nil {
	90	+	log.Warn().Err(err).Msg("init: externals mountpoint")
	91	+	} else if err := mount(devExternals, dst, "ext4", unix.MS_RDONLY\|unix.MS_NOSUID\|unix.MS_NODEV, ""); err != nil {
	92	+	log.Warn().Err(err).Msg("init: mount the externals disk")
	93	+	}
	94	+	}
82	95		// Scratch and the exec disk stay reachable after the pivot.
83	96		if err := mount("/mnt/scratch", "/mnt/newroot"+sandbox.GuestScratch, "", unix.MS_BIND, ""); err != nil {
84	97		return err


@@ -18,6 +18,9 @@ import (
18	const (	18		const (
19	devBase = "/dev/vdb" // shared read-only base image (ext4)	19		devBase = "/dev/vdb" // shared read-only base image (ext4)
20	devScratch = "/dev/vdc" // per-job scratch disk (ext4)	20		devScratch = "/dev/vdc" // per-job scratch disk (ext4)
		21	+	// devExternals is the host's external tools (Node.js), attached last and only
		22	+	// when the command line says ermite.externals=1.
		23	+	devExternals = "/dev/vdd"
21	)	24		)
22		25
23	// KernelArg names of the clock and DNS settings (not part of the contract	26		// KernelArg names of the clock and DNS settings (not part of the contract

@@ -79,6 +82,16 @@ func setupVM(log zerowrap.Logger, args map[string]string) error {
79	return err	82		return err
80	}	83		}
81	}	84		}
		85	+	// The host's external tools, read-only, in GitHub's layout. A failure is not
		86	+	// fatal: only the jobs that need Node.js are affected, and they say so.
		87	+	if args[sandbox.CmdlineExternals] == "1" {
		88	+	dst := "/mnt/newroot" + sandbox.GuestExternalsDir
		89	+	if err := os.MkdirAll(dst, 0o755); err != nil {
		90	+	log.Warn().Err(err).Msg("init: externals mountpoint")
		91	+	} else if err := mount(devExternals, dst, "ext4", unix.MS_RDONLY\|unix.MS_NOSUID\|unix.MS_NODEV, ""); err != nil {
		92	+	log.Warn().Err(err).Msg("init: mount the externals disk")
		93	+	}
		94	+	}
82	// Scratch and the exec disk stay reachable after the pivot.	95		// Scratch and the exec disk stay reachable after the pivot.
83	if err := mount("/mnt/scratch", "/mnt/newroot"+sandbox.GuestScratch, "", unix.MS_BIND, ""); err != nil {	96		if err := mount("/mnt/scratch", "/mnt/newroot"+sandbox.GuestScratch, "", unix.MS_BIND, ""); err != nil {
84	return err	97		return err

M internal/sandbox/config.go +20 -0


@@ -36,6 +36,9 @@ const (
36	36		EnvJobDiskBW = "ERMITE_RUNNER_JOB_DISK_BANDWIDTH"
37	37		EnvJobDiskIOPS = "ERMITE_RUNNER_JOB_DISK_IOPS"
38	38		EnvJobNetBandwidth = "ERMITE_RUNNER_JOB_NET_BANDWIDTH"
	39	+	// EnvNode=0 turns the Node.js externals off (nothing is downloaded or
	40	+	// attached).
	41	+	EnvNode = "ERMITE_RUNNER_NODE"
39	42		)
40	43
41	44		// Defaults.

@@ -117,6 +120,14 @@ type Config struct {
117	120		GuestForge *GuestForge
118	121		// Rate limits of micro-VMs; zero disables.
119	122		DiskBandwidthMiB, DiskIOPS, NetMbit int
	123	+
	124	+	// NodeExternals is false when ERMITE_RUNNER_NODE=0: the caller then sets no
	125	+	// Externals.
	126	+	NodeExternals bool
	127	+	// Externals, when not nil, is the host's set of external tools (Node.js),
	128	+	// attached read-only to every sandbox at GuestExternalsDir. The caller sets
	129	+	// it before building the drivers, like GuestForge.
	130	+	Externals ExternalsSource
120	131		}
121	132
122	133		// LoadConfig reads the sandbox settings from getenv (os.Getenv in production).

@@ -134,6 +145,8 @@ func LoadConfig(getenv func(string) string) (Config, error) {
134	145		JobPids: DefaultJobPids,
135	146		LockFile: DefaultLockFile,
136	147
	148	+	NodeExternals: true,
	149	+
137	150		DiskBandwidthMiB: DefaultDiskBandwidthMiB,
138	151		DiskIOPS: DefaultDiskIOPS,
139	152		NetMbit: DefaultNetMbit,

@@ -203,6 +216,13 @@ func LoadConfig(getenv func(string) string) (Config, error) {
203	216		default:
204	217		return c, fmt.Errorf("%s: %q is not 0 or 1", EnvGuestForge, getenv(EnvGuestForge))
205	218		}
	219	+	switch strings.ToLower(strings.TrimSpace(getenv(EnvNode))) {
	220	+	case "", "1", "true", "yes":
	221	+	case "0", "false", "no":
	222	+	c.NodeExternals = false
	223	+	default:
	224	+	return c, fmt.Errorf("%s: %q is not 0 or 1", EnvNode, getenv(EnvNode))
	225	+	}
206	226		if v := getenv(EnvLockFile); v != "" {
207	227		if !filepath.IsAbs(v) {
208	228		return c, fmt.Errorf("%s: %q must be an absolute path", EnvLockFile, v)


@@ -36,6 +36,9 @@ const (
36	EnvJobDiskBW = "ERMITE_RUNNER_JOB_DISK_BANDWIDTH"	36		EnvJobDiskBW = "ERMITE_RUNNER_JOB_DISK_BANDWIDTH"
37	EnvJobDiskIOPS = "ERMITE_RUNNER_JOB_DISK_IOPS"	37		EnvJobDiskIOPS = "ERMITE_RUNNER_JOB_DISK_IOPS"
38	EnvJobNetBandwidth = "ERMITE_RUNNER_JOB_NET_BANDWIDTH"	38		EnvJobNetBandwidth = "ERMITE_RUNNER_JOB_NET_BANDWIDTH"
		39	+	// EnvNode=0 turns the Node.js externals off (nothing is downloaded or
		40	+	// attached).
		41	+	EnvNode = "ERMITE_RUNNER_NODE"
39	)	42		)
40		43
41	// Defaults.	44		// Defaults.

@@ -117,6 +120,14 @@ type Config struct {
117	GuestForge *GuestForge	120		GuestForge *GuestForge
118	// Rate limits of micro-VMs; zero disables.	121		// Rate limits of micro-VMs; zero disables.
119	DiskBandwidthMiB, DiskIOPS, NetMbit int	122		DiskBandwidthMiB, DiskIOPS, NetMbit int
		123	+
		124	+	// NodeExternals is false when ERMITE_RUNNER_NODE=0: the caller then sets no
		125	+	// Externals.
		126	+	NodeExternals bool
		127	+	// Externals, when not nil, is the host's set of external tools (Node.js),
		128	+	// attached read-only to every sandbox at GuestExternalsDir. The caller sets
		129	+	// it before building the drivers, like GuestForge.
		130	+	Externals ExternalsSource
120	}	131		}
121		132
122	// LoadConfig reads the sandbox settings from getenv (os.Getenv in production).	133		// LoadConfig reads the sandbox settings from getenv (os.Getenv in production).

@@ -134,6 +145,8 @@ func LoadConfig(getenv func(string) string) (Config, error) {
134	JobPids: DefaultJobPids,	145		JobPids: DefaultJobPids,
135	LockFile: DefaultLockFile,	146		LockFile: DefaultLockFile,
136		147
		148	+	NodeExternals: true,
		149	+
137	DiskBandwidthMiB: DefaultDiskBandwidthMiB,	150		DiskBandwidthMiB: DefaultDiskBandwidthMiB,
138	DiskIOPS: DefaultDiskIOPS,	151		DiskIOPS: DefaultDiskIOPS,
139	NetMbit: DefaultNetMbit,	152		NetMbit: DefaultNetMbit,

@@ -203,6 +216,13 @@ func LoadConfig(getenv func(string) string) (Config, error) {
203	default:	216		default:
204	return c, fmt.Errorf("%s: %q is not 0 or 1", EnvGuestForge, getenv(EnvGuestForge))	217		return c, fmt.Errorf("%s: %q is not 0 or 1", EnvGuestForge, getenv(EnvGuestForge))
205	}	218		}
		219	+	switch strings.ToLower(strings.TrimSpace(getenv(EnvNode))) {
		220	+	case "", "1", "true", "yes":
		221	+	case "0", "false", "no":
		222	+	c.NodeExternals = false
		223	+	default:
		224	+	return c, fmt.Errorf("%s: %q is not 0 or 1", EnvNode, getenv(EnvNode))
		225	+	}
206	if v := getenv(EnvLockFile); v != "" {	226		if v := getenv(EnvLockFile); v != "" {
207	if !filepath.IsAbs(v) {	227		if !filepath.IsAbs(v) {
208	return c, fmt.Errorf("%s: %q must be an absolute path", EnvLockFile, v)	228		return c, fmt.Errorf("%s: %q must be an absolute path", EnvLockFile, v)

M internal/sandbox/container/container_test.go +82 -0


@@ -1084,3 +1084,85 @@ func TestNoGuestForgeByDefault(t *testing.T) {
1084	1084		t.Fatalf("override %q", n)
1085	1085		}
1086	1086		}
	1087	+
	1088	+	// The host's external tools are bind-mounted read-only at
	1089	+	// /home/runner/externals/<name> (GitHub's layout), whichever the isolation.
	1090	+	func TestExternalsAreBoundReadOnly(t *testing.T) {
	1091	+	e := newCEnv(t)
	1092	+	src := &sandboxtest.Externals{}
	1093	+	n24 := sandboxtest.MakeExternal(t, "node24", "a", "v24")
	1094	+	n20 := sandboxtest.MakeExternal(t, "node20", "b", "v20")
	1095	+	gone := sandbox.External{Name: "node24_alpine", ID: "c", Dir: "/state/externals/.unavailable/x", Unavailable: true}
	1096	+	src.Set(n24, n20, gone)
	1097	+	e.cfg.Externals = src
	1098	+	d := e.driver(t)
	1099	+	run := func() containerSpec {
	1100	+	t.Helper()
	1101	+	sb, _ := d.New()
	1102	+	conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
	1103	+	if err != nil {
	1104	+	t.Fatal(err)
	1105	+	}
	1106	+	stub(t, conn)
	1107	+	e.eng.mu.Lock()
	1108	+	var spec containerSpec
	1109	+	for _, s := range e.eng.containers {
	1110	+	spec = s
	1111	+	}
	1112	+	e.eng.mu.Unlock()
	1113	+	_ = sb.Stop(context.Background())
	1114	+	return spec
	1115	+	}
	1116	+	spec := run()
	1117	+	got := map[string]bindMount{}
	1118	+	for _, b := range spec.Binds {
	1119	+	got[b.Target] = b
	1120	+	}
	1121	+	for _, w := range []sandbox.External{n24, n20, gone} {
	1122	+	b, ok := got["/home/runner/externals/"+w.Name]
	1123	+	if !ok \|\| !b.ReadOnly \|\| b.Source != w.Dir {
	1124	+	t.Errorf("%s: %+v (binds %+v)", w.Name, b, spec.Binds)
	1125	+	}
	1126	+	}
	1127	+	// The executable and the other binds are untouched.
	1128	+	if b := got["/opt/ermite/ermite-runner"]; !b.ReadOnly \|\| b.Source != e.cfg.Executable {
	1129	+	t.Errorf("executable bind: %+v", b)
	1130	+	}
	1131	+	if len(spec.Binds) != 5+3 {
	1132	+	t.Errorf("binds: %+v", spec.Binds)
	1133	+	}
	1134	+
	1135	+	// A new Node.js is picked up by the next job, without a driver restart.
	1136	+	src.Set(sandboxtest.MakeExternal(t, "node24", "d", "v24.22"))
	1137	+	spec = run()
	1138	+	n := 0
	1139	+	for _, b := range spec.Binds {
	1140	+	if strings.HasPrefix(b.Target, "/home/runner/externals/") {
	1141	+	n++
	1142	+	}
	1143	+	}
	1144	+	if n != 1 {
	1145	+	t.Errorf("externals binds after the change: %+v", spec.Binds)
	1146	+	}
	1147	+	}
	1148	+
	1149	+	func TestNoExternalsNoBinds(t *testing.T) {
	1150	+	e := newCEnv(t)
	1151	+	d := e.driver(t)
	1152	+	sb, _ := d.New()
	1153	+	conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
	1154	+	if err != nil {
	1155	+	t.Fatal(err)
	1156	+	}
	1157	+	stub(t, conn)
	1158	+	defer func() { _ = sb.Stop(context.Background()) }()
	1159	+	e.eng.mu.Lock()
	1160	+	defer e.eng.mu.Unlock()
	1161	+	for _, s := range e.eng.containers {
	1162	+	for _, b := range s.Binds {
	1163	+	if strings.Contains(b.Target, "externals") {
	1164	+	t.Errorf("unexpected bind %+v", b)
	1165	+	}
	1166	+	}
	1167	+	}
	1168	+	}


@@ -1084,3 +1084,85 @@ func TestNoGuestForgeByDefault(t *testing.T) {
1084	t.Fatalf("override %q", n)	1084		t.Fatalf("override %q", n)
1085	}	1085		}
1086	}	1086		}
		1087	+
		1088	+	// The host's external tools are bind-mounted read-only at
		1089	+	// /home/runner/externals/<name> (GitHub's layout), whichever the isolation.
		1090	+	func TestExternalsAreBoundReadOnly(t *testing.T) {
		1091	+	e := newCEnv(t)
		1092	+	src := &sandboxtest.Externals{}
		1093	+	n24 := sandboxtest.MakeExternal(t, "node24", "a", "v24")
		1094	+	n20 := sandboxtest.MakeExternal(t, "node20", "b", "v20")
		1095	+	gone := sandbox.External{Name: "node24_alpine", ID: "c", Dir: "/state/externals/.unavailable/x", Unavailable: true}
		1096	+	src.Set(n24, n20, gone)
		1097	+	e.cfg.Externals = src
		1098	+	d := e.driver(t)
		1099	+	run := func() containerSpec {
		1100	+	t.Helper()
		1101	+	sb, _ := d.New()
		1102	+	conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
		1103	+	if err != nil {
		1104	+	t.Fatal(err)
		1105	+	}
		1106	+	stub(t, conn)
		1107	+	e.eng.mu.Lock()
		1108	+	var spec containerSpec
		1109	+	for _, s := range e.eng.containers {
		1110	+	spec = s
		1111	+	}
		1112	+	e.eng.mu.Unlock()
		1113	+	_ = sb.Stop(context.Background())
		1114	+	return spec
		1115	+	}
		1116	+	spec := run()
		1117	+	got := map[string]bindMount{}
		1118	+	for _, b := range spec.Binds {
		1119	+	got[b.Target] = b
		1120	+	}
		1121	+	for _, w := range []sandbox.External{n24, n20, gone} {
		1122	+	b, ok := got["/home/runner/externals/"+w.Name]
		1123	+	if !ok \|\| !b.ReadOnly \|\| b.Source != w.Dir {
		1124	+	t.Errorf("%s: %+v (binds %+v)", w.Name, b, spec.Binds)
		1125	+	}
		1126	+	}
		1127	+	// The executable and the other binds are untouched.
		1128	+	if b := got["/opt/ermite/ermite-runner"]; !b.ReadOnly \|\| b.Source != e.cfg.Executable {
		1129	+	t.Errorf("executable bind: %+v", b)
		1130	+	}
		1131	+	if len(spec.Binds) != 5+3 {
		1132	+	t.Errorf("binds: %+v", spec.Binds)
		1133	+	}
		1134	+
		1135	+	// A new Node.js is picked up by the next job, without a driver restart.
		1136	+	src.Set(sandboxtest.MakeExternal(t, "node24", "d", "v24.22"))
		1137	+	spec = run()
		1138	+	n := 0
		1139	+	for _, b := range spec.Binds {
		1140	+	if strings.HasPrefix(b.Target, "/home/runner/externals/") {
		1141	+	n++
		1142	+	}
		1143	+	}
		1144	+	if n != 1 {
		1145	+	t.Errorf("externals binds after the change: %+v", spec.Binds)
		1146	+	}
		1147	+	}
		1148	+
		1149	+	func TestNoExternalsNoBinds(t *testing.T) {
		1150	+	e := newCEnv(t)
		1151	+	d := e.driver(t)
		1152	+	sb, _ := d.New()
		1153	+	conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
		1154	+	if err != nil {
		1155	+	t.Fatal(err)
		1156	+	}
		1157	+	stub(t, conn)
		1158	+	defer func() { _ = sb.Stop(context.Background()) }()
		1159	+	e.eng.mu.Lock()
		1160	+	defer e.eng.mu.Unlock()
		1161	+	for _, s := range e.eng.containers {
		1162	+	for _, b := range s.Binds {
		1163	+	if strings.Contains(b.Target, "externals") {
		1164	+	t.Errorf("unexpected bind %+v", b)
		1165	+	}
		1166	+	}
		1167	+	}
		1168	+	}

M internal/sandbox/container/driver.go +25 -12


@@ -13,8 +13,8 @@
13	13		// it after a crash.
14	14		//
15	15		// The executor stream is a Unix socket in a per-job directory of the runner's
16		-	// state directory, bind-mounted into the container. The runner executable is
17		-	// bind-mounted read-only. Both paths are host paths given to the Docker daemon:
	16	+	// state directory, bind-mounted into the container. The runner executable and
	17	+	// the host's external tools (Node.js) are bind-mounted read-only. Both paths are host paths given to the Docker daemon:
18	18		// the daemon must run on the same host (and mount namespace) as the runner.
19	19		package container
20	20

@@ -435,6 +435,14 @@ func (s *containerSandbox) runNetwork(ctx context.Context, netw netpolicy.Net, l
435	435		func (s *containerSandbox) runContainer(ctx context.Context, spec sandbox.JobSpec, netw netpolicy.Net, labels map[string]string) (<-chan exitResult, error) {
436	436		d := s.d
437	437		entry := sandbox.GuestExecDir + "/" + sandbox.GuestExecName
	438	+	binds := []bindMount{
	439	+	{Source: d.exe, Target: entry, ReadOnly: true},
	440	+	{Source: d.sockDir(s.id), Target: sandbox.ContainerRunDir},
	441	+	{Source: d.scratchDir(s.id), Target: sandbox.GuestScratch},
	442	+	{Source: d.workDir(s.id), Target: containerWorkspace},
	443	+	{Source: filepath.Join(d.scratchDir(s.id), "tmp"), Target: containerTmp},
	444	+	}
	445	+	binds = append(binds, externalBinds(d.cfg.ExternalsOf())...)
438	446		cid, err := d.eng.CreateContainer(ctx, containerSpec{
439	447		Name: "ermite-" + s.id,
440	448		Image: d.image,

@@ -446,16 +454,10 @@ func (s *containerSandbox) runContainer(ctx context.Context, spec sandbox.JobSpe
446	454		Network: "ermite-" + s.id,
447	455		IPv4: netw.Guest,
448	456		DNS: netw.Gateway,
449		-	Binds: []bindMount{
450		-	{Source: d.exe, Target: entry, ReadOnly: true},
451		-	{Source: d.sockDir(s.id), Target: sandbox.ContainerRunDir},
452		-	{Source: d.scratchDir(s.id), Target: sandbox.GuestScratch},
453		-	{Source: d.workDir(s.id), Target: containerWorkspace},
454		-	{Source: filepath.Join(d.scratchDir(s.id), "tmp"), Target: containerTmp},
455		-	},
456		-	MemoryMiB: spec.MemoryMiB,
457		-	CPUs: spec.CPUs,
458		-	Pids: d.cfg.JobPids,
	457	+	Binds: binds,
	458	+	MemoryMiB: spec.MemoryMiB,
	459	+	CPUs: spec.CPUs,
	460	+	Pids: d.cfg.JobPids,
459	461		})
460	462		if err != nil {
461	463		return nil, err

@@ -468,6 +470,17 @@ func (s *containerSandbox) runContainer(ctx context.Context, spec sandbox.JobSpe
468	470		return exited, nil
469	471		}
470	472
	473	+	// externalBinds are the read-only bind mounts of the host's external tools
	474	+	// (Node.js) at /home/runner/externals/<name>, GitHub's layout. A container needs
	475	+	// no more than the host path: it reads the tool directly.
	476	+	func externalBinds(set []sandbox.External) []bindMount {
	477	+	var out []bindMount
	478	+	for _, e := range set {
	479	+	out = append(out, bindMount{Source: e.Dir, Target: sandbox.GuestExternalsDir + "/" + e.Name, ReadOnly: true})
	480	+	}
	481	+	return out
	482	+	}
	483	+
471	484		// waitExecutor waits for the executor to connect. Only ctx (the caller's
472	485		// context, or Stop through the boot gate), the container exiting, and the
473	486		// timeout end the wait.

 // it after a crash.
 //
 // The executor stream is a Unix socket in a per-job directory of the runner's
-// state directory, bind-mounted into the container. The runner executable is
+// state directory, bind-mounted into the container. The runner executable and
-// bind-mounted read-only. Both paths are host paths given to the Docker daemon:
+// the host's external tools (Node.js) are bind-mounted read-only. Both paths are host paths given to the Docker daemon:
 // the daemon must run on the same host (and mount namespace) as the runner.
 package container
 func (s *containerSandbox) runContainer(ctx context.Context, spec sandbox.JobSpec, netw netpolicy.Net, labels map[string]string) (<-chan exitResult, error) {
 	d := s.d
 	entry := sandbox.GuestExecDir + "/" + sandbox.GuestExecName
+	binds := []bindMount{
+		{Source: d.exe, Target: entry, ReadOnly: true},
+		{Source: d.sockDir(s.id), Target: sandbox.ContainerRunDir},
+		{Source: d.scratchDir(s.id), Target: sandbox.GuestScratch},
+		{Source: d.workDir(s.id), Target: containerWorkspace},
+		{Source: filepath.Join(d.scratchDir(s.id), "tmp"), Target: containerTmp},
+	}
+	binds = append(binds, externalBinds(d.cfg.ExternalsOf())...)
 	cid, err := d.eng.CreateContainer(ctx, containerSpec{
 		Name:       "ermite-" + s.id,
 		Image:      d.image,
 		Network:    "ermite-" + s.id,
 		IPv4:       netw.Guest,
 		DNS:        netw.Gateway,
-		Binds: []bindMount{
+		Binds:      binds,
-			{Source: d.exe, Target: entry, ReadOnly: true},
+		MemoryMiB:  spec.MemoryMiB,
-			{Source: d.sockDir(s.id), Target: sandbox.ContainerRunDir},
+		CPUs:       spec.CPUs,
-			{Source: d.scratchDir(s.id), Target: sandbox.GuestScratch},
+		Pids:       d.cfg.JobPids,
-			{Source: d.workDir(s.id), Target: containerWorkspace},
-			{Source: filepath.Join(d.scratchDir(s.id), "tmp"), Target: containerTmp},
-		},
-		MemoryMiB: spec.MemoryMiB,
-		CPUs:      spec.CPUs,
-		Pids:      d.cfg.JobPids,
 	})
 	if err != nil {
 		return nil, err
 	return exited, nil
 }
+// externalBinds are the read-only bind mounts of the host's external tools
+// (Node.js) at /home/runner/externals/<name>, GitHub's layout. A container needs
+// no more than the host path: it reads the tool directly.
+func externalBinds(set []sandbox.External) []bindMount {
+	var out []bindMount
+	for _, e := range set {
+		out = append(out, bindMount{Source: e.Dir, Target: sandbox.GuestExternalsDir + "/" + e.Name, ReadOnly: true})
+	}
+	return out
+}
 // waitExecutor waits for the executor to connect. Only ctx (the caller's
 // context, or Stop through the boot gate), the container exiting, and the
 // timeout end the wait.

M internal/sandbox/contract.go +9 -0


@@ -16,6 +16,13 @@ const (
16	16		// ContainerSocket is the executor stream socket inside it.
17	17		ContainerRunDir = "/run/ermite"
18	18		ContainerSocket = "exec.sock"
	19	+	// GuestExternalsDir is where the host's external tools (Node.js) appear in
	20	+	// the sandbox, read-only, in GitHub's layout: <dir>/node24/bin/node.
	21	+	GuestExternalsDir = "/home/runner/externals"
	22	+	// ExternalUnavailableFile, in <GuestExternalsDir>/<name>, tells why a tool
	23	+	// is missing on this runner. A directory that has it has no tool (its
	24	+	// bin/node only prints the reason and fails).
	25	+	ExternalUnavailableFile = "UNAVAILABLE"
19	26		// GuestScratch is where the per-job scratch disk (VM) or volume (container)
20	27		// is mounted; dockerd keeps its data root there.
21	28		GuestScratch = "/scratch"

@@ -27,4 +34,6 @@ const (
27	34		CmdlineSandbox = "ermite.sandbox" // "vm"
28	35		CmdlineConnect = "ermite.connect" // "vsock:1024"
29	36		CmdlineGateway = "ermite.gw" // host side of the /30, also the DNS server
	37	+	// CmdlineExternals=1: the externals disk is attached as the fourth drive.
	38	+	CmdlineExternals = "ermite.externals"
30	39		)


@@ -16,6 +16,13 @@ const (
16	// ContainerSocket is the executor stream socket inside it.	16		// ContainerSocket is the executor stream socket inside it.
17	ContainerRunDir = "/run/ermite"	17		ContainerRunDir = "/run/ermite"
18	ContainerSocket = "exec.sock"	18		ContainerSocket = "exec.sock"
		19	+	// GuestExternalsDir is where the host's external tools (Node.js) appear in
		20	+	// the sandbox, read-only, in GitHub's layout: <dir>/node24/bin/node.
		21	+	GuestExternalsDir = "/home/runner/externals"
		22	+	// ExternalUnavailableFile, in <GuestExternalsDir>/<name>, tells why a tool
		23	+	// is missing on this runner. A directory that has it has no tool (its
		24	+	// bin/node only prints the reason and fails).
		25	+	ExternalUnavailableFile = "UNAVAILABLE"
19	// GuestScratch is where the per-job scratch disk (VM) or volume (container)	26		// GuestScratch is where the per-job scratch disk (VM) or volume (container)
20	// is mounted; dockerd keeps its data root there.	27		// is mounted; dockerd keeps its data root there.
21	GuestScratch = "/scratch"	28		GuestScratch = "/scratch"

@@ -27,4 +34,6 @@ const (
27	CmdlineSandbox = "ermite.sandbox" // "vm"	34		CmdlineSandbox = "ermite.sandbox" // "vm"
28	CmdlineConnect = "ermite.connect" // "vsock:1024"	35		CmdlineConnect = "ermite.connect" // "vsock:1024"
29	CmdlineGateway = "ermite.gw" // host side of the /30, also the DNS server	36		CmdlineGateway = "ermite.gw" // host side of the /30, also the DNS server
		37	+	// CmdlineExternals=1: the externals disk is attached as the fourth drive.
		38	+	CmdlineExternals = "ermite.externals"
30	)	39		)

A internal/sandbox/externals.go +56 -0

@@ -0,0 +1,56 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package sandbox
	4	+
	5	+	import (
	6	+	"crypto/sha256"
	7	+	"encoding/hex"
	8	+	"sort"
	9	+	)
	10	+
	11	+	// External is one tool the host provides outside the image (Node.js), attached
	12	+	// read-only at GuestExternalsDir/<Name>.
	13	+	type External struct {
	14	+	// Name is the directory below GuestExternalsDir: node20, node24,
	15	+	// node20_alpine, node24_alpine.
	16	+	Name string
	17	+	// ID identifies the content: it changes whenever the files (or the reason of
	18	+	// an unavailable tool) change, so that drivers can key caches on it.
	19	+	ID string
	20	+	// Dir is the host directory that is attached. For an unavailable tool it
	21	+	// holds the file ExternalUnavailableFile with the reason, and a bin/node
	22	+	// script that prints it and fails.
	23	+	Dir string
	24	+	// Unavailable is true when the tool is missing on this runner.
	25	+	Unavailable bool
	26	+	}
	27	+
	28	+	// ExternalsSource hands the drivers the current set of externals. It is called
	29	+	// at every sandbox start and may change between calls (a download finishing).
	30	+	type ExternalsSource interface {
	31	+	Externals() []External
	32	+	}
	33	+
	34	+	// ExternalsKey is the cache key of a set: the same tools give the same key, in
	35	+	// any order, and any change of content gives another.
	36	+	func ExternalsKey(set []External) string {
	37	+	ids := make([]string, 0, len(set))
	38	+	for _, e := range set {
	39	+	ids = append(ids, e.Name+"="+e.ID)
	40	+	}
	41	+	sort.Strings(ids)
	42	+	h := sha256.New()
	43	+	for _, id := range ids {
	44	+	h.Write([]byte(id))
	45	+	h.Write([]byte{0})
	46	+	}
	47	+	return hex.EncodeToString(h.Sum(nil))[:16]
	48	+	}
	49	+
	50	+	// ExternalsOf returns the current set of cfg (nil when there is none).
	51	+	func (c Config) ExternalsOf() []External {
	52	+	if c.Externals == nil {
	53	+	return nil
	54	+	}
	55	+	return c.Externals.Externals()
	56	+	}

@@ -0,0 +1,56 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package sandbox
	4	+
	5	+	import (
	6	+	"crypto/sha256"
	7	+	"encoding/hex"
	8	+	"sort"
	9	+	)
	10	+
	11	+	// External is one tool the host provides outside the image (Node.js), attached
	12	+	// read-only at GuestExternalsDir/<Name>.
	13	+	type External struct {
	14	+	// Name is the directory below GuestExternalsDir: node20, node24,
	15	+	// node20_alpine, node24_alpine.
	16	+	Name string
	17	+	// ID identifies the content: it changes whenever the files (or the reason of
	18	+	// an unavailable tool) change, so that drivers can key caches on it.
	19	+	ID string
	20	+	// Dir is the host directory that is attached. For an unavailable tool it
	21	+	// holds the file ExternalUnavailableFile with the reason, and a bin/node
	22	+	// script that prints it and fails.
	23	+	Dir string
	24	+	// Unavailable is true when the tool is missing on this runner.
	25	+	Unavailable bool
	26	+	}
	27	+
	28	+	// ExternalsSource hands the drivers the current set of externals. It is called
	29	+	// at every sandbox start and may change between calls (a download finishing).
	30	+	type ExternalsSource interface {
	31	+	Externals() []External
	32	+	}
	33	+
	34	+	// ExternalsKey is the cache key of a set: the same tools give the same key, in
	35	+	// any order, and any change of content gives another.
	36	+	func ExternalsKey(set []External) string {
	37	+	ids := make([]string, 0, len(set))
	38	+	for _, e := range set {
	39	+	ids = append(ids, e.Name+"="+e.ID)
	40	+	}
	41	+	sort.Strings(ids)
	42	+	h := sha256.New()
	43	+	for _, id := range ids {
	44	+	h.Write([]byte(id))
	45	+	h.Write([]byte{0})
	46	+	}
	47	+	return hex.EncodeToString(h.Sum(nil))[:16]
	48	+	}
	49	+
	50	+	// ExternalsOf returns the current set of cfg (nil when there is none).
	51	+	func (c Config) ExternalsOf() []External {
	52	+	if c.Externals == nil {
	53	+	return nil
	54	+	}
	55	+	return c.Externals.Externals()
	56	+	}

M internal/sandbox/integration/vm_test.go +97 -0


@@ -4,7 +4,9 @@ package integration
4	4
5	5		import (
6	6		"context"
	7	+	"net/http"
7	8		"os"
	9	+	"path/filepath"
8	10		"strings"
9	11		"syscall"
10	12		"testing"

@@ -12,6 +14,7 @@ import (
12	14
13	15		"bnema.dev/ermite-runner/internal/execstream"
14	16		"bnema.dev/ermite-runner/internal/execstream/execpb"
	17	+	"bnema.dev/ermite-runner/internal/externals"
15	18		"bnema.dev/ermite-runner/internal/sandbox"
16	19		"bnema.dev/ermite-runner/internal/sandbox/drivers"
17	20		)

@@ -208,3 +211,97 @@ jobs:
208	211		}
209	212		t.Logf("cancelled job ended %v after the start", time.Since(start))
210	213		}
	214	+
	215	+	// nodeCache serves the pinned Node.js archives from a local directory
	216	+	// (ERMITE_RUNNER_IT_NODE_DIR holding node-v24.21.0-linux-x64.tar.gz, say): the
	217	+	// sandbox of the VM tests has no internet, and this way the real archive is
	218	+	// still checked against the pinned SHA-256 and unpacked by the real code.
	219	+	type nodeCache string
	220	+
	221	+	func (d nodeCache) RoundTrip(r http.Request) (http.Response, error) {
	222	+	f, err := os.Open(filepath.Join(string(d), filepath.Base(r.URL.Path)))
	223	+	if err != nil {
	224	+	return &http.Response{StatusCode: http.StatusNotFound, Status: "404 Not Found", Body: http.NoBody, Request: r}, nil
	225	+	}
	226	+	return &http.Response{StatusCode: http.StatusOK, Status: "200 OK", Body: f, Request: r}, nil
	227	+	}
	228	+
	229	+	// TestVMNodeExternals runs the pinned Node.js 24, provided by the host outside
	230	+	// the image, inside a job's micro-VM: the externals disk is the fourth drive,
	231	+	// mounted read-only at /home/runner/externals, in GitHub's layout.
	232	+	//
	233	+	// Needs the archive of the pinned node24 for this architecture, either in
	234	+	// ERMITE_RUNNER_IT_NODE_DIR (offline) or downloadable from nodejs.org.
	235	+	func TestVMNodeExternals(t *testing.T) {
	236	+	if os.Getenv("ERMITE_RUNNER_IT_VM") != "1" {
	237	+	t.Skip("set ERMITE_RUNNER_IT_VM=1 (needs KVM, firecracker, a kernel and a base image; see README)")
	238	+	}
	239	+	need(t, "mkfs.ext4")
	240	+	needNetAdmin(t)
	241	+	ctx := ctxWithLog(t)
	242	+	cfg := loadConfig(t, sandbox.VM)
	243	+	if why := sandbox.VMUnavailable(cfg, sandbox.Prober{}); len(why) > 0 {
	244	+	t.Fatalf("vm isolation is not available here: %s", strings.Join(why, "; "))
	245	+	}
	246	+
	247	+	m := &externals.Manager{Dir: filepath.Join(cfg.StateDir, "externals"), Arch: externals.HostArch()}
	248	+	for _, a := range externals.ForArch(externals.Pinned, m.Arch) {
	249	+	if a.Name == "node24" {
	250	+	m.Table = append(m.Table, a)
	251	+	}
	252	+	}
	253	+	if len(m.Table) != 1 {
	254	+	t.Skipf("no pinned node24 for %q", m.Arch)
	255	+	}
	256	+	if dir := os.Getenv("ERMITE_RUNNER_IT_NODE_DIR"); dir != "" {
	257	+	m.Client = &http.Client{Transport: nodeCache(dir)}
	258	+	}
	259	+	if err := m.Sync(ctx); err != nil {
	260	+	t.Skipf("cannot get the pinned Node.js (set ERMITE_RUNNER_IT_NODE_DIR to a directory with %s): %v", filepath.Base(m.Table[0].URL), err)
	261	+	}
	262	+	cfg.Externals = m
	263	+
	264	+	drv, err := sandbox.Select(ctx, cfg, drivers.Factories(), sandbox.Prober{})
	265	+	if err != nil {
	266	+	t.Fatal(err)
	267	+	}
	268	+	t.Cleanup(func() {
	269	+	_ = drv.Close()
	270	+	if d2, err := sandbox.Select(context.Background(), cfg, drivers.Factories(), sandbox.Prober{}); err == nil {
	271	+	_ = d2.Reap(context.Background())
	272	+	_ = d2.Close()
	273	+	}
	274	+	})
	275	+	if err := drv.Reap(ctx); err != nil {
	276	+	t.Fatal(err)
	277	+	}
	278	+	sb, err := drv.New()
	279	+	if err != nil {
	280	+	t.Fatal(err)
	281	+	}
	282	+	defer func() { _ = sb.Stop(context.Background()) }()
	283	+	conn, err := sb.Start(ctx, sandbox.JobSpec{JobID: "node-externals"})
	284	+	if err != nil {
	285	+	t.Fatal(err)
	286	+	}
	287	+	r, err := runJob(ctx, conn, `
	288	+	N=/home/runner/externals/node24/bin/node
	289	+	$N -e 'console.log(process.version)'
	290	+	echo "has-node24: $(test -x /home/runner/externals/node24/bin/node && echo yes)"
	291	+	touch /home/runner/externals/node24/probe 2>/dev/null && echo "WRITABLE" \|\| echo "read-only"
	292	+	grep ' /home/runner/externals ' /proc/mounts
	293	+	`)
	294	+	if err != nil {
	295	+	t.Fatal(err)
	296	+	}
	297	+	t.Log("\n" + r.out + r.errOut)
	298	+	want := m.Table[0].Version
	299	+	for _, w := range []string{"v" + want + "\n", "has-node24: yes\n", "read-only\n", "/dev/vdd /home/runner/externals ext4 ro"} {
	300	+	if !strings.Contains(r.out, w) {
	301	+	t.Errorf("output lacks %q", w)
	302	+	}
	303	+	}
	304	+	if strings.Contains(r.out, "WRITABLE") {
	305	+	t.Error("the externals are writable from the guest")
	306	+	}
	307	+	}


@@ -4,7 +4,9 @@ package integration
4		4
5	import (	5		import (
6	"context"	6		"context"
		7	+	"net/http"
7	"os"	8		"os"
		9	+	"path/filepath"
8	"strings"	10		"strings"
9	"syscall"	11		"syscall"
10	"testing"	12		"testing"

@@ -12,6 +14,7 @@ import (
12		14
13	"bnema.dev/ermite-runner/internal/execstream"	15		"bnema.dev/ermite-runner/internal/execstream"
14	"bnema.dev/ermite-runner/internal/execstream/execpb"	16		"bnema.dev/ermite-runner/internal/execstream/execpb"
		17	+	"bnema.dev/ermite-runner/internal/externals"
15	"bnema.dev/ermite-runner/internal/sandbox"	18		"bnema.dev/ermite-runner/internal/sandbox"
16	"bnema.dev/ermite-runner/internal/sandbox/drivers"	19		"bnema.dev/ermite-runner/internal/sandbox/drivers"
17	)	20		)

@@ -208,3 +211,97 @@ jobs:
208	}	211		}
209	t.Logf("cancelled job ended %v after the start", time.Since(start))	212		t.Logf("cancelled job ended %v after the start", time.Since(start))
210	}	213		}
		214	+
		215	+	// nodeCache serves the pinned Node.js archives from a local directory
		216	+	// (ERMITE_RUNNER_IT_NODE_DIR holding node-v24.21.0-linux-x64.tar.gz, say): the
		217	+	// sandbox of the VM tests has no internet, and this way the real archive is
		218	+	// still checked against the pinned SHA-256 and unpacked by the real code.
		219	+	type nodeCache string
		220	+
		221	+	func (d nodeCache) RoundTrip(r http.Request) (http.Response, error) {
		222	+	f, err := os.Open(filepath.Join(string(d), filepath.Base(r.URL.Path)))
		223	+	if err != nil {
		224	+	return &http.Response{StatusCode: http.StatusNotFound, Status: "404 Not Found", Body: http.NoBody, Request: r}, nil
		225	+	}
		226	+	return &http.Response{StatusCode: http.StatusOK, Status: "200 OK", Body: f, Request: r}, nil
		227	+	}
		228	+
		229	+	// TestVMNodeExternals runs the pinned Node.js 24, provided by the host outside
		230	+	// the image, inside a job's micro-VM: the externals disk is the fourth drive,
		231	+	// mounted read-only at /home/runner/externals, in GitHub's layout.
		232	+	//
		233	+	// Needs the archive of the pinned node24 for this architecture, either in
		234	+	// ERMITE_RUNNER_IT_NODE_DIR (offline) or downloadable from nodejs.org.
		235	+	func TestVMNodeExternals(t *testing.T) {
		236	+	if os.Getenv("ERMITE_RUNNER_IT_VM") != "1" {
		237	+	t.Skip("set ERMITE_RUNNER_IT_VM=1 (needs KVM, firecracker, a kernel and a base image; see README)")
		238	+	}
		239	+	need(t, "mkfs.ext4")
		240	+	needNetAdmin(t)
		241	+	ctx := ctxWithLog(t)
		242	+	cfg := loadConfig(t, sandbox.VM)
		243	+	if why := sandbox.VMUnavailable(cfg, sandbox.Prober{}); len(why) > 0 {
		244	+	t.Fatalf("vm isolation is not available here: %s", strings.Join(why, "; "))
		245	+	}
		246	+
		247	+	m := &externals.Manager{Dir: filepath.Join(cfg.StateDir, "externals"), Arch: externals.HostArch()}
		248	+	for _, a := range externals.ForArch(externals.Pinned, m.Arch) {
		249	+	if a.Name == "node24" {
		250	+	m.Table = append(m.Table, a)
		251	+	}
		252	+	}
		253	+	if len(m.Table) != 1 {
		254	+	t.Skipf("no pinned node24 for %q", m.Arch)
		255	+	}
		256	+	if dir := os.Getenv("ERMITE_RUNNER_IT_NODE_DIR"); dir != "" {
		257	+	m.Client = &http.Client{Transport: nodeCache(dir)}
		258	+	}
		259	+	if err := m.Sync(ctx); err != nil {
		260	+	t.Skipf("cannot get the pinned Node.js (set ERMITE_RUNNER_IT_NODE_DIR to a directory with %s): %v", filepath.Base(m.Table[0].URL), err)
		261	+	}
		262	+	cfg.Externals = m
		263	+
		264	+	drv, err := sandbox.Select(ctx, cfg, drivers.Factories(), sandbox.Prober{})
		265	+	if err != nil {
		266	+	t.Fatal(err)
		267	+	}
		268	+	t.Cleanup(func() {
		269	+	_ = drv.Close()
		270	+	if d2, err := sandbox.Select(context.Background(), cfg, drivers.Factories(), sandbox.Prober{}); err == nil {
		271	+	_ = d2.Reap(context.Background())
		272	+	_ = d2.Close()
		273	+	}
		274	+	})
		275	+	if err := drv.Reap(ctx); err != nil {
		276	+	t.Fatal(err)
		277	+	}
		278	+	sb, err := drv.New()
		279	+	if err != nil {
		280	+	t.Fatal(err)
		281	+	}
		282	+	defer func() { _ = sb.Stop(context.Background()) }()
		283	+	conn, err := sb.Start(ctx, sandbox.JobSpec{JobID: "node-externals"})
		284	+	if err != nil {
		285	+	t.Fatal(err)
		286	+	}
		287	+	r, err := runJob(ctx, conn, `
		288	+	N=/home/runner/externals/node24/bin/node
		289	+	$N -e 'console.log(process.version)'
		290	+	echo "has-node24: $(test -x /home/runner/externals/node24/bin/node && echo yes)"
		291	+	touch /home/runner/externals/node24/probe 2>/dev/null && echo "WRITABLE" \|\| echo "read-only"
		292	+	grep ' /home/runner/externals ' /proc/mounts
		293	+	`)
		294	+	if err != nil {
		295	+	t.Fatal(err)
		296	+	}
		297	+	t.Log("\n" + r.out + r.errOut)
		298	+	want := m.Table[0].Version
		299	+	for _, w := range []string{"v" + want + "\n", "has-node24: yes\n", "read-only\n", "/dev/vdd /home/runner/externals ext4 ro"} {
		300	+	if !strings.Contains(r.out, w) {
		301	+	t.Errorf("output lacks %q", w)
		302	+	}
		303	+	}
		304	+	if strings.Contains(r.out, "WRITABLE") {
		305	+	t.Error("the externals are writable from the guest")
		306	+	}
		307	+	}

M internal/sandbox/sandbox_test.go +32 -0


@@ -269,3 +269,35 @@ func TestCheckStatic(t *testing.T) {
269	269		t.Skip("/bin/sh is static here")
270	270		}
271	271		}
	272	+
	273	+	func TestNodeSettingAndExternalsKey(t *testing.T) {
	274	+	c, err := LoadConfig(env(nil))
	275	+	if err != nil \|\| !c.NodeExternals {
	276	+	t.Fatalf("on by default: %+v %v", c, err)
	277	+	}
	278	+	for v, want := range map[string]bool{"0": false, "false": false, "NO": false, "1": true, "yes": true, "": true} {
	279	+	c, err := LoadConfig(env(map[string]string{EnvNode: v}))
	280	+	if err != nil \|\| c.NodeExternals != want {
	281	+	t.Errorf("%s=%q: %v %v", EnvNode, v, c.NodeExternals, err)
	282	+	}
	283	+	}
	284	+	if _, err := LoadConfig(env(map[string]string{EnvNode: "maybe"})); err == nil \|\| !strings.Contains(err.Error(), EnvNode) {
	285	+	t.Errorf("got %v", err)
	286	+	}
	287	+
	288	+	a := External{Name: "node24", ID: "x1", Dir: "/a"}
	289	+	b := External{Name: "node20", ID: "y1", Dir: "/b"}
	290	+	k := ExternalsKey([]External{a, b})
	291	+	if k != ExternalsKey([]External{b, a}) {
	292	+	t.Error("the key depends on the order")
	293	+	}
	294	+	if k != ExternalsKey([]External{{Name: "node24", ID: "x1", Dir: "/elsewhere"}, b}) {
	295	+	t.Error("the key depends on the host path, which does not change the content")
	296	+	}
	297	+	if k == ExternalsKey([]External{{Name: "node24", ID: "x2"}, b}) \|\| k == ExternalsKey([]External{a}) {
	298	+	t.Error("a change of content keeps the key")
	299	+	}
	300	+	if (Config{}).ExternalsOf() != nil {
	301	+	t.Error("no source, no externals")
	302	+	}
	303	+	}


@@ -269,3 +269,35 @@ func TestCheckStatic(t *testing.T) {
269	t.Skip("/bin/sh is static here")	269		t.Skip("/bin/sh is static here")
270	}	270		}
271	}	271		}
		272	+
		273	+	func TestNodeSettingAndExternalsKey(t *testing.T) {
		274	+	c, err := LoadConfig(env(nil))
		275	+	if err != nil \|\| !c.NodeExternals {
		276	+	t.Fatalf("on by default: %+v %v", c, err)
		277	+	}
		278	+	for v, want := range map[string]bool{"0": false, "false": false, "NO": false, "1": true, "yes": true, "": true} {
		279	+	c, err := LoadConfig(env(map[string]string{EnvNode: v}))
		280	+	if err != nil \|\| c.NodeExternals != want {
		281	+	t.Errorf("%s=%q: %v %v", EnvNode, v, c.NodeExternals, err)
		282	+	}
		283	+	}
		284	+	if _, err := LoadConfig(env(map[string]string{EnvNode: "maybe"})); err == nil \|\| !strings.Contains(err.Error(), EnvNode) {
		285	+	t.Errorf("got %v", err)
		286	+	}
		287	+
		288	+	a := External{Name: "node24", ID: "x1", Dir: "/a"}
		289	+	b := External{Name: "node20", ID: "y1", Dir: "/b"}
		290	+	k := ExternalsKey([]External{a, b})
		291	+	if k != ExternalsKey([]External{b, a}) {
		292	+	t.Error("the key depends on the order")
		293	+	}
		294	+	if k != ExternalsKey([]External{{Name: "node24", ID: "x1", Dir: "/elsewhere"}, b}) {
		295	+	t.Error("the key depends on the host path, which does not change the content")
		296	+	}
		297	+	if k == ExternalsKey([]External{{Name: "node24", ID: "x2"}, b}) \|\| k == ExternalsKey([]External{a}) {
		298	+	t.Error("a change of content keeps the key")
		299	+	}
		300	+	if (Config{}).ExternalsOf() != nil {
		301	+	t.Error("no source, no externals")
		302	+	}
		303	+	}

M internal/sandbox/sandboxtest/sandboxtest.go +35 -0


@@ -9,11 +9,13 @@ import (
9	9		"io"
10	10		"net/netip"
11	11		"os"
	12	+	"path/filepath"
12	13		"sync"
13	14		"testing"
14	15
15	16		"bnema.dev/ermite-runner/internal/cmdrun"
16	17		"bnema.dev/ermite-runner/internal/netpolicy"
	18	+	"bnema.dev/ermite-runner/internal/sandbox"
17	19		)
18	20
19	21		// NoMountRunner runs real commands except mount and umount, which succeed

@@ -105,3 +107,36 @@ func (p *Plumbing) Live() int {
105	107		defer p.mu.Unlock()
106	108		return len(p.Attached)
107	109		}
	110	+
	111	+	// Externals is a fixed, replaceable sandbox.ExternalsSource.
	112	+	type Externals struct {
	113	+	mu sync.Mutex
	114	+	set []sandbox.External
	115	+	}
	116	+
	117	+	// Set replaces the set the drivers see.
	118	+	func (e *Externals) Set(set ...sandbox.External) { e.mu.Lock(); e.set = set; e.mu.Unlock() }
	119	+
	120	+	// Externals implements sandbox.ExternalsSource.
	121	+	func (e *Externals) Externals() []sandbox.External {
	122	+	e.mu.Lock()
	123	+	defer e.mu.Unlock()
	124	+	return append([]sandbox.External(nil), e.set...)
	125	+	}
	126	+
	127	+	// MakeExternal creates a host directory holding bin/node with content and
	128	+	// returns it as an external called name.
	129	+	func MakeExternal(t testing.TB, name, id, content string) sandbox.External {
	130	+	t.Helper()
	131	+	dir := filepath.Join(t.TempDir(), name)
	132	+	if err := os.MkdirAll(filepath.Join(dir, "bin"), 0o755); err != nil {
	133	+	t.Fatal(err)
	134	+	}
	135	+	if err := os.WriteFile(filepath.Join(dir, "bin", "node"), []byte(content), 0o755); err != nil {
	136	+	t.Fatal(err)
	137	+	}
	138	+	if err := os.Symlink("bin/node", filepath.Join(dir, "nodejs")); err != nil {
	139	+	t.Fatal(err)
	140	+	}
	141	+	return sandbox.External{Name: name, ID: id, Dir: dir}
	142	+	}


@@ -9,11 +9,13 @@ import (
9	"io"	9		"io"
10	"net/netip"	10		"net/netip"
11	"os"	11		"os"
		12	+	"path/filepath"
12	"sync"	13		"sync"
13	"testing"	14		"testing"
14		15
15	"bnema.dev/ermite-runner/internal/cmdrun"	16		"bnema.dev/ermite-runner/internal/cmdrun"
16	"bnema.dev/ermite-runner/internal/netpolicy"	17		"bnema.dev/ermite-runner/internal/netpolicy"
		18	+	"bnema.dev/ermite-runner/internal/sandbox"
17	)	19		)
18		20
19	// NoMountRunner runs real commands except mount and umount, which succeed	21		// NoMountRunner runs real commands except mount and umount, which succeed

@@ -105,3 +107,36 @@ func (p *Plumbing) Live() int {
105	defer p.mu.Unlock()	107		defer p.mu.Unlock()
106	return len(p.Attached)	108		return len(p.Attached)
107	}	109		}
		110	+
		111	+	// Externals is a fixed, replaceable sandbox.ExternalsSource.
		112	+	type Externals struct {
		113	+	mu sync.Mutex
		114	+	set []sandbox.External
		115	+	}
		116	+
		117	+	// Set replaces the set the drivers see.
		118	+	func (e *Externals) Set(set ...sandbox.External) { e.mu.Lock(); e.set = set; e.mu.Unlock() }
		119	+
		120	+	// Externals implements sandbox.ExternalsSource.
		121	+	func (e *Externals) Externals() []sandbox.External {
		122	+	e.mu.Lock()
		123	+	defer e.mu.Unlock()
		124	+	return append([]sandbox.External(nil), e.set...)
		125	+	}
		126	+
		127	+	// MakeExternal creates a host directory holding bin/node with content and
		128	+	// returns it as an external called name.
		129	+	func MakeExternal(t testing.TB, name, id, content string) sandbox.External {
		130	+	t.Helper()
		131	+	dir := filepath.Join(t.TempDir(), name)
		132	+	if err := os.MkdirAll(filepath.Join(dir, "bin"), 0o755); err != nil {
		133	+	t.Fatal(err)
		134	+	}
		135	+	if err := os.WriteFile(filepath.Join(dir, "bin", "node"), []byte(content), 0o755); err != nil {
		136	+	t.Fatal(err)
		137	+	}
		138	+	if err := os.Symlink("bin/node", filepath.Join(dir, "nodejs")); err != nil {
		139	+	t.Fatal(err)
		140	+	}
		141	+	return sandbox.External{Name: name, ID: id, Dir: dir}
		142	+	}

M internal/sandbox/vm/bootargs.go +7 -3


@@ -19,8 +19,9 @@ import (
19	19		// - root=/dev/vda ro: the exec disk; init=/ermite-runner -- init: the runner
20	20		// executable is PID 1 and runs its init subcommand;
21	21		// - ip=…: static address, gateway and DNS from the kernel, no DHCP client;
22		-	// - ermite.*: read by init from /proc/cmdline (connect target, gateway, time).
23		-	func bootArgs(guest, gateway netip.Addr, unixSeconds int64) string {
	22	+	// - ermite.*: read by init from /proc/cmdline (connect target, gateway, time,
	23	+	// and whether the externals disk is attached as the fourth drive).
	24	+	func bootArgs(guest, gateway netip.Addr, unixSeconds int64, externals bool) string {
24	25		args := []string{
25	26		"console=ttyS0", "reboot=k", "panic=1", "pci=off", "acpi=off", "8250.nr_uarts=1",
26	27		"ipv6.disable=1", "random.trust_cpu=on", "i8042.noaux", "i8042.nomux", "i8042.nopnp", "i8042.dumbkbd",

@@ -31,8 +32,11 @@ func bootArgs(guest, gateway netip.Addr, unixSeconds int64) string {
31	32		sandbox.CmdlineConnect + "=vsock:" + strconv.Itoa(sandbox.VsockPort),
32	33		sandbox.CmdlineGateway + "=" + gateway.String(),
33	34		"ermite.time=" + strconv.FormatInt(unixSeconds, 10),
34		-	"--", "init",
35	35		}
	36	+	if externals {
	37	+	args = append(args, sandbox.CmdlineExternals+"=1")
	38	+	}
	39	+	args = append(args, "--", "init")
36	40		return strings.Join(args, " ")
37	41		}
38	42

 //   - root=/dev/vda ro: the exec disk; init=/ermite-runner -- init: the runner
 //     executable is PID 1 and runs its init subcommand;
 //   - ip=…: static address, gateway and DNS from the kernel, no DHCP client;
-//   - ermite.*: read by init from /proc/cmdline (connect target, gateway, time).
+//   - ermite.*: read by init from /proc/cmdline (connect target, gateway, time,
-func bootArgs(guest, gateway netip.Addr, unixSeconds int64) string {
+//     and whether the externals disk is attached as the fourth drive).
+func bootArgs(guest, gateway netip.Addr, unixSeconds int64, externals bool) string {
 	args := []string{
 		"console=ttyS0", "reboot=k", "panic=1", "pci=off", "acpi=off", "8250.nr_uarts=1",
 		"ipv6.disable=1", "random.trust_cpu=on", "i8042.noaux", "i8042.nomux", "i8042.nopnp", "i8042.dumbkbd",
 		sandbox.CmdlineConnect + "=vsock:" + strconv.Itoa(sandbox.VsockPort),
 		sandbox.CmdlineGateway + "=" + gateway.String(),
 		"ermite.time=" + strconv.FormatInt(unixSeconds, 10),
-		"--", "init",
 	}
+	if externals {
+		args = append(args, sandbox.CmdlineExternals+"=1")
+	}
+	args = append(args, "--", "init")
 	return strings.Join(args, " ")
 }

M internal/sandbox/vm/driver.go +42 -10


@@ -68,9 +68,12 @@ type Driver struct {
68	68		fcPath string
69	69		jailer string
70	70		execDisk string
71		-	store store
72		-	net netsetup.Plumbing
73		-	lock *sandbox.HostLock
	71	+	// extMu serialises the building of the externals disk with the linking of it
	72	+	// into a sandbox (see shareExternals).
	73	+	extMu sync.Mutex
	74	+	store store
	75	+	net netsetup.Plumbing
	76	+	lock *sandbox.HostLock
74	77
75	78		mu sync.Mutex
76	79		slots *sandbox.Slots

@@ -370,7 +373,7 @@ func (s vmSandbox) bootVMM(ctx context.Context, spec sandbox.JobSpec) (apiClie
370	373		}
371	374
372	375		// vmFiles are the paths Firecracker is given.
373		-	type vmFiles struct{ execDisk, baseDisk, kernel string }
	376	+	type vmFiles struct{ execDisk, baseDisk, kernel, externals string }
374	377
375	378		// bootFiles makes the shared files available and creates the scratch disk.
376	379		func (s *vmSandbox) bootFiles(ctx context.Context, spec sandbox.JobSpec) (vmFiles, error) {

@@ -386,6 +389,7 @@ func (s *vmSandbox) bootFiles(ctx context.Context, spec sandbox.JobSpec) (vmFile
386	389		if f.kernel, err = s.lay.share(d.cfg.KernelPath(), "kernel"); err != nil {
387	390		return f, err
388	391		}
	392	+	f.externals = s.shareExternals(ctx)
389	393		owner := -1
390	394		if s.lay.jailed {
391	395		owner = s.lay.uid

@@ -393,6 +397,29 @@ func (s *vmSandbox) bootFiles(ctx context.Context, spec sandbox.JobSpec) (vmFile
393	397		return f, disk.CreateScratch(ctx, d.opt.Run, s.lay.host("scratch.ext4"), spec.DiskMiB, owner)
394	398		}
395	399
	400	+	// shareExternals makes the read-only disk of the host's external tools available
	401	+	// to Firecracker; "" means the sandbox has none. A disk that cannot be built
	402	+	// does not stop the job: it starts without the tools and the failure is logged.
	403	+	func (s *vmSandbox) shareExternals(ctx context.Context) string {
	404	+	d := s.d
	405	+	set := d.cfg.ExternalsOf()
	406	+	if len(set) == 0 {
	407	+	return ""
	408	+	}
	409	+	log := zerowrap.FromCtx(ctx)
	410	+	d.extMu.Lock()
	411	+	defer d.extMu.Unlock()
	412	+	path, err := ensureExternalsDisk(ctx, d.opt.Run, set, d.cfg.StateDir)
	413	+	if err == nil && path != "" {
	414	+	path, err = s.lay.share(path, "externals.ext4")
	415	+	}
	416	+	if err != nil {
	417	+	log.Error().Err(err).Str("vm", s.id).Msg("external tools (Node.js) not attached to this job")
	418	+	return ""
	419	+	}
	420	+	return path
	421	+	}
	422	+
396	423		// bootVsock listens on the host side of the vsock stream: Firecracker connects
397	424		// to <uds_path>_<port> when the guest connects to the host CID.
398	425		func (s *vmSandbox) bootVsock() (<-chan acceptResult, error) {

@@ -476,14 +503,19 @@ func checkJailBaseInfo(dir string, fi fs.FileInfo, euid int) error {
476	503
477	504		func (s *vmSandbox) vmConfig(spec sandbox.JobSpec, netw netpolicy.Net, f vmFiles) vmConfig {
478	505		cfg := s.d.cfg
	506	+	// The order decides the guest device names (vda, vdb, ...): new drives go last.
	507	+	drives := []drive{
	508	+	{DriveID: "rootfs", PathOnHost: f.execDisk, IsRootDevice: true, IsReadOnly: true},
	509	+	{DriveID: "base", PathOnHost: f.baseDisk, IsReadOnly: true},
	510	+	{DriveID: "scratch", PathOnHost: s.lay.fc("scratch.ext4"), RateLimiter: diskLimiter(cfg.DiskBandwidthMiB, cfg.DiskIOPS)},
	511	+	}
	512	+	if f.externals != "" {
	513	+	drives = append(drives, drive{DriveID: "externals", PathOnHost: f.externals, IsReadOnly: true})
	514	+	}
479	515		return vmConfig{
480	516		Machine: machineConfig{VcpuCount: spec.CPUs, MemSizeMiB: spec.MemoryMiB},
481		-	Boot: bootSource{KernelImagePath: f.kernel, BootArgs: bootArgs(netw.Guest, netw.Gateway, time.Now().Unix())},
482		-	Drives: []drive{
483		-	{DriveID: "rootfs", PathOnHost: f.execDisk, IsRootDevice: true, IsReadOnly: true},
484		-	{DriveID: "base", PathOnHost: f.baseDisk, IsReadOnly: true},
485		-	{DriveID: "scratch", PathOnHost: s.lay.fc("scratch.ext4"), RateLimiter: diskLimiter(cfg.DiskBandwidthMiB, cfg.DiskIOPS)},
486		-	},
	517	+	Boot: bootSource{KernelImagePath: f.kernel, BootArgs: bootArgs(netw.Guest, netw.Gateway, time.Now().Unix(), f.externals != "")},
	518	+	Drives: drives,
487	519		Net: networkInterface{
488	520		IfaceID: "eth0", HostDevName: s.tap, GuestMAC: macFor(s.slot),
489	521		RxRateLimiter: netLimiter(cfg.NetMbit), TxRateLimiter: netLimiter(cfg.NetMbit),

 	fcPath   string
 	jailer   string
 	execDisk string
-	store    store
+	// extMu serialises the building of the externals disk with the linking of it
-	net      netsetup.Plumbing
+	// into a sandbox (see shareExternals).
-	lock     *sandbox.HostLock
+	extMu sync.Mutex
+	store store
+	net   netsetup.Plumbing
+	lock  *sandbox.HostLock
 	mu    sync.Mutex
 	slots *sandbox.Slots
 }
 // vmFiles are the paths Firecracker is given.
-type vmFiles struct{ execDisk, baseDisk, kernel string }
+type vmFiles struct{ execDisk, baseDisk, kernel, externals string }
 // bootFiles makes the shared files available and creates the scratch disk.
 func (s *vmSandbox) bootFiles(ctx context.Context, spec sandbox.JobSpec) (vmFiles, error) {
 	if f.kernel, err = s.lay.share(d.cfg.KernelPath(), "kernel"); err != nil {
 		return f, err
 	}
+	f.externals = s.shareExternals(ctx)
 	owner := -1
 	if s.lay.jailed {
 		owner = s.lay.uid
 	return f, disk.CreateScratch(ctx, d.opt.Run, s.lay.host("scratch.ext4"), spec.DiskMiB, owner)
 }
+// shareExternals makes the read-only disk of the host's external tools available
+// to Firecracker; "" means the sandbox has none. A disk that cannot be built
+// does not stop the job: it starts without the tools and the failure is logged.
+func (s *vmSandbox) shareExternals(ctx context.Context) string {
+	d := s.d
+	set := d.cfg.ExternalsOf()
+	if len(set) == 0 {
+		return ""
+	}
+	log := zerowrap.FromCtx(ctx)
+	d.extMu.Lock()
+	defer d.extMu.Unlock()
+	path, err := ensureExternalsDisk(ctx, d.opt.Run, set, d.cfg.StateDir)
+	if err == nil && path != "" {
+		path, err = s.lay.share(path, "externals.ext4")
+	}
+	if err != nil {
+		log.Error().Err(err).Str("vm", s.id).Msg("external tools (Node.js) not attached to this job")
+		return ""
+	}
+	return path
+}
 // bootVsock listens on the host side of the vsock stream: Firecracker connects
 // to <uds_path>_<port> when the guest connects to the host CID.
 func (s *vmSandbox) bootVsock() (<-chan acceptResult, error) {
 func (s *vmSandbox) vmConfig(spec sandbox.JobSpec, netw netpolicy.Net, f vmFiles) vmConfig {
 	cfg := s.d.cfg
+	// The order decides the guest device names (vda, vdb, ...): new drives go last.
+	drives := []drive{
+		{DriveID: "rootfs", PathOnHost: f.execDisk, IsRootDevice: true, IsReadOnly: true},
+		{DriveID: "base", PathOnHost: f.baseDisk, IsReadOnly: true},
+		{DriveID: "scratch", PathOnHost: s.lay.fc("scratch.ext4"), RateLimiter: diskLimiter(cfg.DiskBandwidthMiB, cfg.DiskIOPS)},
+	}
+	if f.externals != "" {
+		drives = append(drives, drive{DriveID: "externals", PathOnHost: f.externals, IsReadOnly: true})
+	}
 	return vmConfig{
 		Machine: machineConfig{VcpuCount: spec.CPUs, MemSizeMiB: spec.MemoryMiB},
-		Boot:    bootSource{KernelImagePath: f.kernel, BootArgs: bootArgs(netw.Guest, netw.Gateway, time.Now().Unix())},
+		Boot:    bootSource{KernelImagePath: f.kernel, BootArgs: bootArgs(netw.Guest, netw.Gateway, time.Now().Unix(), f.externals != "")},
-		Drives: []drive{
+		Drives:  drives,
-			{DriveID: "rootfs", PathOnHost: f.execDisk, IsRootDevice: true, IsReadOnly: true},
-			{DriveID: "base", PathOnHost: f.baseDisk, IsReadOnly: true},
-			{DriveID: "scratch", PathOnHost: s.lay.fc("scratch.ext4"), RateLimiter: diskLimiter(cfg.DiskBandwidthMiB, cfg.DiskIOPS)},
-		},
 		Net: networkInterface{
 			IfaceID: "eth0", HostDevName: s.tap, GuestMAC: macFor(s.slot),
 			RxRateLimiter: netLimiter(cfg.NetMbit), TxRateLimiter: netLimiter(cfg.NetMbit),

M internal/sandbox/vm/images.go +120 -0


@@ -8,8 +8,11 @@ import (
8	8		"encoding/hex"
9	9		"fmt"
10	10		"io"
	11	+	"io/fs"
11	12		"os"
12	13		"path/filepath"
	14	+	"strconv"
	15	+	"time"
13	16
14	17		"bnema.dev/ermite-runner/internal/cmdrun"
15	18		"bnema.dev/ermite-runner/internal/sandbox"

@@ -95,3 +98,120 @@ func hashFile(path string) (sum string, size int64, err error) {
95	98		}
96	99		return hex.EncodeToString(h.Sum(nil)), n, nil
97	100		}
	101	+
	102	+	// ensureExternalsDisk builds (or reuses) the read-only ext4 disk that carries
	103	+	// the host's external tools (Node.js): one directory per tool at its root, so
	104	+	// that mounted at /home/runner/externals it gives the GitHub layout
	105	+	// externals/node24/bin/node. It is named after the content of the set
	106	+	// (sandbox.ExternalsKey), so a new Node.js version builds a new disk and leaves
	107	+	// the exec disk and the base image alone. It returns "" for an empty set. The
	108	+	// caller holds the driver's externals lock, so that two jobs do not build the
	109	+	// same disk.
	110	+	func ensureExternalsDisk(ctx context.Context, run cmdrun.Runner, set []sandbox.External, stateDir string) (string, error) {
	111	+	if len(set) == 0 {
	112	+	return "", nil
	113	+	}
	114	+	cache := filepath.Join(stateDir, "cache")
	115	+	if err := os.MkdirAll(cache, 0o755); err != nil {
	116	+	return "", err
	117	+	}
	118	+	name := "extdisk-" + sandbox.ExternalsKey(set) + ".ext4"
	119	+	dest := filepath.Join(cache, name)
	120	+	if _, err := os.Stat(dest); err == nil {
	121	+	return dest, nil
	122	+	}
	123	+	// Drop disks of other contents, once they are an hour old: without the
	124	+	// jailer a VM that is about to start may still be about to open the previous
	125	+	// one in place (with the jailer it holds a hard link).
	126	+	olds, _ := filepath.Glob(filepath.Join(cache, "extdisk-*"))
	127	+	for _, o := range olds {
	128	+	if fi, err := os.Stat(o); err == nil && time.Since(fi.ModTime()) > time.Hour {
	129	+	_ = os.Remove(o)
	130	+	}
	131	+	}
	132	+
	133	+	tmp, err := os.MkdirTemp(cache, "extroot-")
	134	+	if err != nil {
	135	+	return "", err
	136	+	}
	137	+	defer func() { _ = os.RemoveAll(tmp) }()
	138	+	var size, files int64
	139	+	for _, e := range set {
	140	+	sz, n, err := cloneTree(e.Dir, filepath.Join(tmp, e.Name))
	141	+	if err != nil {
	142	+	return "", fmt.Errorf("stage %s: %w", e.Name, err)
	143	+	}
	144	+	size, files = size+sz, files+n
	145	+	}
	146	+
	147	+	part := dest + ".part"
	148	+	_ = os.Remove(part)
	149	+	f, err := os.OpenFile(part, os.O_WRONLY\|os.O_CREATE\|os.O_EXCL, 0o644)
	150	+	if err != nil {
	151	+	return "", err
	152	+	}
	153	+	// The files, ext4 overhead (inodes, directories, block rounding of many
	154	+	// small files) and some room; the disk is read-only and sparse.
	155	+	mib := size/(1<<20)125/100 + files4096/(1<<20) + 16
	156	+	if err := f.Truncate(mib << 20); err != nil {
	157	+	_ = f.Close()
	158	+	return "", err
	159	+	}
	160	+	_ = f.Close()
	161	+	if _, err := run.Run(ctx, nil, "mkfs.ext4", "-q", "-F", "-L", "ermite-ext", "-m", "0",
	162	+	"-O", "^has_journal", "-N", strconv.FormatInt(files*13/10+256, 10),
	163	+	"-E", "root_owner=0:0", "-d", tmp, part); err != nil {
	164	+	_ = os.Remove(part)
	165	+	return "", fmt.Errorf("build externals disk: %w", err)
	166	+	}
	167	+	if err := os.Rename(part, dest); err != nil {
	168	+	return "", err
	169	+	}
	170	+	return dest, nil
	171	+	}
	172	+
	173	+	// cloneTree recreates the tree src at dst (directories, regular files, symbolic
	174	+	// links) for mkfs.ext4 -d, hard-linking files when the filesystem allows and
	175	+	// copying them otherwise. It returns the bytes of file content and the number
	176	+	// of entries.
	177	+	func cloneTree(src, dst string) (size, entries int64, err error) {
	178	+	err = filepath.WalkDir(src, func(p string, d fs.DirEntry, werr error) error {
	179	+	if werr != nil {
	180	+	return werr
	181	+	}
	182	+	rel, err := filepath.Rel(src, p)
	183	+	if err != nil {
	184	+	return err
	185	+	}
	186	+	to := filepath.Join(dst, rel)
	187	+	entries++
	188	+	switch {
	189	+	case d.IsDir():
	190	+	if err := os.MkdirAll(to, 0o755); err != nil {
	191	+	return err
	192	+	}
	193	+	return os.Chmod(to, 0o755) // whatever the umask
	194	+	case d.Type()&fs.ModeSymlink != 0:
	195	+	target, err := os.Readlink(p)
	196	+	if err != nil {
	197	+	return err
	198	+	}
	199	+	return os.Symlink(target, to)
	200	+	case d.Type().IsRegular():
	201	+	fi, err := d.Info()
	202	+	if err != nil {
	203	+	return err
	204	+	}
	205	+	size += fi.Size()
	206	+	if err := os.Link(p, to); err == nil {
	207	+	return nil
	208	+	}
	209	+	if err := copyFile(p, to); err != nil {
	210	+	return err
	211	+	}
	212	+	return os.Chmod(to, fi.Mode().Perm())
	213	+	}
	214	+	return fmt.Errorf("%s: unsupported file type %v", p, d.Type())
	215	+	})
	216	+	return size, entries, err
	217	+	}


@@ -8,8 +8,11 @@ import (
8	"encoding/hex"	8		"encoding/hex"
9	"fmt"	9		"fmt"
10	"io"	10		"io"
		11	+	"io/fs"
11	"os"	12		"os"
12	"path/filepath"	13		"path/filepath"
		14	+	"strconv"
		15	+	"time"
13		16
14	"bnema.dev/ermite-runner/internal/cmdrun"	17		"bnema.dev/ermite-runner/internal/cmdrun"
15	"bnema.dev/ermite-runner/internal/sandbox"	18		"bnema.dev/ermite-runner/internal/sandbox"

@@ -95,3 +98,120 @@ func hashFile(path string) (sum string, size int64, err error) {
95	}	98		}
96	return hex.EncodeToString(h.Sum(nil)), n, nil	99		return hex.EncodeToString(h.Sum(nil)), n, nil
97	}	100		}
		101	+
		102	+	// ensureExternalsDisk builds (or reuses) the read-only ext4 disk that carries
		103	+	// the host's external tools (Node.js): one directory per tool at its root, so
		104	+	// that mounted at /home/runner/externals it gives the GitHub layout
		105	+	// externals/node24/bin/node. It is named after the content of the set
		106	+	// (sandbox.ExternalsKey), so a new Node.js version builds a new disk and leaves
		107	+	// the exec disk and the base image alone. It returns "" for an empty set. The
		108	+	// caller holds the driver's externals lock, so that two jobs do not build the
		109	+	// same disk.
		110	+	func ensureExternalsDisk(ctx context.Context, run cmdrun.Runner, set []sandbox.External, stateDir string) (string, error) {
		111	+	if len(set) == 0 {
		112	+	return "", nil
		113	+	}
		114	+	cache := filepath.Join(stateDir, "cache")
		115	+	if err := os.MkdirAll(cache, 0o755); err != nil {
		116	+	return "", err
		117	+	}
		118	+	name := "extdisk-" + sandbox.ExternalsKey(set) + ".ext4"
		119	+	dest := filepath.Join(cache, name)
		120	+	if _, err := os.Stat(dest); err == nil {
		121	+	return dest, nil
		122	+	}
		123	+	// Drop disks of other contents, once they are an hour old: without the
		124	+	// jailer a VM that is about to start may still be about to open the previous
		125	+	// one in place (with the jailer it holds a hard link).
		126	+	olds, _ := filepath.Glob(filepath.Join(cache, "extdisk-*"))
		127	+	for _, o := range olds {
		128	+	if fi, err := os.Stat(o); err == nil && time.Since(fi.ModTime()) > time.Hour {
		129	+	_ = os.Remove(o)
		130	+	}
		131	+	}
		132	+
		133	+	tmp, err := os.MkdirTemp(cache, "extroot-")
		134	+	if err != nil {
		135	+	return "", err
		136	+	}
		137	+	defer func() { _ = os.RemoveAll(tmp) }()
		138	+	var size, files int64
		139	+	for _, e := range set {
		140	+	sz, n, err := cloneTree(e.Dir, filepath.Join(tmp, e.Name))
		141	+	if err != nil {
		142	+	return "", fmt.Errorf("stage %s: %w", e.Name, err)
		143	+	}
		144	+	size, files = size+sz, files+n
		145	+	}
		146	+
		147	+	part := dest + ".part"
		148	+	_ = os.Remove(part)
		149	+	f, err := os.OpenFile(part, os.O_WRONLY\|os.O_CREATE\|os.O_EXCL, 0o644)
		150	+	if err != nil {
		151	+	return "", err
		152	+	}
		153	+	// The files, ext4 overhead (inodes, directories, block rounding of many
		154	+	// small files) and some room; the disk is read-only and sparse.
		155	+	mib := size/(1<<20)125/100 + files4096/(1<<20) + 16
		156	+	if err := f.Truncate(mib << 20); err != nil {
		157	+	_ = f.Close()
		158	+	return "", err
		159	+	}
		160	+	_ = f.Close()
		161	+	if _, err := run.Run(ctx, nil, "mkfs.ext4", "-q", "-F", "-L", "ermite-ext", "-m", "0",
		162	+	"-O", "^has_journal", "-N", strconv.FormatInt(files*13/10+256, 10),
		163	+	"-E", "root_owner=0:0", "-d", tmp, part); err != nil {
		164	+	_ = os.Remove(part)
		165	+	return "", fmt.Errorf("build externals disk: %w", err)
		166	+	}
		167	+	if err := os.Rename(part, dest); err != nil {
		168	+	return "", err
		169	+	}
		170	+	return dest, nil
		171	+	}
		172	+
		173	+	// cloneTree recreates the tree src at dst (directories, regular files, symbolic
		174	+	// links) for mkfs.ext4 -d, hard-linking files when the filesystem allows and
		175	+	// copying them otherwise. It returns the bytes of file content and the number
		176	+	// of entries.
		177	+	func cloneTree(src, dst string) (size, entries int64, err error) {
		178	+	err = filepath.WalkDir(src, func(p string, d fs.DirEntry, werr error) error {
		179	+	if werr != nil {
		180	+	return werr
		181	+	}
		182	+	rel, err := filepath.Rel(src, p)
		183	+	if err != nil {
		184	+	return err
		185	+	}
		186	+	to := filepath.Join(dst, rel)
		187	+	entries++
		188	+	switch {
		189	+	case d.IsDir():
		190	+	if err := os.MkdirAll(to, 0o755); err != nil {
		191	+	return err
		192	+	}
		193	+	return os.Chmod(to, 0o755) // whatever the umask
		194	+	case d.Type()&fs.ModeSymlink != 0:
		195	+	target, err := os.Readlink(p)
		196	+	if err != nil {
		197	+	return err
		198	+	}
		199	+	return os.Symlink(target, to)
		200	+	case d.Type().IsRegular():
		201	+	fi, err := d.Info()
		202	+	if err != nil {
		203	+	return err
		204	+	}
		205	+	size += fi.Size()
		206	+	if err := os.Link(p, to); err == nil {
		207	+	return nil
		208	+	}
		209	+	if err := copyFile(p, to); err != nil {
		210	+	return err
		211	+	}
		212	+	return os.Chmod(to, fi.Mode().Perm())
		213	+	}
		214	+	return fmt.Errorf("%s: unsupported file type %v", p, d.Type())
		215	+	})
		216	+	return size, entries, err
		217	+	}

M internal/sandbox/vm/vm_test.go +182 -1


@@ -621,7 +621,7 @@ func TestJailedShareHardLinks(t *testing.T) {
621	621		}
622	622
623	623		func TestBootArgs(t *testing.T) {
624		-	a := bootArgs(mustAddr("10.200.0.6"), mustAddr("10.200.0.5"), 1791818522)
	624	+	a := bootArgs(mustAddr("10.200.0.6"), mustAddr("10.200.0.5"), 1791818522, false)
625	625		for _, want := range []string{
626	626		"ipv6.disable=1", "root=/dev/vda ro", "init=/ermite-runner", "reboot=k", "panic=1",
627	627		"ip=10.200.0.6::10.200.0.5:255.255.255.252:ermite-job:eth0:off:10.200.0.5",

@@ -634,6 +634,12 @@ func TestBootArgs(t *testing.T) {
634	634		if !strings.HasSuffix(a, "-- init") {
635	635		t.Errorf("init subcommand missing: %s", a)
636	636		}
	637	+	if strings.Contains(a, "ermite.externals") {
	638	+	t.Errorf("externals announced without a disk: %s", a)
	639	+	}
	640	+	if b := bootArgs(mustAddr("10.200.0.6"), mustAddr("10.200.0.5"), 1791818522, true); !strings.Contains(b, " ermite.externals=1 -- init") {
	641	+	t.Errorf("externals not announced: %s", b)
	642	+	}
637	643		if macFor(0) != "06:00:ac:10:00:00" \|\| macFor(257) != "06:00:ac:10:01:01" {
638	644		t.Fatal(macFor(257))
639	645		}

@@ -1348,3 +1354,178 @@ func TestGuestForgeIsAttachedAndWarnedAtEveryJob(t *testing.T) {
1348	1354		t.Fatalf("the warning must be logged at every job start (1 + 2), got %d", n)
1349	1355		}
1350	1356		}
	1357	+
	1358	+	// The host's external tools ride on a fourth, read-only drive, keyed by their
	1359	+	// content: a new Node.js builds a new disk and leaves the exec disk alone.
	1360	+	func TestExternalsDiskIsAttachedReadOnlyAndKeyedByContent(t *testing.T) {
	1361	+	e := newEnv(t, "")
	1362	+	src := &sandboxtest.Externals{}
	1363	+	n24 := sandboxtest.MakeExternal(t, "node24", "node24-24.21.0-x64-aaaa", "v24.21.0")
	1364	+	n20 := sandboxtest.MakeExternal(t, "node20", "node20-20.20.2-x64-bbbb", "v20.20.2")
	1365	+	src.Set(n24, n20)
	1366	+	e.cfg.Externals = src
	1367	+	d := e.driver(t)
	1368	+	execDisk := d.execDisk
	1369	+
	1370	+	boot := func() (drives []string, log string) {
	1371	+	t.Helper()
	1372	+	_ = os.Remove(e.log)
	1373	+	sb, _ := d.New()
	1374	+	conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
	1375	+	if err != nil {
	1376	+	t.Fatal(err)
	1377	+	}
	1378	+	runStub(t, conn)
	1379	+	defer func() { _ = sb.Stop(context.Background()) }()
	1380	+	b, _ := os.ReadFile(e.log)
	1381	+	log = string(b)
	1382	+	for _, l := range strings.Split(log, "\n") {
	1383	+	if f := strings.Fields(l); len(f) >= 2 && f[0] == "PUT" && strings.HasPrefix(f[1], "/drives/") {
	1384	+	drives = append(drives, f[1])
	1385	+	}
	1386	+	}
	1387	+	return drives, log
	1388	+	}
	1389	+	drives, log := boot()
	1390	+	if strings.Join(drives, ",") != "/drives/rootfs,/drives/base,/drives/scratch,/drives/externals" {
	1391	+	t.Fatalf("drives: %v", drives)
	1392	+	}
	1393	+	if !strings.Contains(log, `"drive_id":"externals"`) \|\| !strings.Contains(log, `"is_root_device":false,"is_read_only":true`) \|\| !strings.Contains(log, "ermite.externals=1") {
	1394	+	t.Fatalf("API log:\n%s", log)
	1395	+	}
	1396	+	// The scratch drive is still the third (vdc), and the only writable one.
	1397	+	if strings.Count(log, `"is_read_only":false`) != 1 {
	1398	+	t.Errorf("only the scratch drive is writable:\n%s", log)
	1399	+	}
	1400	+	first, _ := filepath.Glob(filepath.Join(e.cfg.StateDir, "cache", "extdisk-*"))
	1401	+	if len(first) != 1 {
	1402	+	t.Fatalf("externals disks: %v", first)
	1403	+	}
	1404	+	mkfs := func() (n int) {
	1405	+	for _, c := range e.run.Calls {
	1406	+	if c.Name == "mkfs.ext4" && len(c.Args) > 1 {
	1407	+	for _, a := range c.Args {
	1408	+	if a == "ermite-ext" {
	1409	+	n++
	1410	+	}
	1411	+	}
	1412	+	}
	1413	+	}
	1414	+	return
	1415	+	}
	1416	+	if mkfs() != 1 {
	1417	+	t.Fatalf("externals disk built %d times", mkfs())
	1418	+	}
	1419	+
	1420	+	// A second job with the same tools reuses the disk.
	1421	+	boot()
	1422	+	if mkfs() != 1 {
	1423	+	t.Fatal("externals disk rebuilt for the same content")
	1424	+	}
	1425	+
	1426	+	// A new Node.js: another disk, and the exec disk is untouched.
	1427	+	src.Set(sandboxtest.MakeExternal(t, "node24", "node24-24.22.0-x64-cccc", "v24.22.0"), n20)
	1428	+	boot()
	1429	+	if mkfs() != 2 {
	1430	+	t.Fatal("externals disk not rebuilt for a new version")
	1431	+	}
	1432	+	if d.execDisk != execDisk {
	1433	+	t.Fatal("exec disk changed")
	1434	+	}
	1435	+	if _, err := os.Stat(execDisk); err != nil {
	1436	+	t.Fatal("exec disk removed")
	1437	+	}
	1438	+	if now, _ := filepath.Glob(filepath.Join(e.cfg.StateDir, "cache", "extdisk-*")); len(now) != 2 \|\| (now[0] != first[0] && now[1] != first[0]) {
	1439	+	// The previous disk may still be linked by a starting VM: kept for an hour.
	1440	+	t.Fatalf("externals disks: %v", now)
	1441	+	}
	1442	+	}
	1443	+
	1444	+	func TestNoExternalsMeansThreeDrives(t *testing.T) {
	1445	+	for _, src := range []sandbox.ExternalsSource{nil, &sandboxtest.Externals{}} {
	1446	+	e := newEnv(t, "")
	1447	+	e.cfg.Externals = src
	1448	+	d := e.driver(t)
	1449	+	sb, _ := d.New()
	1450	+	conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
	1451	+	if err != nil {
	1452	+	t.Fatal(err)
	1453	+	}
	1454	+	runStub(t, conn)
	1455	+	b, _ := os.ReadFile(e.log)
	1456	+	if strings.Contains(string(b), "externals") {
	1457	+	t.Errorf("externals attached without any:\n%s", b)
	1458	+	}
	1459	+	_ = sb.Stop(context.Background())
	1460	+	if left, _ := filepath.Glob(filepath.Join(e.cfg.StateDir, "cache", "extdisk-*")); len(left) != 0 {
	1461	+	t.Errorf("disk built for nothing: %v", left)
	1462	+	}
	1463	+	}
	1464	+	}
	1465	+
	1466	+	func TestExternalsDiskFailureDoesNotStopTheJob(t *testing.T) {
	1467	+	e := newEnv(t, "")
	1468	+	src := &sandboxtest.Externals{}
	1469	+	src.Set(sandbox.External{Name: "node24", ID: "x", Dir: filepath.Join(t.TempDir(), "missing")})
	1470	+	e.cfg.Externals = src
	1471	+	var buf bytes.Buffer
	1472	+	ctx := zerowrap.WithCtx(context.Background(), zerowrap.New(zerowrap.Config{Level: "debug", Format: "json", Output: &buf}))
	1473	+	d := e.driver(t)
	1474	+	sb, _ := d.New()
	1475	+	conn, err := sb.Start(ctx, sandbox.JobSpec{JobID: "j"})
	1476	+	if err != nil {
	1477	+	t.Fatalf("a broken externals set must not stop the job: %v", err)
	1478	+	}
	1479	+	runStub(t, conn)
	1480	+	_ = sb.Stop(ctx)
	1481	+	if !strings.Contains(buf.String(), "external tools (Node.js) not attached") {
	1482	+	t.Fatalf("failure not logged: %s", buf.String())
	1483	+	}
	1484	+	b, _ := os.ReadFile(e.log)
	1485	+	if strings.Contains(string(b), `"externals"`) {
	1486	+	t.Fatal("drive attached after a failure")
	1487	+	}
	1488	+	}
	1489	+
	1490	+	// With the real mkfs.ext4 the disk holds <name>/bin/node at its root, which the
	1491	+	// guest mounts at /home/runner/externals.
	1492	+	func TestExternalsDiskContent(t *testing.T) {
	1493	+	if _, err := exec.LookPath("mkfs.ext4"); err != nil {
	1494	+	t.Skip("mkfs.ext4 needed")
	1495	+	}
	1496	+	dbg, err := exec.LookPath("debugfs")
	1497	+	if err != nil {
	1498	+	t.Skip("debugfs needed")
	1499	+	}
	1500	+	set := []sandbox.External{
	1501	+	sandboxtest.MakeExternal(t, "node24", "a", "v24"),
	1502	+	sandboxtest.MakeExternal(t, "node20", "b", "v20"),
	1503	+	}
	1504	+	path, err := ensureExternalsDisk(context.Background(), cmdrun.Exec{}, set, t.TempDir())
	1505	+	if err != nil {
	1506	+	t.Fatal(err)
	1507	+	}
	1508	+	out := func(cmd string) string {
	1509	+	b, err := exec.Command(dbg, "-R", cmd, path).CombinedOutput()
	1510	+	if err != nil {
	1511	+	t.Fatalf("debugfs %s: %v\n%s", cmd, err, b)
	1512	+	}
	1513	+	_, body, _ := strings.Cut(string(b), "\n") // the banner line
	1514	+	return body
	1515	+	}
	1516	+	if ls := out("ls -l /"); !strings.Contains(ls, "node24") \|\| !strings.Contains(ls, "node20") {
	1517	+	t.Fatalf("root:\n%s", ls)
	1518	+	}
	1519	+	if got := out("cat /node24/bin/node"); got != "v24" {
	1520	+	t.Fatalf("node24/bin/node: %q", got)
	1521	+	}
	1522	+	if ls := out("stat /node20/bin/node"); !strings.Contains(ls, "Mode: 0755") {
	1523	+	t.Fatalf("mode:\n%s", ls)
	1524	+	}
	1525	+	if ls := out("stat /node20/nodejs"); !strings.Contains(ls, "Type: symlink") \|\| !strings.Contains(ls, `"bin/node"`) {
	1526	+	t.Fatalf("symlink:\n%s", ls)
	1527	+	}
	1528	+	if st, _ := os.Stat(path); st.Size() > 64<<20 {
	1529	+	t.Fatalf("a tiny tree gave a %d byte disk", st.Size())
	1530	+	}
	1531	+	}


@@ -621,7 +621,7 @@ func TestJailedShareHardLinks(t *testing.T) {
621		}	621		}
622			622
623		func TestBootArgs(t *testing.T) {	623		func TestBootArgs(t *testing.T) {
624	-	a := bootArgs(mustAddr("10.200.0.6"), mustAddr("10.200.0.5"), 1791818522)	624	+	a := bootArgs(mustAddr("10.200.0.6"), mustAddr("10.200.0.5"), 1791818522, false)
625		for _, want := range []string{	625		for _, want := range []string{
626		"ipv6.disable=1", "root=/dev/vda ro", "init=/ermite-runner", "reboot=k", "panic=1",	626		"ipv6.disable=1", "root=/dev/vda ro", "init=/ermite-runner", "reboot=k", "panic=1",
627		"ip=10.200.0.6::10.200.0.5:255.255.255.252:ermite-job:eth0:off:10.200.0.5",	627		"ip=10.200.0.6::10.200.0.5:255.255.255.252:ermite-job:eth0:off:10.200.0.5",

@@ -634,6 +634,12 @@ func TestBootArgs(t *testing.T) {
634		if !strings.HasSuffix(a, "-- init") {	634		if !strings.HasSuffix(a, "-- init") {
635		t.Errorf("init subcommand missing: %s", a)	635		t.Errorf("init subcommand missing: %s", a)
636		}	636		}
			637	+	if strings.Contains(a, "ermite.externals") {
			638	+	t.Errorf("externals announced without a disk: %s", a)
			639	+	}
			640	+	if b := bootArgs(mustAddr("10.200.0.6"), mustAddr("10.200.0.5"), 1791818522, true); !strings.Contains(b, " ermite.externals=1 -- init") {
			641	+	t.Errorf("externals not announced: %s", b)
			642	+	}
637		if macFor(0) != "06:00:ac:10:00:00" \|\| macFor(257) != "06:00:ac:10:01:01" {	643		if macFor(0) != "06:00:ac:10:00:00" \|\| macFor(257) != "06:00:ac:10:01:01" {
638		t.Fatal(macFor(257))	644		t.Fatal(macFor(257))
639		}	645		}

@@ -1348,3 +1354,178 @@ func TestGuestForgeIsAttachedAndWarnedAtEveryJob(t *testing.T) {
1348		t.Fatalf("the warning must be logged at every job start (1 + 2), got %d", n)	1354		t.Fatalf("the warning must be logged at every job start (1 + 2), got %d", n)
1349		}	1355		}
1350		}	1356		}
			1357	+
			1358	+	// The host's external tools ride on a fourth, read-only drive, keyed by their
			1359	+	// content: a new Node.js builds a new disk and leaves the exec disk alone.
			1360	+	func TestExternalsDiskIsAttachedReadOnlyAndKeyedByContent(t *testing.T) {
			1361	+	e := newEnv(t, "")
			1362	+	src := &sandboxtest.Externals{}
			1363	+	n24 := sandboxtest.MakeExternal(t, "node24", "node24-24.21.0-x64-aaaa", "v24.21.0")
			1364	+	n20 := sandboxtest.MakeExternal(t, "node20", "node20-20.20.2-x64-bbbb", "v20.20.2")
			1365	+	src.Set(n24, n20)
			1366	+	e.cfg.Externals = src
			1367	+	d := e.driver(t)
			1368	+	execDisk := d.execDisk
			1369	+
			1370	+	boot := func() (drives []string, log string) {
			1371	+	t.Helper()
			1372	+	_ = os.Remove(e.log)
			1373	+	sb, _ := d.New()
			1374	+	conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
			1375	+	if err != nil {
			1376	+	t.Fatal(err)
			1377	+	}
			1378	+	runStub(t, conn)
			1379	+	defer func() { _ = sb.Stop(context.Background()) }()
			1380	+	b, _ := os.ReadFile(e.log)
			1381	+	log = string(b)
			1382	+	for _, l := range strings.Split(log, "\n") {
			1383	+	if f := strings.Fields(l); len(f) >= 2 && f[0] == "PUT" && strings.HasPrefix(f[1], "/drives/") {
			1384	+	drives = append(drives, f[1])
			1385	+	}
			1386	+	}
			1387	+	return drives, log
			1388	+	}
			1389	+	drives, log := boot()
			1390	+	if strings.Join(drives, ",") != "/drives/rootfs,/drives/base,/drives/scratch,/drives/externals" {
			1391	+	t.Fatalf("drives: %v", drives)
			1392	+	}
			1393	+	if !strings.Contains(log, `"drive_id":"externals"`) \|\| !strings.Contains(log, `"is_root_device":false,"is_read_only":true`) \|\| !strings.Contains(log, "ermite.externals=1") {
			1394	+	t.Fatalf("API log:\n%s", log)
			1395	+	}
			1396	+	// The scratch drive is still the third (vdc), and the only writable one.
			1397	+	if strings.Count(log, `"is_read_only":false`) != 1 {
			1398	+	t.Errorf("only the scratch drive is writable:\n%s", log)
			1399	+	}
			1400	+	first, _ := filepath.Glob(filepath.Join(e.cfg.StateDir, "cache", "extdisk-*"))
			1401	+	if len(first) != 1 {
			1402	+	t.Fatalf("externals disks: %v", first)
			1403	+	}
			1404	+	mkfs := func() (n int) {
			1405	+	for _, c := range e.run.Calls {
			1406	+	if c.Name == "mkfs.ext4" && len(c.Args) > 1 {
			1407	+	for _, a := range c.Args {
			1408	+	if a == "ermite-ext" {
			1409	+	n++
			1410	+	}
			1411	+	}
			1412	+	}
			1413	+	}
			1414	+	return
			1415	+	}
			1416	+	if mkfs() != 1 {
			1417	+	t.Fatalf("externals disk built %d times", mkfs())
			1418	+	}
			1419	+
			1420	+	// A second job with the same tools reuses the disk.
			1421	+	boot()
			1422	+	if mkfs() != 1 {
			1423	+	t.Fatal("externals disk rebuilt for the same content")
			1424	+	}
			1425	+
			1426	+	// A new Node.js: another disk, and the exec disk is untouched.
			1427	+	src.Set(sandboxtest.MakeExternal(t, "node24", "node24-24.22.0-x64-cccc", "v24.22.0"), n20)
			1428	+	boot()
			1429	+	if mkfs() != 2 {
			1430	+	t.Fatal("externals disk not rebuilt for a new version")
			1431	+	}
			1432	+	if d.execDisk != execDisk {
			1433	+	t.Fatal("exec disk changed")
			1434	+	}
			1435	+	if _, err := os.Stat(execDisk); err != nil {
			1436	+	t.Fatal("exec disk removed")
			1437	+	}
			1438	+	if now, _ := filepath.Glob(filepath.Join(e.cfg.StateDir, "cache", "extdisk-*")); len(now) != 2 \|\| (now[0] != first[0] && now[1] != first[0]) {
			1439	+	// The previous disk may still be linked by a starting VM: kept for an hour.
			1440	+	t.Fatalf("externals disks: %v", now)
			1441	+	}
			1442	+	}
			1443	+
			1444	+	func TestNoExternalsMeansThreeDrives(t *testing.T) {
			1445	+	for _, src := range []sandbox.ExternalsSource{nil, &sandboxtest.Externals{}} {
			1446	+	e := newEnv(t, "")
			1447	+	e.cfg.Externals = src
			1448	+	d := e.driver(t)
			1449	+	sb, _ := d.New()
			1450	+	conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
			1451	+	if err != nil {
			1452	+	t.Fatal(err)
			1453	+	}
			1454	+	runStub(t, conn)
			1455	+	b, _ := os.ReadFile(e.log)
			1456	+	if strings.Contains(string(b), "externals") {
			1457	+	t.Errorf("externals attached without any:\n%s", b)
			1458	+	}
			1459	+	_ = sb.Stop(context.Background())
			1460	+	if left, _ := filepath.Glob(filepath.Join(e.cfg.StateDir, "cache", "extdisk-*")); len(left) != 0 {
			1461	+	t.Errorf("disk built for nothing: %v", left)
			1462	+	}
			1463	+	}
			1464	+	}
			1465	+
			1466	+	func TestExternalsDiskFailureDoesNotStopTheJob(t *testing.T) {
			1467	+	e := newEnv(t, "")
			1468	+	src := &sandboxtest.Externals{}
			1469	+	src.Set(sandbox.External{Name: "node24", ID: "x", Dir: filepath.Join(t.TempDir(), "missing")})
			1470	+	e.cfg.Externals = src
			1471	+	var buf bytes.Buffer
			1472	+	ctx := zerowrap.WithCtx(context.Background(), zerowrap.New(zerowrap.Config{Level: "debug", Format: "json", Output: &buf}))
			1473	+	d := e.driver(t)
			1474	+	sb, _ := d.New()
			1475	+	conn, err := sb.Start(ctx, sandbox.JobSpec{JobID: "j"})
			1476	+	if err != nil {
			1477	+	t.Fatalf("a broken externals set must not stop the job: %v", err)
			1478	+	}
			1479	+	runStub(t, conn)
			1480	+	_ = sb.Stop(ctx)
			1481	+	if !strings.Contains(buf.String(), "external tools (Node.js) not attached") {
			1482	+	t.Fatalf("failure not logged: %s", buf.String())
			1483	+	}
			1484	+	b, _ := os.ReadFile(e.log)
			1485	+	if strings.Contains(string(b), `"externals"`) {
			1486	+	t.Fatal("drive attached after a failure")
			1487	+	}
			1488	+	}
			1489	+
			1490	+	// With the real mkfs.ext4 the disk holds <name>/bin/node at its root, which the
			1491	+	// guest mounts at /home/runner/externals.
			1492	+	func TestExternalsDiskContent(t *testing.T) {
			1493	+	if _, err := exec.LookPath("mkfs.ext4"); err != nil {
			1494	+	t.Skip("mkfs.ext4 needed")
			1495	+	}
			1496	+	dbg, err := exec.LookPath("debugfs")
			1497	+	if err != nil {
			1498	+	t.Skip("debugfs needed")
			1499	+	}
			1500	+	set := []sandbox.External{
			1501	+	sandboxtest.MakeExternal(t, "node24", "a", "v24"),
			1502	+	sandboxtest.MakeExternal(t, "node20", "b", "v20"),
			1503	+	}
			1504	+	path, err := ensureExternalsDisk(context.Background(), cmdrun.Exec{}, set, t.TempDir())
			1505	+	if err != nil {
			1506	+	t.Fatal(err)
			1507	+	}
			1508	+	out := func(cmd string) string {
			1509	+	b, err := exec.Command(dbg, "-R", cmd, path).CombinedOutput()
			1510	+	if err != nil {
			1511	+	t.Fatalf("debugfs %s: %v\n%s", cmd, err, b)
			1512	+	}
			1513	+	_, body, _ := strings.Cut(string(b), "\n") // the banner line
			1514	+	return body
			1515	+	}
			1516	+	if ls := out("ls -l /"); !strings.Contains(ls, "node24") \|\| !strings.Contains(ls, "node20") {
			1517	+	t.Fatalf("root:\n%s", ls)
			1518	+	}
			1519	+	if got := out("cat /node24/bin/node"); got != "v24" {
			1520	+	t.Fatalf("node24/bin/node: %q", got)
			1521	+	}
			1522	+	if ls := out("stat /node20/bin/node"); !strings.Contains(ls, "Mode: 0755") {
			1523	+	t.Fatalf("mode:\n%s", ls)
			1524	+	}
			1525	+	if ls := out("stat /node20/nodejs"); !strings.Contains(ls, "Type: symlink") \|\| !strings.Contains(ls, `"bin/node"`) {
			1526	+	t.Fatalf("symlink:\n%s", ls)
			1527	+	}
			1528	+	if st, _ := os.Stat(path); st.Size() > 64<<20 {
			1529	+	t.Fatalf("a tiny tree gave a %d byte disk", st.Size())
			1530	+	}
			1531	+	}

M sandbox_check.go +1 -0


@@ -29,6 +29,7 @@ func sandboxCheck(ctx context.Context, args []string) int {
29	29		fmt.Fprintln(os.Stderr, "ermite-runner:", err)
30	30		return 2
31	31		}
	32	+	setupExternals(ctx, &cfg)
32	33		drv, err := sandbox.Select(ctx, cfg, drivers.Factories(), sandbox.Prober{})
33	34		if err != nil {
34	35		fmt.Fprintln(os.Stderr, "ermite-runner:", err)


@@ -29,6 +29,7 @@ func sandboxCheck(ctx context.Context, args []string) int {
29	fmt.Fprintln(os.Stderr, "ermite-runner:", err)	29		fmt.Fprintln(os.Stderr, "ermite-runner:", err)
30	return 2	30		return 2
31	}	31		}
		32	+	setupExternals(ctx, &cfg)
32	drv, err := sandbox.Select(ctx, cfg, drivers.Factories(), sandbox.Prober{})	33		drv, err := sandbox.Select(ctx, cfg, drivers.Factories(), sandbox.Prober{})
33	if err != nil {	34		if err != nil {
34	fmt.Fprintln(os.Stderr, "ermite-runner:", err)	35		fmt.Fprintln(os.Stderr, "ermite-runner:", err)