feat(externals): provide pinned Node.js to sandboxes outside the image

The host downloads the Node.js builds of actions/runner (node20 20.20.2,
node24 24.21.0; the alpine builds are listed but not available yet) into
$STATE_DIR/externals, verifies their pinned SHA-256 before extracting,
extracts safely, keeps only pinned versions, and attaches them read-only at
/home/runner/externals/<name>: a bind mount in container mode, a fourth ext4
disk keyed by content in vm mode (mounted by the guest init).

A build that cannot be provided never stops the runner: the job gets a bin/node
that prints why. ERMITE_RUNNER_NODE=0 turns the feature off.

commit: ad47140bcc687b08eebef018b18e9f04cf62b880
author: brice · 9d ago · 2026-10-13 09:55 UTC
parent: 93b85fe
signature: GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email

20 files · +2125 -29

M README.md +24 -3


@@ -16,7 +16,7 @@ Both modes run the same executable, the runner's own, attached read-only to
16	16		### vm
17	17
18	18		- Firecracker is driven through its HTTP API on a unix socket with `net/http`, not through `firecracker-go-sdk` (its last release, v1.0.0 from 2022, predates Firecracker 1.17 and pulls CNI, go-openapi and netlink for a handful of PUT requests).
19		-	- Disks: the runner executable (read-only boot root, ext4 built with `mkfs.ext4 -d`, cached by the executable's hash), the shared read-only base image, and a per-job sparse scratch disk (`ERMITE_RUNNER_JOB_DISK`). The guest init builds an overlayfs root (lower = base, upper = scratch) and keeps dockerd's data root on the scratch disk. The scratch disk and `eth0` are rate limited (defaults 400 MiB/s and 20000 IOPS, 1 Gbit/s each way; `ERMITE_RUNNER_JOB_DISK_BANDWIDTH` in MiB/s, `_JOB_DISK_IOPS`, `_JOB_NET_BANDWIDTH` in Mbit/s; 0 disables).
	19	+	- Disks: the runner executable (read-only boot root, ext4 built with `mkfs.ext4 -d`, cached by the executable's hash), the shared read-only base image, a per-job sparse scratch disk (`ERMITE_RUNNER_JOB_DISK`) and, when there are externals (below), a fourth read-only disk with the host's Node.js builds, cached by their content. The guest init builds an overlayfs root (lower = base, upper = scratch), mounts the externals disk (`/dev/vdd`) at `/home/runner/externals` and keeps dockerd's data root on the scratch disk. The scratch disk and `eth0` are rate limited (defaults 400 MiB/s and 20000 IOPS, 1 Gbit/s each way; `ERMITE_RUNNER_JOB_DISK_BANDWIDTH` in MiB/s, `_JOB_DISK_IOPS`, `_JOB_NET_BANDWIDTH` in Mbit/s; 0 disables).
20	20		- The jailer runs each VM as uid/gid `ERMITE_RUNNER_UID_BASE` + slot in its own chroot under the state directory, in a new PID namespace and in a cgroup v2 with `memory.max` (guest memory + 256 MiB for the VMM) and `pids.max`. The kernel, the base image and the state directory must be on the same filesystem (files are hard-linked into the jail) and the kernel and base image must be world-readable.
21	21		- The state directory and every parent of it must be owned by root and not writable by group or others. The jailer runs as root and builds the chroots under `<state dir>/jail`; the driver checks that `jail` itself is a real directory, root-owned and not group/other-writable, but it does not walk the parents: a user who can rename or replace a parent directory can still redirect root's file operations. `/var/lib/ermite-runner` (the default, created by the unit's `StateDirectory`) is fine; do not point `ERMITE_RUNNER_STATE_DIR` into a home directory or `/tmp`.
22	22		- One TAP (`ertap<slot>`) per VM with a /30 from `ERMITE_RUNNER_NETWORK` (default `10.200.0.0/16`), vsock for the executor stream, IPv6 disabled in the guest.

@@ -31,6 +31,25 @@ Both modes run the same executable, the runner's own, attached read-only to
31	31		- Per job: a bridge network (`erbr<slot>`), the scratch disk, a Unix socket in a bind-mounted per-job directory of the state directory (the job sees only that directory, not the scratch file). Every Docker object carries the label `dev.bnema.ermite-runner.job`; leftovers are reaped by label at start. The state directory path must keep the socket path under 107 bytes.
32	32		- The Docker daemon must run on the same host as the runner (host paths are bind-mounted).
33	33
	34	+	### Externals (Node.js)
	35	+
	36	+	The host provides Node.js to the jobs outside the image, like its own executable, so that a runner upgrade or a new Node.js version never needs an image rebuild. The builds are the ones of `actions/runner` (`Misc/externals.sh`), pinned in `internal/externals/table.go` with the SHA-256 of each archive:
	37	+
	38	+	\| Name \| Node.js \| Source \| Architectures \|
	39	+	\| --- \| --- \| --- \| --- \|
	40	+	\| `node20` \| 20.20.2 \| `nodejs.org/dist/v20.20.2/node-v20.20.2-linux-<arch>.tar.gz` \| x64, arm64 \|
	41	+	\| `node24` \| 24.21.0 \| `nodejs.org/dist/v24.21.0/node-v24.21.0-linux-<arch>.tar.gz` \| x64, arm64 \|
	42	+	\| `node20_alpine` \| 20.20.2 \| `github.com/actions/alpine_nodejs` release `v20.20.2` \| x64 \|
	43	+	\| `node24_alpine` \| 24.21.0 \| `github.com/actions/alpine_nodejs` release `v24.21.0` \| x64 \|
	44	+
	45	+	- Download. `ermite-runner run` (and `sandbox-check`) fetch what is missing at start, into `$ERMITE_RUNNER_STATE_DIR/externals/<name>-<version>-<arch>/`, in the background; the start waits for them at most 2 minutes (a runner with a network normally takes its first job with everything in place, one without never waits long). The archive is the `.tar.gz` (the Go standard library has no xz); its SHA-256 is checked before anything is unpacked. Redirects must stay on https.
	46	+	- Extraction keeps the top directory out and refuses absolute names, `..`, devices and fifos, symbolic links that are absolute or climb out of the tree, entries below a link, hard links to anything but an unpacked file, and archives over 200,000 entries or 2 GiB; writes go through `os.Root`. The tree is built in a temporary directory and renamed into place, so a directory under its final name is always complete.
	47	+	- Only pinned versions are kept: at every start everything else in `externals/` (other versions, other architectures, interrupted downloads) is removed.
	48	+	- Layout in the sandbox (GitHub's): `/home/runner/externals/node20/bin/node`, `/node24/bin/node`, and the alpine ones for job containers, read-only. In `container` mode they are read-only bind mounts of the host directories; in `vm` mode one ext4 disk (`mkfs.ext4 -d`) named after the content of the set (`cache/extdisk-<key>.ext4`), so a new Node.js builds a new disk and leaves the executable's disk and the base image alone. Superseded disks are removed an hour after the last build. The set is read at every job start: a download that finishes later is used by the next job.
	49	+	- When a build cannot be provided (offline, HTTP error, wrong checksum, unsafe archive, no build for the architecture) the runner still runs: it logs why, retries network failures every 5 minutes in the background, and attaches in its place a directory with the file `UNAVAILABLE` and a `bin/node` script that prints the reason on stderr and exits 1, so a job that needs the tool fails with a message such as `node24 (Node.js 24.21.0) is not available on this runner: download …: SHA-256 mismatch …` rather than `No such file or directory`.
	50	+	- The alpine builds are not available yet. They are hosted on github.com (`actions/alpine_nodejs`) and their SHA-256 is not recorded in `table.go` yet: nothing is downloaded for them, and a job container that needs `node20_alpine` or `node24_alpine` fails with `… is not available yet on this runner: the checksum of the actions/alpine_nodejs build has not been recorded in this runner yet`.
	51	+	- `ERMITE_RUNNER_NODE=0` turns all of this off: nothing is downloaded or attached.
	52	+
34	53		### One runner per host
35	54
36	55		Reaping removes every process, device, firewall rule and Docker object the runner owns, so two runners on one host would destroy each other's jobs. Both modes take an exclusive lock at a fixed host path, `/run/ermite-runner.lock` (`ERMITE_RUNNER_LOCK_FILE` overrides it, for tests), and refuse to start when it is held, whatever their state directories or isolation modes. The kernel drops the lock when the process dies. A runner in a container or in another mount namespace must share that path (or it is not protected): do not run it so.

@@ -209,8 +228,8 @@ mounts, nftables in the host namespace, Docker.
209	228
210	229		A job runs in the image named by `ERMITE_RUNNER_IMAGE` (container: an OCI
211	230		reference; vm: the directory with `vmlinux` and `base.ext4`, both built from
212		-	`images/ubuntu-24.04/Dockerfile`). The runner binary is attached to the sandbox
213		-	by the host and is never part of the image. What the executor needs from the
	231	+	`images/ubuntu-24.04/Dockerfile`). The runner binary and Node.js (see "Externals") are attached to the sandbox
	232	+	by the host and are never part of the image. What the executor needs from the
214	233		image:
215	234
216	235		- a user `runner` with uid and gid 1001 and the home `/home/runner`; the

@@ -261,6 +280,7 @@ the moment it starts.
261	280		\| `ERMITE_RUNNER_JOB_PIDS` \| `32768` \| container process limit \|
262	281		\| `ERMITE_RUNNER_DENY_PRIVILEGED` \| `0` \| `1` refuses container mode without sysbox \|
263	282		\| `ERMITE_RUNNER_GUEST_FORGE` \| `0` \| `1` lets sandboxes reach the host and port of `ERMITE_RUNNER_URL` on the host (staging only, see "Guest forge access") \|
	283	+	\| `ERMITE_RUNNER_NODE` \| `1` \| `0` turns the Node.js externals off (see "Externals") \|
264	284		\| `ERMITE_RUNNER_LOCK_FILE` \| `/run/ermite-runner.lock` \| host-wide single-instance lock (tests only) \|
265	285		\| `ERMITE_RUNNER_JOB_DISK_BANDWIDTH`, `_JOB_DISK_IOPS`, `_JOB_NET_BANDWIDTH` \| `400`, `20000`, `1000` \| VM rate limits in MiB/s, IOPS and Mbit/s; 0 disables \|
266	286		\| `ERMITE_RUNNER_UID_BASE` \| `900000` \| jailer uid pool start \|

@@ -317,6 +337,7 @@ ERMITE_RUNNER_IT_E2E=1 ERMITE_RUNNER_TEST_IMAGE=ermite-runner-ubuntu-24.04:dev G
317	337		- `TestContainerRunsAJobAsTheRunnerUser` and `TestContainerCancelStopsTheJobAndOrphans` drive a job over the stream: steps run as `runner` (1001), file ownership, `GITHUB_PATH` and `GITHUB_ENV`, `python`, cancellation by SIGINT, the `always()` step, and the orphan cleanup of "Complete job".
318	338		- `TestChainCorpus` runs the whole chain: a fake `RunnerService` (`internal/host/hosttest`, which checks the offsets, limits and tokens of `runner.proto`) hands out 22 jobs of the ermite-conformance corpus, the real host loop starts them in containers (two at a time), and what the fake stored (steps, conclusions, outputs, log lines) is compared with the recordings made on github.com. It reads `ERMITE_CONFORMANCE_DIR` (default: a sibling checkout of ermite-conformance) and never writes to it.
319	339		- `TestVMRunsAJob` (`internal/sandbox/integration`, run by `scripts/it-vm-userns.sh`) does the first of these in a micro-VM.
	340	+	- `TestVMNodeExternals` (same place) downloads the pinned `node24` (or takes it from `ERMITE_RUNNER_IT_NODE_DIR`, a directory with `node-v24.21.0-linux-x64.tar.gz`, for an offline host; the checksum and the extraction are the real ones), boots a micro-VM and runs `/home/runner/externals/node24/bin/node -e 'console.log(process.version)'` in it, checking that the mount is read-only.
320	341
321	342		Checked as root on a real KVM host (Arch Linux, kernel 6.18 LTS, Firecracker and jailer 1.17, rootful podman as the Docker API): the jailer (uid drop, chroot, `memory.max` and `pids.max` cgroups, PID namespace), the VM and container integration scenarios, the firewall policy and NAT, the nested `dockerd` on the loop-mounted scratch disk (full disk gives `ENOSPC`), and `internal/e2e` including `TestChainCorpus`. Not checked: sysbox, and a real Docker Engine (podman's compatibility API is not a drop-in, see below).
322	343

 ### vm
 - Firecracker is driven through its HTTP API on a unix socket with `net/http`, not through `firecracker-go-sdk` (its last release, v1.0.0 from 2022, predates Firecracker 1.17 and pulls CNI, go-openapi and netlink for a handful of PUT requests).
-- Disks: the runner executable (read-only boot root, ext4 built with `mkfs.ext4 -d`, cached by the executable's hash), the shared read-only base image, and a per-job sparse scratch disk (`ERMITE_RUNNER_JOB_DISK`). The guest init builds an overlayfs root (lower = base, upper = scratch) and keeps dockerd's data root on the scratch disk. The scratch disk and `eth0` are rate limited (defaults 400 MiB/s and 20000 IOPS, 1 Gbit/s each way; `ERMITE_RUNNER_JOB_DISK_BANDWIDTH` in MiB/s, `_JOB_DISK_IOPS`, `_JOB_NET_BANDWIDTH` in Mbit/s; 0 disables).
+- Disks: the runner executable (read-only boot root, ext4 built with `mkfs.ext4 -d`, cached by the executable's hash), the shared read-only base image, a per-job sparse scratch disk (`ERMITE_RUNNER_JOB_DISK`) and, when there are externals (below), a fourth read-only disk with the host's Node.js builds, cached by their content. The guest init builds an overlayfs root (lower = base, upper = scratch), mounts the externals disk (`/dev/vdd`) at `/home/runner/externals` and keeps dockerd's data root on the scratch disk. The scratch disk and `eth0` are rate limited (defaults 400 MiB/s and 20000 IOPS, 1 Gbit/s each way; `ERMITE_RUNNER_JOB_DISK_BANDWIDTH` in MiB/s, `_JOB_DISK_IOPS`, `_JOB_NET_BANDWIDTH` in Mbit/s; 0 disables).
 - The jailer runs each VM as uid/gid `ERMITE_RUNNER_UID_BASE` + slot in its own chroot under the state directory, in a new PID namespace and in a cgroup v2 with `memory.max` (guest memory + 256 MiB for the VMM) and `pids.max`. The kernel, the base image and the state directory must be on the same filesystem (files are hard-linked into the jail) and the kernel and base image must be world-readable.
 - **The state directory and every parent of it must be owned by root and not writable by group or others.** The jailer runs as root and builds the chroots under `<state dir>/jail`; the driver checks that `jail` itself is a real directory, root-owned and not group/other-writable, but it does not walk the parents: a user who can rename or replace a parent directory can still redirect root's file operations. `/var/lib/ermite-runner` (the default, created by the unit's `StateDirectory`) is fine; do not point `ERMITE_RUNNER_STATE_DIR` into a home directory or `/tmp`.
 - One TAP (`ertap<slot>`) per VM with a /30 from `ERMITE_RUNNER_NETWORK` (default `10.200.0.0/16`), vsock for the executor stream, IPv6 disabled in the guest.
 - Per job: a bridge network (`erbr<slot>`), the scratch disk, a Unix socket in a bind-mounted per-job directory of the state directory (the job sees only that directory, not the scratch file). Every Docker object carries the label `dev.bnema.ermite-runner.job`; leftovers are reaped by label at start. The state directory path must keep the socket path under 107 bytes.
 - The Docker daemon must run on the same host as the runner (host paths are bind-mounted).
+### Externals (Node.js)
+The host provides Node.js to the jobs **outside the image**, like its own executable, so that a runner upgrade or a new Node.js version never needs an image rebuild. The builds are the ones of `actions/runner` (`Misc/externals.sh`), pinned in `internal/externals/table.go` with the SHA-256 of each archive:
+| Name | Node.js | Source | Architectures |
+| --- | --- | --- | --- |
+| `node20` | 20.20.2 | `nodejs.org/dist/v20.20.2/node-v20.20.2-linux-<arch>.tar.gz` | x64, arm64 |
+| `node24` | 24.21.0 | `nodejs.org/dist/v24.21.0/node-v24.21.0-linux-<arch>.tar.gz` | x64, arm64 |
+| `node20_alpine` | 20.20.2 | `github.com/actions/alpine_nodejs` release `v20.20.2` | x64 |
+| `node24_alpine` | 24.21.0 | `github.com/actions/alpine_nodejs` release `v24.21.0` | x64 |
+- **Download.** `ermite-runner run` (and `sandbox-check`) fetch what is missing at start, into `$ERMITE_RUNNER_STATE_DIR/externals/<name>-<version>-<arch>/`, in the background; the start waits for them at most 2 minutes (a runner with a network normally takes its first job with everything in place, one without never waits long). The archive is the `.tar.gz` (the Go standard library has no xz); its SHA-256 is checked **before** anything is unpacked. Redirects must stay on https.
+- **Extraction** keeps the top directory out and refuses absolute names, `..`, devices and fifos, symbolic links that are absolute or climb out of the tree, entries below a link, hard links to anything but an unpacked file, and archives over 200,000 entries or 2 GiB; writes go through `os.Root`. The tree is built in a temporary directory and renamed into place, so a directory under its final name is always complete.
+- **Only pinned versions are kept**: at every start everything else in `externals/` (other versions, other architectures, interrupted downloads) is removed.
+- **Layout in the sandbox** (GitHub's): `/home/runner/externals/node20/bin/node`, `/node24/bin/node`, and the alpine ones for job containers, read-only. In `container` mode they are read-only bind mounts of the host directories; in `vm` mode one ext4 disk (`mkfs.ext4 -d`) named after the content of the set (`cache/extdisk-<key>.ext4`), so a new Node.js builds a new disk and leaves the executable's disk and the base image alone. Superseded disks are removed an hour after the last build. The set is read at every job start: a download that finishes later is used by the next job.
+- **When a build cannot be provided** (offline, HTTP error, wrong checksum, unsafe archive, no build for the architecture) the runner **still runs**: it logs why, retries network failures every 5 minutes in the background, and attaches in its place a directory with the file `UNAVAILABLE` and a `bin/node` script that prints the reason on stderr and exits 1, so a job that needs the tool fails with a message such as `node24 (Node.js 24.21.0) is not available on this runner: download …: SHA-256 mismatch …` rather than `No such file or directory`.
+- **The alpine builds are not available yet.** They are hosted on github.com (`actions/alpine_nodejs`) and their SHA-256 is not recorded in `table.go` yet: nothing is downloaded for them, and a job container that needs `node20_alpine` or `node24_alpine` fails with `… is not available yet on this runner: the checksum of the actions/alpine_nodejs build has not been recorded in this runner yet`.
+- `ERMITE_RUNNER_NODE=0` turns all of this off: nothing is downloaded or attached.
 ### One runner per host
 Reaping removes every process, device, firewall rule and Docker object the runner owns, so two runners on one host would destroy each other's jobs. Both modes take an exclusive lock at a fixed host path, `/run/ermite-runner.lock` (`ERMITE_RUNNER_LOCK_FILE` overrides it, for tests), and refuse to start when it is held, whatever their state directories or isolation modes. The kernel drops the lock when the process dies. A runner in a container or in another mount namespace must share that path (or it is not protected): do not run it so.
 A job runs in the image named by `ERMITE_RUNNER_IMAGE` (container: an OCI
 reference; vm: the directory with `vmlinux` and `base.ext4`, both built from
-`images/ubuntu-24.04/Dockerfile`). The runner binary is attached to the sandbox
+`images/ubuntu-24.04/Dockerfile`). The runner binary and Node.js (see "Externals") are attached to the sandbox
-by the host and is never part of the image. What the executor needs from the
+by the host and are never part of the image. What the executor needs from the
 image:
 - a user **`runner`** with uid and gid 1001 and the home `/home/runner`; the
 | `ERMITE_RUNNER_JOB_PIDS` | `32768` | container process limit |
 | `ERMITE_RUNNER_DENY_PRIVILEGED` | `0` | `1` refuses container mode without sysbox |
 | `ERMITE_RUNNER_GUEST_FORGE` | `0` | `1` lets sandboxes reach the host and port of `ERMITE_RUNNER_URL` on the host (staging only, see "Guest forge access") |
+| `ERMITE_RUNNER_NODE` | `1` | `0` turns the Node.js externals off (see "Externals") |
 | `ERMITE_RUNNER_LOCK_FILE` | `/run/ermite-runner.lock` | host-wide single-instance lock (tests only) |
 | `ERMITE_RUNNER_JOB_DISK_BANDWIDTH`, `_JOB_DISK_IOPS`, `_JOB_NET_BANDWIDTH` | `400`, `20000`, `1000` | VM rate limits in MiB/s, IOPS and Mbit/s; 0 disables |
 | `ERMITE_RUNNER_UID_BASE` | `900000` | jailer uid pool start |
 - `TestContainerRunsAJobAsTheRunnerUser` and `TestContainerCancelStopsTheJobAndOrphans` drive a job over the stream: steps run as `runner` (1001), file ownership, `GITHUB_PATH` and `GITHUB_ENV`, `python`, cancellation by SIGINT, the `always()` step, and the orphan cleanup of "Complete job".
 - `TestChainCorpus` runs the whole chain: a fake `RunnerService` (`internal/host/hosttest`, which checks the offsets, limits and tokens of `runner.proto`) hands out 22 jobs of the ermite-conformance corpus, the real host loop starts them in containers (two at a time), and what the fake stored (steps, conclusions, outputs, log lines) is compared with the recordings made on github.com. It reads `ERMITE_CONFORMANCE_DIR` (default: a sibling checkout of ermite-conformance) and never writes to it.
 - `TestVMRunsAJob` (`internal/sandbox/integration`, run by `scripts/it-vm-userns.sh`) does the first of these in a micro-VM.
+- `TestVMNodeExternals` (same place) downloads the pinned `node24` (or takes it from `ERMITE_RUNNER_IT_NODE_DIR`, a directory with `node-v24.21.0-linux-x64.tar.gz`, for an offline host; the checksum and the extraction are the real ones), boots a micro-VM and runs `/home/runner/externals/node24/bin/node -e 'console.log(process.version)'` in it, checking that the mount is read-only.
 Checked as root on a real KVM host (Arch Linux, kernel 6.18 LTS, Firecracker and jailer 1.17, rootful podman as the Docker API): the jailer (uid drop, chroot, `memory.max` and `pids.max` cgroups, PID namespace), the VM and container integration scenarios, the firewall policy and NAT, the nested `dockerd` on the loop-mounted scratch disk (full disk gives `ENOSPC`), and `internal/e2e` including `TestChainCorpus`. Not checked: sysbox, and a real Docker Engine (podman's compatibility API is not a drop-in, see below).

M host_cmd.go +28 -0


@@ -9,11 +9,13 @@ import (
9	9		"fmt"
10	10		"net"
11	11		"os"
	12	+	"time"
12	13
13	14		"github.com/bnema/zerowrap"
14	15
15	16		"bnema.dev/ermite-api/gen/ermite/ermiteconnect"
16	17
	18	+	"bnema.dev/ermite-runner/internal/externals"
17	19		"bnema.dev/ermite-runner/internal/host"
18	20		"bnema.dev/ermite-runner/internal/sandbox"
19	21		"bnema.dev/ermite-runner/internal/sandbox/drivers"

@@ -31,6 +33,31 @@ func loadHostConfig() (sandbox.Config, host.Config, error) {
31	33		return scfg, hcfg, err
32	34		}
33	35
	36	+	// externalsWait is how long the start of the runner waits for the Node.js
	37	+	// downloads. A runner with a working network is then ready for its first job; one
	38	+	// without goes on after the first failure (nothing waits for a dead network), and
	39	+	// the downloads that are still running when the time is up finish in the
	40	+	// background.
	41	+	const externalsWait = 2 * time.Minute
	42	+
	43	+	// setupExternals downloads the Node.js externals (README, "Externals") and hands
	44	+	// them to the sandbox drivers through scfg. It never fails the start: whatever
	45	+	// cannot be provided is reported to the jobs that need it.
	46	+	func setupExternals(ctx context.Context, scfg *sandbox.Config) {
	47	+	log := zerowrap.FromCtx(ctx)
	48	+	if !scfg.NodeExternals {
	49	+	log.Info().Msg("Node.js externals are off (" + sandbox.EnvNode + "=0)")
	50	+	return
	51	+	}
	52	+	m, ok := externals.New(scfg.StateDir)
	53	+	if !ok {
	54	+	log.Warn().Msg("no Node.js builds for this architecture: jobs get no externals")
	55	+	return
	56	+	}
	57	+	m.Start(ctx, externalsWait)
	58	+	scfg.Externals = m
	59	+	}
	60	+
34	61		func buildDriver(ctx context.Context, scfg sandbox.Config) (sandbox.Driver, error) {
35	62		return sandbox.Select(ctx, scfg, drivers.Factories(), sandbox.Prober{})
36	63		}

@@ -116,6 +143,7 @@ func runCmd(ctx context.Context) int {
116	143		scfg.GuestForge = gf
117	144		hcfg.GuestForgeOrigin = gf.Origin
118	145		}
	146	+	setupExternals(ctx, &scfg)
119	147		drv, err := buildDriver(ctx, scfg)
120	148		if err != nil {
121	149		fmt.Fprintln(os.Stderr, "ermite-runner:", err)


@@ -9,11 +9,13 @@ import (
9	"fmt"	9		"fmt"
10	"net"	10		"net"
11	"os"	11		"os"
		12	+	"time"
12		13
13	"github.com/bnema/zerowrap"	14		"github.com/bnema/zerowrap"
14		15
15	"bnema.dev/ermite-api/gen/ermite/ermiteconnect"	16		"bnema.dev/ermite-api/gen/ermite/ermiteconnect"
16		17
		18	+	"bnema.dev/ermite-runner/internal/externals"
17	"bnema.dev/ermite-runner/internal/host"	19		"bnema.dev/ermite-runner/internal/host"
18	"bnema.dev/ermite-runner/internal/sandbox"	20		"bnema.dev/ermite-runner/internal/sandbox"
19	"bnema.dev/ermite-runner/internal/sandbox/drivers"	21		"bnema.dev/ermite-runner/internal/sandbox/drivers"

@@ -31,6 +33,31 @@ func loadHostConfig() (sandbox.Config, host.Config, error) {
31	return scfg, hcfg, err	33		return scfg, hcfg, err
32	}	34		}
33		35
		36	+	// externalsWait is how long the start of the runner waits for the Node.js
		37	+	// downloads. A runner with a working network is then ready for its first job; one
		38	+	// without goes on after the first failure (nothing waits for a dead network), and
		39	+	// the downloads that are still running when the time is up finish in the
		40	+	// background.
		41	+	const externalsWait = 2 * time.Minute
		42	+
		43	+	// setupExternals downloads the Node.js externals (README, "Externals") and hands
		44	+	// them to the sandbox drivers through scfg. It never fails the start: whatever
		45	+	// cannot be provided is reported to the jobs that need it.
		46	+	func setupExternals(ctx context.Context, scfg *sandbox.Config) {
		47	+	log := zerowrap.FromCtx(ctx)
		48	+	if !scfg.NodeExternals {
		49	+	log.Info().Msg("Node.js externals are off (" + sandbox.EnvNode + "=0)")
		50	+	return
		51	+	}
		52	+	m, ok := externals.New(scfg.StateDir)
		53	+	if !ok {
		54	+	log.Warn().Msg("no Node.js builds for this architecture: jobs get no externals")
		55	+	return
		56	+	}
		57	+	m.Start(ctx, externalsWait)
		58	+	scfg.Externals = m
		59	+	}
		60	+
34	func buildDriver(ctx context.Context, scfg sandbox.Config) (sandbox.Driver, error) {	61		func buildDriver(ctx context.Context, scfg sandbox.Config) (sandbox.Driver, error) {
35	return sandbox.Select(ctx, scfg, drivers.Factories(), sandbox.Prober{})	62		return sandbox.Select(ctx, scfg, drivers.Factories(), sandbox.Prober{})
36	}	63		}

@@ -116,6 +143,7 @@ func runCmd(ctx context.Context) int {
116	scfg.GuestForge = gf	143		scfg.GuestForge = gf
117	hcfg.GuestForgeOrigin = gf.Origin	144		hcfg.GuestForgeOrigin = gf.Origin
118	}	145		}
		146	+	setupExternals(ctx, &scfg)
119	drv, err := buildDriver(ctx, scfg)	147		drv, err := buildDriver(ctx, scfg)
120	if err != nil {	148		if err != nil {
121	fmt.Fprintln(os.Stderr, "ermite-runner:", err)	149		fmt.Fprintln(os.Stderr, "ermite-runner:", err)

A internal/externals/externals_test.go +632 -0

@@ -0,0 +1,632 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package externals
	4	+
	5	+	import (
	6	+	"archive/tar"
	7	+	"bytes"
	8	+	"compress/gzip"
	9	+	"context"
	10	+	"crypto/sha256"
	11	+	"encoding/hex"
	12	+	"net/http"
	13	+	"net/http/httptest"
	14	+	"net/url"
	15	+	"os"
	16	+	"path/filepath"
	17	+	"regexp"
	18	+	"strings"
	19	+	"sync/atomic"
	20	+	"syscall"
	21	+	"testing"
	22	+	"time"
	23	+
	24	+	"bnema.dev/ermite-runner/internal/sandbox"
	25	+	)
	26	+
	27	+	// ---- the table
	28	+
	29	+	func TestPinnedTable(t *testing.T) {
	30	+	hex64 := regexp.MustCompile(`^[0-9a-f]{64}$`)
	31	+	seen := map[string]bool{}
	32	+	for _, a := range Pinned {
	33	+	key := a.Name + "/" + a.Arch
	34	+	if seen[key] {
	35	+	t.Errorf("%s listed twice", key)
	36	+	}
	37	+	seen[key] = true
	38	+	if !strings.HasPrefix(a.URL, "https://") \|\| !strings.HasSuffix(a.URL, ".tar.gz") \|\| !strings.Contains(a.URL, "v"+a.Version+"/") {
	39	+	t.Errorf("%s: URL %q", key, a.URL)
	40	+	}
	41	+	if a.SHA256 == "" {
	42	+	if a.Pending == "" {
	43	+	t.Errorf("%s has neither a checksum nor a reason", key)
	44	+	}
	45	+	if !strings.HasPrefix(a.URL, alpineDist) {
	46	+	t.Errorf("%s: only the alpine builds may lack a checksum", key)
	47	+	}
	48	+	} else if !hex64.MatchString(a.SHA256) {
	49	+	t.Errorf("%s: SHA-256 %q", key, a.SHA256)
	50	+	}
	51	+	}
	52	+	// actions/runner's externals.sh: node20 and node24 for both architectures,
	53	+	// the alpine builds for x64.
	54	+	for _, k := range []string{"node20/x64", "node20/arm64", "node24/x64", "node24/arm64", "node20_alpine/x64", "node24_alpine/x64"} {
	55	+	if !seen[k] {
	56	+	t.Errorf("%s missing", k)
	57	+	}
	58	+	}
	59	+	for _, a := range Pinned {
	60	+	if strings.HasPrefix(a.Name, "node20") && a.Version != "20.20.2" \|\| strings.HasPrefix(a.Name, "node24") && a.Version != "24.21.0" {
	61	+	t.Errorf("%s pins %s", a.Name, a.Version)
	62	+	}
	63	+	}
	64	+	if got := len(ForArch(Pinned, "arm64")); got != 2 {
	65	+	t.Errorf("arm64 has %d artifacts", got)
	66	+	}
	67	+	}
	68	+
	69	+	func TestVerifySum(t *testing.T) {
	70	+	if err := VerifySum("AB12", "ab12"); err != nil {
	71	+	t.Error(err)
	72	+	}
	73	+	if err := VerifySum("ab12", "ab13"); err == nil \|\| !strings.Contains(err.Error(), "mismatch") {
	74	+	t.Errorf("got %v", err)
	75	+	}
	76	+	}
	77	+
	78	+	// ---- archives
	79	+
	80	+	type ent struct {
	81	+	name, link string
	82	+	typ byte
	83	+	mode int64
	84	+	body string
	85	+	}
	86	+
	87	+	func targz(t *testing.T, ents ...ent) []byte {
	88	+	t.Helper()
	89	+	var buf bytes.Buffer
	90	+	gz := gzip.NewWriter(&buf)
	91	+	tw := tar.NewWriter(gz)
	92	+	for _, e := range ents {
	93	+	typ := e.typ
	94	+	if typ == 0 {
	95	+	typ = tar.TypeReg
	96	+	}
	97	+	h := &tar.Header{Name: e.name, Linkname: e.link, Typeflag: typ, Mode: e.mode, Size: int64(len(e.body))}
	98	+	if typ != tar.TypeReg {
	99	+	h.Size = 0
	100	+	}
	101	+	if err := tw.WriteHeader(h); err != nil {
	102	+	t.Fatal(err)
	103	+	}
	104	+	if typ == tar.TypeReg {
	105	+	if _, err := tw.Write([]byte(e.body)); err != nil {
	106	+	t.Fatal(err)
	107	+	}
	108	+	}
	109	+	}
	110	+	if err := tw.Close(); err != nil {
	111	+	t.Fatal(err)
	112	+	}
	113	+	if err := gz.Close(); err != nil {
	114	+	t.Fatal(err)
	115	+	}
	116	+	return buf.Bytes()
	117	+	}
	118	+
	119	+	func nodeTree() []ent {
	120	+	return []ent{
	121	+	{name: "node-v1-linux-x64/", typ: tar.TypeDir, mode: 0o755},
	122	+	{name: "node-v1-linux-x64/bin/", typ: tar.TypeDir, mode: 0o755},
	123	+	{name: "node-v1-linux-x64/bin/node", mode: 0o755, body: "#!/bin/sh\necho v1\n"},
	124	+	{name: "node-v1-linux-x64/lib/node_modules/npm/bin/npm-cli.js", mode: 0o644, body: "x"},
	125	+	{name: "node-v1-linux-x64/bin/npm", typ: tar.TypeSymlink, link: "../lib/node_modules/npm/bin/npm-cli.js", mode: 0o777},
	126	+	{name: "node-v1-linux-x64/bin/nodejs", typ: tar.TypeLink, link: "node-v1-linux-x64/bin/node", mode: 0o755},
	127	+	}
	128	+	}
	129	+
	130	+	func extractBytes(t *testing.T, data []byte) (string, error) {
	131	+	t.Helper()
	132	+	dir := t.TempDir()
	133	+	arch := filepath.Join(dir, "a.tar.gz")
	134	+	if err := os.WriteFile(arch, data, 0o600); err != nil {
	135	+	t.Fatal(err)
	136	+	}
	137	+	dst := filepath.Join(dir, "out")
	138	+	if err := os.Mkdir(dst, 0o755); err != nil {
	139	+	t.Fatal(err)
	140	+	}
	141	+	return dst, extractTarGz(arch, dst)
	142	+	}
	143	+
	144	+	func TestExtractLayoutAndModes(t *testing.T) {
	145	+	dst, err := extractBytes(t, targz(t, nodeTree()...))
	146	+	if err != nil {
	147	+	t.Fatal(err)
	148	+	}
	149	+	st, err := os.Stat(filepath.Join(dst, "bin", "node"))
	150	+	if err != nil \|\| st.Mode().Perm() != 0o755 {
	151	+	t.Fatalf("bin/node: %v %v", st, err)
	152	+	}
	153	+	if st, _ := os.Stat(filepath.Join(dst, "lib/node_modules/npm/bin/npm-cli.js")); st == nil \|\| st.Mode().Perm() != 0o644 {
	154	+	t.Fatalf("npm-cli.js: %v", st)
	155	+	}
	156	+	if l, err := os.Readlink(filepath.Join(dst, "bin", "npm")); err != nil \|\| l != "../lib/node_modules/npm/bin/npm-cli.js" {
	157	+	t.Fatalf("npm link: %q %v", l, err)
	158	+	}
	159	+	a, _ := os.Stat(filepath.Join(dst, "bin", "node"))
	160	+	b, _ := os.Stat(filepath.Join(dst, "bin", "nodejs"))
	161	+	if !os.SameFile(a, b) {
	162	+	t.Fatal("hard link not recreated")
	163	+	}
	164	+	if !isReady(dst) {
	165	+	t.Fatal("not ready")
	166	+	}
	167	+	}
	168	+
	169	+	func TestModesDoNotDependOnTheUmask(t *testing.T) {
	170	+	old := syscall.Umask(0o077)
	171	+	defer syscall.Umask(old)
	172	+	dst, err := extractBytes(t, targz(t, nodeTree()...))
	173	+	if err != nil {
	174	+	t.Fatal(err)
	175	+	}
	176	+	for p, want := range map[string]os.FileMode{
	177	+	dst: 0o755, filepath.Join(dst, "bin"): 0o755, filepath.Join(dst, "lib/node_modules"): 0o755,
	178	+	filepath.Join(dst, "bin/node"): 0o755, filepath.Join(dst, "lib/node_modules/npm/bin/npm-cli.js"): 0o644,
	179	+	} {
	180	+	if st, err := os.Stat(p); err != nil \|\| st.Mode().Perm() != want {
	181	+	t.Errorf("%s: %v %v, want %v", p, st, err, want)
	182	+	}
	183	+	}
	184	+	m := &Manager{Dir: filepath.Join(t.TempDir(), "e"), Arch: "x64"}
	185	+	e := m.unavailable("node24", "why", false)
	186	+	for p, want := range map[string]os.FileMode{
	187	+	filepath.Dir(e.ext.Dir): 0o755, e.ext.Dir: 0o755, filepath.Join(e.ext.Dir, "bin"): 0o755,
	188	+	filepath.Join(e.ext.Dir, "bin/node"): 0o755, filepath.Join(e.ext.Dir, "UNAVAILABLE"): 0o644,
	189	+	} {
	190	+	if st, err := os.Stat(p); err != nil \|\| st.Mode().Perm() != want {
	191	+	t.Errorf("%s: %v %v, want %v", p, st, err, want)
	192	+	}
	193	+	}
	194	+	}
	195	+
	196	+	func TestExtractRefusesUnsafeArchives(t *testing.T) {
	197	+	top := "node-v1-linux-x64/"
	198	+	for name, ents := range map[string][]ent{
	199	+	"absolute path": {{name: "/etc/passwd", body: "x"}},
	200	+	"dot dot": {{name: top + "../../evil", body: "x"}},
	201	+	"dot dot in middle": {{name: top + "a/../../evil", body: "x"}},
	202	+	"backslash": {{name: top + `a\b`, body: "x"}},
	203	+	"absolute symlink": {{name: top + "l", typ: tar.TypeSymlink, link: "/etc"}},
	204	+	"escaping symlink": {{name: top + "l", typ: tar.TypeSymlink, link: "../../.."}},
	205	+	"escaping from deep": {{name: top + "a/b/l", typ: tar.TypeSymlink, link: "../../../x"}},
	206	+	"climb after a name": {{name: top + "a/l", typ: tar.TypeSymlink, link: "b/../../../x"}},
	207	+	"empty link target": {{name: top + "l", typ: tar.TypeSymlink, link: ""}},
	208	+	"write through link": {
	209	+	{name: top + "d", typ: tar.TypeSymlink, link: "."},
	210	+	{name: top + "d/file", body: "x"},
	211	+	},
	212	+	"write through dir link": {
	213	+	{name: top + "sub/", typ: tar.TypeDir},
	214	+	{name: top + "sub/up", typ: tar.TypeSymlink, link: ".."},
	215	+	{name: top + "sub/up/file", body: "x"},
	216	+	},
	217	+	"hard link outside": {{name: top + "h", typ: tar.TypeLink, link: "/etc/passwd"}},
	218	+	"hard link dot dot": {{name: top + "h", typ: tar.TypeLink, link: top + "../x"}},
	219	+	"hard link to top": {{name: top + "h", typ: tar.TypeLink, link: top}},
	220	+	"hard link to a link": {
	221	+	{name: top + "l", typ: tar.TypeSymlink, link: "x"},
	222	+	{name: top + "h", typ: tar.TypeLink, link: top + "l"},
	223	+	},
	224	+	"device": {{name: top + "dev", typ: tar.TypeChar}},
	225	+	"fifo": {{name: top + "fifo", typ: tar.TypeFifo}},
	226	+	"duplicate": {{name: top + "f", body: "1"}, {name: top + "f", body: "2"}},
	227	+	} {
	228	+	t.Run(name, func(t *testing.T) {
	229	+	dst, err := extractBytes(t, targz(t, ents...))
	230	+	if err == nil {
	231	+	t.Fatal("accepted")
	232	+	}
	233	+	// Nothing was written outside the destination, whatever the order.
	234	+	parent := filepath.Dir(dst)
	235	+	if _, err := os.Stat(filepath.Join(parent, "evil")); err == nil {
	236	+	t.Fatal("wrote outside")
	237	+	}
	238	+	if _, err := os.Stat(filepath.Join(dst, "..", "file")); err == nil {
	239	+	t.Fatal("wrote outside through a link")
	240	+	}
	241	+	})
	242	+	}
	243	+	}
	244	+
	245	+	func TestExtractRefusesGarbageAndTooMany(t *testing.T) {
	246	+	if _, err := extractBytes(t, []byte("not gzip")); err == nil {
	247	+	t.Error("garbage accepted")
	248	+	}
	249	+	var ents []ent
	250	+	for i := 0; i < maxEntries+1; i++ {
	251	+	ents = append(ents, ent{name: "t/d/", typ: tar.TypeDir})
	252	+	}
	253	+	if _, err := extractBytes(t, targz(t, ents...)); err == nil \|\| !strings.Contains(err.Error(), "entries") {
	254	+	t.Errorf("entry limit: %v", err)
	255	+	}
	256	+	}
	257	+
	258	+	func TestLinkTargetRule(t *testing.T) {
	259	+	for _, c := range []struct {
	260	+	name, target string
	261	+	ok bool
	262	+	}{
	263	+	{"t/bin/npm", "../lib/x", true},
	264	+	{"t/bin/npm", "../../x", false}, // one above the tree
	265	+	{"t/lib/a/b/l", "../../../x", true},
	266	+	{"t/lib/a/b/l", "../../../../x", false},
	267	+	{"t/l", "bin/node", true},
	268	+	{"t/l", "../x", false},
	269	+	{"t/l", "/abs", false},
	270	+	{"t/a/l", "b/../c", false},
	271	+	} {
	272	+	if err := checkLinkTarget(c.name, c.target); (err == nil) != c.ok {
	273	+	t.Errorf("%s -> %s: %v", c.name, c.target, err)
	274	+	}
	275	+	}
	276	+	}
	277	+
	278	+	// ---- the manager
	279	+
	280	+	type fixture struct {
	281	+	srv *httptest.Server
	282	+	hits atomic.Int64
	283	+	archive []byte
	284	+	sum string
	285	+	table []Artifact
	286	+	dir string
	287	+	m *Manager
	288	+	}
	289	+
	290	+	func newFixture(t testing.T) fixture {
	291	+	t.Helper()
	292	+	f := &fixture{}
	293	+	f.archive = targz(t, nodeTree()...)
	294	+	s := sha256.Sum256(f.archive)
	295	+	f.sum = hex.EncodeToString(s[:])
	296	+	f.srv = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	297	+	f.hits.Add(1)
	298	+	switch {
	299	+	case strings.HasSuffix(r.URL.Path, "-linux-x64.tar.gz"):
	300	+	_, _ = w.Write(f.archive)
	301	+	case strings.HasSuffix(r.URL.Path, "tampered.tar.gz"):
	302	+	_, _ = w.Write(append([]byte(nil), f.archive[:len(f.archive)-1]...))
	303	+	default:
	304	+	http.NotFound(w, r)
	305	+	}
	306	+	}))
	307	+	t.Cleanup(f.srv.Close)
	308	+	f.dir = filepath.Join(t.TempDir(), "externals")
	309	+	f.table = []Artifact{
	310	+	{Name: "node20", Version: "1.0.0", Arch: "x64", URL: f.srv.URL + "/v1.0.0/node-v1.0.0-linux-x64.tar.gz", SHA256: f.sum},
	311	+	{Name: "node24", Version: "2.0.0", Arch: "x64", URL: f.srv.URL + "/v2.0.0/node-v2.0.0-linux-x64.tar.gz", SHA256: f.sum},
	312	+	{Name: "node20_alpine", Version: "1.0.0", Arch: "x64", URL: f.srv.URL + "/v1.0.0/alpine.tar.gz", Pending: "no checksum yet"},
	313	+	}
	314	+	f.m = &Manager{Dir: f.dir, Arch: "x64", Table: f.table, Client: f.srv.Client()}
	315	+	return f
	316	+	}
	317	+
	318	+	func byName(set []sandbox.External) map[string]sandbox.External {
	319	+	m := map[string]sandbox.External{}
	320	+	for _, e := range set {
	321	+	m[e.Name] = e
	322	+	}
	323	+	return m
	324	+	}
	325	+
	326	+	func TestSyncDownloadsVerifiesAndReuses(t *testing.T) {
	327	+	f := newFixture(t)
	328	+	if err := f.m.Sync(context.Background()); err != nil {
	329	+	t.Fatal(err)
	330	+	}
	331	+	set := byName(f.m.Externals())
	332	+	n20, n24, al := set["node20"], set["node24"], set["node20_alpine"]
	333	+	if n20.Unavailable \|\| n24.Unavailable \|\| n20.Dir != filepath.Join(f.dir, "node20-1.0.0-x64") {
	334	+	t.Fatalf("set: %+v", set)
	335	+	}
	336	+	if n20.ID == n24.ID {
	337	+	t.Error("two tools share an ID")
	338	+	}
	339	+	if b, err := os.ReadFile(filepath.Join(n24.Dir, "bin", "node")); err != nil \|\| !strings.Contains(string(b), "echo v1") {
	340	+	t.Fatalf("node24: %q %v", b, err)
	341	+	}
	342	+	// The alpine build has no checksum: not downloaded, with the reason.
	343	+	if !al.Unavailable {
	344	+	t.Fatalf("alpine: %+v", al)
	345	+	}
	346	+	reason, err := os.ReadFile(filepath.Join(al.Dir, sandbox.ExternalUnavailableFile))
	347	+	if err != nil \|\| !strings.Contains(string(reason), "not available yet") \|\| !strings.Contains(string(reason), "no checksum yet") {
	348	+	t.Fatalf("reason: %q %v", reason, err)
	349	+	}
	350	+	stub, err := os.ReadFile(filepath.Join(al.Dir, "bin", "node"))
	351	+	if err != nil \|\| !strings.Contains(string(stub), "not available yet") {
	352	+	t.Fatalf("stub: %q %v", stub, err)
	353	+	}
	354	+	hits := f.hits.Load()
	355	+	if hits != 2 {
	356	+	t.Fatalf("downloads: %d", hits)
	357	+	}
	358	+
	359	+	// A second start finds everything in place and downloads nothing.
	360	+	m2 := &Manager{Dir: f.dir, Arch: "x64", Table: f.table, Client: f.srv.Client()}
	361	+	if err := m2.Sync(context.Background()); err != nil {
	362	+	t.Fatal(err)
	363	+	}
	364	+	if f.hits.Load() != hits {
	365	+	t.Fatal("downloaded again")
	366	+	}
	367	+	if got := byName(m2.Externals())["node24"]; got.ID != n24.ID {
	368	+	t.Fatalf("ID changed: %q -> %q", n24.ID, got.ID)
	369	+	}
	370	+	}
	371	+
	372	+	func TestSyncRefusesAChecksumMismatchBeforeExtracting(t *testing.T) {
	373	+	f := newFixture(t)
	374	+	f.table[0].SHA256 = strings.Repeat("0", 64)
	375	+	f.m.Table = f.table
	376	+	err := f.m.Sync(context.Background())
	377	+	if err == nil \|\| !strings.Contains(err.Error(), "SHA-256 mismatch") {
	378	+	t.Fatalf("got %v", err)
	379	+	}
	380	+	n20 := byName(f.m.Externals())["node20"]
	381	+	if !n20.Unavailable {
	382	+	t.Fatal("a tool with a wrong checksum is available")
	383	+	}
	384	+	if _, err := os.Stat(filepath.Join(f.dir, "node20-1.0.0-x64")); err == nil {
	385	+	t.Fatal("extracted although the checksum differs")
	386	+	}
	387	+	reason, _ := os.ReadFile(filepath.Join(n20.Dir, sandbox.ExternalUnavailableFile))
	388	+	if !strings.Contains(string(reason), "SHA-256 mismatch") {
	389	+	t.Fatalf("reason: %q", reason)
	390	+	}
	391	+	// The other tool is unaffected.
	392	+	if byName(f.m.Externals())["node24"].Unavailable {
	393	+	t.Fatal("node24 lost to node20's failure")
	394	+	}
	395	+	// A checksum failure is a verdict: no retry is scheduled for it.
	396	+	if f.m.state["node20"].retry {
	397	+	t.Fatal("retry scheduled for a mismatch")
	398	+	}
	399	+	ents, _ := os.ReadDir(f.dir)
	400	+	for _, e := range ents {
	401	+	if strings.HasPrefix(e.Name(), ".tmp-") {
	402	+	t.Fatalf("leftover %s", e.Name())
	403	+	}
	404	+	}
	405	+	}
	406	+
	407	+	func TestSyncRefusesAnUnsafeArchiveWithTheRightChecksum(t *testing.T) {
	408	+	f := newFixture(t)
	409	+	bad := targz(t, ent{name: "node-v1-linux-x64/bin/node", mode: 0o755, body: "x"}, ent{name: "node-v1-linux-x64/l", typ: tar.TypeSymlink, link: "/etc"})
	410	+	f.archive = bad
	411	+	s := sha256.Sum256(bad)
	412	+	f.table[0].SHA256, f.table[1].SHA256 = hex.EncodeToString(s[:]), hex.EncodeToString(s[:])
	413	+	f.m.Table = f.table
	414	+	err := f.m.Sync(context.Background())
	415	+	if err == nil \|\| !strings.Contains(err.Error(), "absolute") {
	416	+	t.Fatalf("got %v", err)
	417	+	}
	418	+	if _, err := os.Stat(filepath.Join(f.dir, "node20-1.0.0-x64")); err == nil {
	419	+	t.Fatal("unsafe archive left in place")
	420	+	}
	421	+	}
	422	+
	423	+	func TestSyncOfflineKeepsTheRunnerGoing(t *testing.T) {
	424	+	f := newFixture(t)
	425	+	f.srv.Close() // offline
	426	+	err := f.m.Sync(context.Background())
	427	+	if err == nil {
	428	+	t.Fatal("no error offline")
	429	+	}
	430	+	for _, n := range []string{"node20", "node24"} {
	431	+	e := byName(f.m.Externals())[n]
	432	+	if !e.Unavailable {
	433	+	t.Fatalf("%s available offline", n)
	434	+	}
	435	+	stub, _ := os.ReadFile(filepath.Join(e.Dir, "bin", "node"))
	436	+	if !strings.Contains(string(stub), "is not available on this runner") \|\| !strings.HasPrefix(string(stub), "#!/bin/sh") {
	437	+	t.Fatalf("stub: %q", stub)
	438	+	}
	439	+	}
	440	+	if !f.m.state["node20"].retry {
	441	+	t.Fatal("a network failure must be retried")
	442	+	}
	443	+	}
	444	+
	445	+	func TestBadStatusAndRedirectToHTTP(t *testing.T) {
	446	+	f := newFixture(t)
	447	+	f.table[0].URL = f.srv.URL + "/missing.tar.gz"
	448	+	f.m.Table = f.table
	449	+	if err := f.m.Sync(context.Background()); err == nil \|\| !strings.Contains(err.Error(), "404") {
	450	+	t.Fatalf("got %v", err)
	451	+	}
	452	+	if !byName(f.m.Externals())["node20"].Unavailable {
	453	+	t.Fatal("404 gives a tool")
	454	+	}
	455	+
	456	+	// The default client refuses a redirect to plain http, and too many.
	457	+	check := newClient().CheckRedirect
	458	+	if err := check(&http.Request{URL: mustURL(t, "http://example.org/x")}, nil); err == nil {
	459	+	t.Error("redirect to http followed")
	460	+	}
	461	+	if err := check(&http.Request{URL: mustURL(t, "https://example.org/x")}, make([]*http.Request, 5)); err == nil {
	462	+	t.Error("redirect loop followed")
	463	+	}
	464	+	if err := check(&http.Request{URL: mustURL(t, "https://example.org/x")}, make([]*http.Request, 1)); err != nil {
	465	+	t.Error(err)
	466	+	}
	467	+	// A plain http URL in the table is refused outright.
	468	+	m2 := &Manager{Dir: filepath.Join(t.TempDir(), "e"), Arch: "x64", Client: f.srv.Client()}
	469	+	m2.Table = []Artifact{{Name: "node20", Version: "1", Arch: "x64", URL: "http://example.invalid/a.tar.gz", SHA256: f.sum}}
	470	+	if err := m2.Sync(context.Background()); err == nil \|\| !strings.Contains(err.Error(), "not an https URL") {
	471	+	t.Fatalf("got %v", err)
	472	+	}
	473	+	}
	474	+
	475	+	func mustURL(t testing.T, s string) url.URL {
	476	+	t.Helper()
	477	+	u, err := url.Parse(s)
	478	+	if err != nil {
	479	+	t.Fatal(err)
	480	+	}
	481	+	return u
	482	+	}
	483	+
	484	+	func TestSyncKeepsOnlyPinnedVersions(t *testing.T) {
	485	+	f := newFixture(t)
	486	+	if err := f.m.Sync(context.Background()); err != nil {
	487	+	t.Fatal(err)
	488	+	}
	489	+	// Leftovers: an old version, an interrupted download, a stray file, an
	490	+	// incomplete directory under a pinned name.
	491	+	for _, d := range []string{"node20-0.9.0-x64", ".tmp-123", "node24-2.0.0-arm64", "node-old"} {
	492	+	if err := os.MkdirAll(filepath.Join(f.dir, d, "bin"), 0o755); err != nil {
	493	+	t.Fatal(err)
	494	+	}
	495	+	}
	496	+	if err := os.WriteFile(filepath.Join(f.dir, "stray"), []byte("x"), 0o644); err != nil {
	497	+	t.Fatal(err)
	498	+	}
	499	+	if err := f.m.Sync(context.Background()); err != nil {
	500	+	t.Fatal(err)
	501	+	}
	502	+	var got []string
	503	+	ents, _ := os.ReadDir(f.dir)
	504	+	for _, e := range ents {
	505	+	got = append(got, e.Name())
	506	+	}
	507	+	want := ".unavailable node20-1.0.0-x64 node24-2.0.0-x64"
	508	+	if strings.Join(got, " ") != want {
	509	+	t.Fatalf("kept %v, want %s", got, want)
	510	+	}
	511	+	}
	512	+
	513	+	func TestSyncRebuildsAnIncompleteDirectory(t *testing.T) {
	514	+	f := newFixture(t)
	515	+	final := filepath.Join(f.dir, "node20-1.0.0-x64")
	516	+	if err := os.MkdirAll(filepath.Join(final, "bin"), 0o755); err != nil { // no bin/node
	517	+	t.Fatal(err)
	518	+	}
	519	+	if err := f.m.Sync(context.Background()); err != nil {
	520	+	t.Fatal(err)
	521	+	}
	522	+	if !isReady(final) {
	523	+	t.Fatal("incomplete directory kept")
	524	+	}
	525	+	}
	526	+
	527	+	func TestANewVersionChangesTheIDAndDropsTheOldDirectory(t *testing.T) {
	528	+	f := newFixture(t)
	529	+	if err := f.m.Sync(context.Background()); err != nil {
	530	+	t.Fatal(err)
	531	+	}
	532	+	old := byName(f.m.Externals())["node24"]
	533	+	f.table[1].Version = "2.1.0"
	534	+	f.table[1].URL = f.srv.URL + "/v2.1.0/node-v2.1.0-linux-x64.tar.gz"
	535	+	if err := f.m.Sync(context.Background()); err != nil {
	536	+	t.Fatal(err)
	537	+	}
	538	+	n := byName(f.m.Externals())["node24"]
	539	+	if n.ID == old.ID \|\| n.Dir == old.Dir {
	540	+	t.Fatalf("same tool after an upgrade: %+v %+v", old, n)
	541	+	}
	542	+	if _, err := os.Stat(old.Dir); err == nil {
	543	+	t.Fatal("old version kept")
	544	+	}
	545	+	}
	546	+
	547	+	func TestNoBuildForTheArchitecture(t *testing.T) {
	548	+	f := newFixture(t)
	549	+	f.m.Arch = "arm64" // the fixture table is x64 only
	550	+	if err := f.m.Sync(context.Background()); err != nil {
	551	+	t.Fatal(err)
	552	+	}
	553	+	for _, e := range f.m.Externals() {
	554	+	reason, _ := os.ReadFile(filepath.Join(e.Dir, sandbox.ExternalUnavailableFile))
	555	+	if !e.Unavailable \|\| !strings.Contains(string(reason), "no "+e.Name+" build for linux-arm64") {
	556	+	t.Errorf("%s: %+v %q", e.Name, e, reason)
	557	+	}
	558	+	}
	559	+	if f.hits.Load() != 0 {
	560	+	t.Error("downloaded for the wrong architecture")
	561	+	}
	562	+	}
	563	+
	564	+	func TestBeforeTheFirstSyncTheToolsAreBeingDownloaded(t *testing.T) {
	565	+	f := newFixture(t)
	566	+	set := f.m.Externals()
	567	+	if len(set) != 3 {
	568	+	t.Fatalf("set: %+v", set)
	569	+	}
	570	+	for _, e := range set {
	571	+	reason, _ := os.ReadFile(filepath.Join(e.Dir, sandbox.ExternalUnavailableFile))
	572	+	if !e.Unavailable \|\| !strings.Contains(string(reason), "still being downloaded") {
	573	+	t.Errorf("%s: %q", e.Name, reason)
	574	+	}
	575	+	}
	576	+	}
	577	+
	578	+	func TestFailedDownloadIsRetriedInTheBackground(t *testing.T) {
	579	+	f := newFixture(t)
	580	+	var up atomic.Bool
	581	+	srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	582	+	if !up.Load() {
	583	+	http.Error(w, "down", http.StatusServiceUnavailable)
	584	+	return
	585	+	}
	586	+	_, _ = w.Write(f.archive)
	587	+	}))
	588	+	defer srv.Close()
	589	+	m := &Manager{Dir: f.dir, Arch: "x64", Client: srv.Client(), RetryEvery: time.Millisecond}
	590	+	m.Table = []Artifact{{Name: "node24", Version: "2.0.0", Arch: "x64", URL: srv.URL + "/a-linux-x64.tar.gz", SHA256: f.sum}}
	591	+	ctx, cancel := context.WithCancel(context.Background())
	592	+	defer cancel()
	593	+	m.Start(ctx, time.Minute)
	594	+	if !m.Externals()[0].Unavailable {
	595	+	t.Fatal("available while the server is down")
	596	+	}
	597	+	up.Store(true)
	598	+	deadline := time.Now().Add(10 * time.Second)
	599	+	for time.Now().Before(deadline) {
	600	+	time.Sleep(20 * time.Millisecond)
	601	+	if !m.Externals()[0].Unavailable {
	602	+	return
	603	+	}
	604	+	}
	605	+	t.Fatal("never retried")
	606	+	}
	607	+
	608	+	func TestStartDoesNotWaitLongerThanAsked(t *testing.T) {
	609	+	block := make(chan struct{})
	610	+	srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { <-block }))
	611	+	defer srv.Close()
	612	+	m := &Manager{Dir: filepath.Join(t.TempDir(), "e"), Arch: "x64", Client: srv.Client()}
	613	+	m.Table = []Artifact{{Name: "node24", Version: "2.0.0", Arch: "x64", URL: srv.URL + "/a.tar.gz", SHA256: strings.Repeat("a", 64)}}
	614	+	ctx, cancel := context.WithCancel(context.Background())
	615	+	start := time.Now()
	616	+	m.Start(ctx, 100*time.Millisecond)
	617	+	if d := time.Since(start); d > 5*time.Second {
	618	+	t.Fatalf("waited %v", d)
	619	+	}
	620	+	// The download goes on in the background; stop it before the directory goes.
	621	+	cancel()
	622	+	close(block)
	623	+	for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(10 * time.Millisecond) {
	624	+	m.mu.Lock()
	625	+	busy := m.syncing
	626	+	m.mu.Unlock()
	627	+	if !busy {
	628	+	return
	629	+	}
	630	+	}
	631	+	t.Fatal("the background sync did not stop")
	632	+	}

@@ -0,0 +1,632 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package externals
	4	+
	5	+	import (
	6	+	"archive/tar"
	7	+	"bytes"
	8	+	"compress/gzip"
	9	+	"context"
	10	+	"crypto/sha256"
	11	+	"encoding/hex"
	12	+	"net/http"
	13	+	"net/http/httptest"
	14	+	"net/url"
	15	+	"os"
	16	+	"path/filepath"
	17	+	"regexp"
	18	+	"strings"
	19	+	"sync/atomic"
	20	+	"syscall"
	21	+	"testing"
	22	+	"time"
	23	+
	24	+	"bnema.dev/ermite-runner/internal/sandbox"
	25	+	)
	26	+
	27	+	// ---- the table
	28	+
	29	+	func TestPinnedTable(t *testing.T) {
	30	+	hex64 := regexp.MustCompile(`^[0-9a-f]{64}$`)
	31	+	seen := map[string]bool{}
	32	+	for _, a := range Pinned {
	33	+	key := a.Name + "/" + a.Arch
	34	+	if seen[key] {
	35	+	t.Errorf("%s listed twice", key)
	36	+	}
	37	+	seen[key] = true
	38	+	if !strings.HasPrefix(a.URL, "https://") \|\| !strings.HasSuffix(a.URL, ".tar.gz") \|\| !strings.Contains(a.URL, "v"+a.Version+"/") {
	39	+	t.Errorf("%s: URL %q", key, a.URL)
	40	+	}
	41	+	if a.SHA256 == "" {
	42	+	if a.Pending == "" {
	43	+	t.Errorf("%s has neither a checksum nor a reason", key)
	44	+	}
	45	+	if !strings.HasPrefix(a.URL, alpineDist) {
	46	+	t.Errorf("%s: only the alpine builds may lack a checksum", key)
	47	+	}
	48	+	} else if !hex64.MatchString(a.SHA256) {
	49	+	t.Errorf("%s: SHA-256 %q", key, a.SHA256)
	50	+	}
	51	+	}
	52	+	// actions/runner's externals.sh: node20 and node24 for both architectures,
	53	+	// the alpine builds for x64.
	54	+	for _, k := range []string{"node20/x64", "node20/arm64", "node24/x64", "node24/arm64", "node20_alpine/x64", "node24_alpine/x64"} {
	55	+	if !seen[k] {
	56	+	t.Errorf("%s missing", k)
	57	+	}
	58	+	}
	59	+	for _, a := range Pinned {
	60	+	if strings.HasPrefix(a.Name, "node20") && a.Version != "20.20.2" \|\| strings.HasPrefix(a.Name, "node24") && a.Version != "24.21.0" {
	61	+	t.Errorf("%s pins %s", a.Name, a.Version)
	62	+	}
	63	+	}
	64	+	if got := len(ForArch(Pinned, "arm64")); got != 2 {
	65	+	t.Errorf("arm64 has %d artifacts", got)
	66	+	}
	67	+	}
	68	+
	69	+	func TestVerifySum(t *testing.T) {
	70	+	if err := VerifySum("AB12", "ab12"); err != nil {
	71	+	t.Error(err)
	72	+	}
	73	+	if err := VerifySum("ab12", "ab13"); err == nil \|\| !strings.Contains(err.Error(), "mismatch") {
	74	+	t.Errorf("got %v", err)
	75	+	}
	76	+	}
	77	+
	78	+	// ---- archives
	79	+
	80	+	type ent struct {
	81	+	name, link string
	82	+	typ byte
	83	+	mode int64
	84	+	body string
	85	+	}
	86	+
	87	+	func targz(t *testing.T, ents ...ent) []byte {
	88	+	t.Helper()
	89	+	var buf bytes.Buffer
	90	+	gz := gzip.NewWriter(&buf)
	91	+	tw := tar.NewWriter(gz)
	92	+	for _, e := range ents {
	93	+	typ := e.typ
	94	+	if typ == 0 {
	95	+	typ = tar.TypeReg
	96	+	}
	97	+	h := &tar.Header{Name: e.name, Linkname: e.link, Typeflag: typ, Mode: e.mode, Size: int64(len(e.body))}
	98	+	if typ != tar.TypeReg {
	99	+	h.Size = 0
	100	+	}
	101	+	if err := tw.WriteHeader(h); err != nil {
	102	+	t.Fatal(err)
	103	+	}
	104	+	if typ == tar.TypeReg {
	105	+	if _, err := tw.Write([]byte(e.body)); err != nil {
	106	+	t.Fatal(err)
	107	+	}
	108	+	}
	109	+	}
	110	+	if err := tw.Close(); err != nil {
	111	+	t.Fatal(err)
	112	+	}
	113	+	if err := gz.Close(); err != nil {
	114	+	t.Fatal(err)
	115	+	}
	116	+	return buf.Bytes()
	117	+	}
	118	+
	119	+	func nodeTree() []ent {
	120	+	return []ent{
	121	+	{name: "node-v1-linux-x64/", typ: tar.TypeDir, mode: 0o755},
	122	+	{name: "node-v1-linux-x64/bin/", typ: tar.TypeDir, mode: 0o755},
	123	+	{name: "node-v1-linux-x64/bin/node", mode: 0o755, body: "#!/bin/sh\necho v1\n"},
	124	+	{name: "node-v1-linux-x64/lib/node_modules/npm/bin/npm-cli.js", mode: 0o644, body: "x"},
	125	+	{name: "node-v1-linux-x64/bin/npm", typ: tar.TypeSymlink, link: "../lib/node_modules/npm/bin/npm-cli.js", mode: 0o777},
	126	+	{name: "node-v1-linux-x64/bin/nodejs", typ: tar.TypeLink, link: "node-v1-linux-x64/bin/node", mode: 0o755},
	127	+	}
	128	+	}
	129	+
	130	+	func extractBytes(t *testing.T, data []byte) (string, error) {
	131	+	t.Helper()
	132	+	dir := t.TempDir()
	133	+	arch := filepath.Join(dir, "a.tar.gz")
	134	+	if err := os.WriteFile(arch, data, 0o600); err != nil {
	135	+	t.Fatal(err)
	136	+	}
	137	+	dst := filepath.Join(dir, "out")
	138	+	if err := os.Mkdir(dst, 0o755); err != nil {
	139	+	t.Fatal(err)
	140	+	}
	141	+	return dst, extractTarGz(arch, dst)
	142	+	}
	143	+
	144	+	func TestExtractLayoutAndModes(t *testing.T) {
	145	+	dst, err := extractBytes(t, targz(t, nodeTree()...))
	146	+	if err != nil {
	147	+	t.Fatal(err)
	148	+	}
	149	+	st, err := os.Stat(filepath.Join(dst, "bin", "node"))
	150	+	if err != nil \|\| st.Mode().Perm() != 0o755 {
	151	+	t.Fatalf("bin/node: %v %v", st, err)
	152	+	}
	153	+	if st, _ := os.Stat(filepath.Join(dst, "lib/node_modules/npm/bin/npm-cli.js")); st == nil \|\| st.Mode().Perm() != 0o644 {
	154	+	t.Fatalf("npm-cli.js: %v", st)
	155	+	}
	156	+	if l, err := os.Readlink(filepath.Join(dst, "bin", "npm")); err != nil \|\| l != "../lib/node_modules/npm/bin/npm-cli.js" {
	157	+	t.Fatalf("npm link: %q %v", l, err)
	158	+	}
	159	+	a, _ := os.Stat(filepath.Join(dst, "bin", "node"))
	160	+	b, _ := os.Stat(filepath.Join(dst, "bin", "nodejs"))
	161	+	if !os.SameFile(a, b) {
	162	+	t.Fatal("hard link not recreated")
	163	+	}
	164	+	if !isReady(dst) {
	165	+	t.Fatal("not ready")
	166	+	}
	167	+	}
	168	+
	169	+	func TestModesDoNotDependOnTheUmask(t *testing.T) {
	170	+	old := syscall.Umask(0o077)
	171	+	defer syscall.Umask(old)
	172	+	dst, err := extractBytes(t, targz(t, nodeTree()...))
	173	+	if err != nil {
	174	+	t.Fatal(err)
	175	+	}
	176	+	for p, want := range map[string]os.FileMode{
	177	+	dst: 0o755, filepath.Join(dst, "bin"): 0o755, filepath.Join(dst, "lib/node_modules"): 0o755,
	178	+	filepath.Join(dst, "bin/node"): 0o755, filepath.Join(dst, "lib/node_modules/npm/bin/npm-cli.js"): 0o644,
	179	+	} {
	180	+	if st, err := os.Stat(p); err != nil \|\| st.Mode().Perm() != want {
	181	+	t.Errorf("%s: %v %v, want %v", p, st, err, want)
	182	+	}
	183	+	}
	184	+	m := &Manager{Dir: filepath.Join(t.TempDir(), "e"), Arch: "x64"}
	185	+	e := m.unavailable("node24", "why", false)
	186	+	for p, want := range map[string]os.FileMode{
	187	+	filepath.Dir(e.ext.Dir): 0o755, e.ext.Dir: 0o755, filepath.Join(e.ext.Dir, "bin"): 0o755,
	188	+	filepath.Join(e.ext.Dir, "bin/node"): 0o755, filepath.Join(e.ext.Dir, "UNAVAILABLE"): 0o644,
	189	+	} {
	190	+	if st, err := os.Stat(p); err != nil \|\| st.Mode().Perm() != want {
	191	+	t.Errorf("%s: %v %v, want %v", p, st, err, want)
	192	+	}
	193	+	}
	194	+	}
	195	+
	196	+	func TestExtractRefusesUnsafeArchives(t *testing.T) {
	197	+	top := "node-v1-linux-x64/"
	198	+	for name, ents := range map[string][]ent{
	199	+	"absolute path": {{name: "/etc/passwd", body: "x"}},
	200	+	"dot dot": {{name: top + "../../evil", body: "x"}},
	201	+	"dot dot in middle": {{name: top + "a/../../evil", body: "x"}},
	202	+	"backslash": {{name: top + `a\b`, body: "x"}},
	203	+	"absolute symlink": {{name: top + "l", typ: tar.TypeSymlink, link: "/etc"}},
	204	+	"escaping symlink": {{name: top + "l", typ: tar.TypeSymlink, link: "../../.."}},
	205	+	"escaping from deep": {{name: top + "a/b/l", typ: tar.TypeSymlink, link: "../../../x"}},
	206	+	"climb after a name": {{name: top + "a/l", typ: tar.TypeSymlink, link: "b/../../../x"}},
	207	+	"empty link target": {{name: top + "l", typ: tar.TypeSymlink, link: ""}},
	208	+	"write through link": {
	209	+	{name: top + "d", typ: tar.TypeSymlink, link: "."},
	210	+	{name: top + "d/file", body: "x"},
	211	+	},
	212	+	"write through dir link": {
	213	+	{name: top + "sub/", typ: tar.TypeDir},
	214	+	{name: top + "sub/up", typ: tar.TypeSymlink, link: ".."},
	215	+	{name: top + "sub/up/file", body: "x"},
	216	+	},
	217	+	"hard link outside": {{name: top + "h", typ: tar.TypeLink, link: "/etc/passwd"}},
	218	+	"hard link dot dot": {{name: top + "h", typ: tar.TypeLink, link: top + "../x"}},
	219	+	"hard link to top": {{name: top + "h", typ: tar.TypeLink, link: top}},
	220	+	"hard link to a link": {
	221	+	{name: top + "l", typ: tar.TypeSymlink, link: "x"},
	222	+	{name: top + "h", typ: tar.TypeLink, link: top + "l"},
	223	+	},
	224	+	"device": {{name: top + "dev", typ: tar.TypeChar}},
	225	+	"fifo": {{name: top + "fifo", typ: tar.TypeFifo}},
	226	+	"duplicate": {{name: top + "f", body: "1"}, {name: top + "f", body: "2"}},
	227	+	} {
	228	+	t.Run(name, func(t *testing.T) {
	229	+	dst, err := extractBytes(t, targz(t, ents...))
	230	+	if err == nil {
	231	+	t.Fatal("accepted")
	232	+	}
	233	+	// Nothing was written outside the destination, whatever the order.
	234	+	parent := filepath.Dir(dst)
	235	+	if _, err := os.Stat(filepath.Join(parent, "evil")); err == nil {
	236	+	t.Fatal("wrote outside")
	237	+	}
	238	+	if _, err := os.Stat(filepath.Join(dst, "..", "file")); err == nil {
	239	+	t.Fatal("wrote outside through a link")
	240	+	}
	241	+	})
	242	+	}
	243	+	}
	244	+
	245	+	func TestExtractRefusesGarbageAndTooMany(t *testing.T) {
	246	+	if _, err := extractBytes(t, []byte("not gzip")); err == nil {
	247	+	t.Error("garbage accepted")
	248	+	}
	249	+	var ents []ent
	250	+	for i := 0; i < maxEntries+1; i++ {
	251	+	ents = append(ents, ent{name: "t/d/", typ: tar.TypeDir})
	252	+	}
	253	+	if _, err := extractBytes(t, targz(t, ents...)); err == nil \|\| !strings.Contains(err.Error(), "entries") {
	254	+	t.Errorf("entry limit: %v", err)
	255	+	}
	256	+	}
	257	+
	258	+	func TestLinkTargetRule(t *testing.T) {
	259	+	for _, c := range []struct {
	260	+	name, target string
	261	+	ok bool
	262	+	}{
	263	+	{"t/bin/npm", "../lib/x", true},
	264	+	{"t/bin/npm", "../../x", false}, // one above the tree
	265	+	{"t/lib/a/b/l", "../../../x", true},
	266	+	{"t/lib/a/b/l", "../../../../x", false},
	267	+	{"t/l", "bin/node", true},
	268	+	{"t/l", "../x", false},
	269	+	{"t/l", "/abs", false},
	270	+	{"t/a/l", "b/../c", false},
	271	+	} {
	272	+	if err := checkLinkTarget(c.name, c.target); (err == nil) != c.ok {
	273	+	t.Errorf("%s -> %s: %v", c.name, c.target, err)
	274	+	}
	275	+	}
	276	+	}
	277	+
	278	+	// ---- the manager
	279	+
	280	+	type fixture struct {
	281	+	srv *httptest.Server
	282	+	hits atomic.Int64
	283	+	archive []byte
	284	+	sum string
	285	+	table []Artifact
	286	+	dir string
	287	+	m *Manager
	288	+	}
	289	+
	290	+	func newFixture(t testing.T) fixture {
	291	+	t.Helper()
	292	+	f := &fixture{}
	293	+	f.archive = targz(t, nodeTree()...)
	294	+	s := sha256.Sum256(f.archive)
	295	+	f.sum = hex.EncodeToString(s[:])
	296	+	f.srv = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	297	+	f.hits.Add(1)
	298	+	switch {
	299	+	case strings.HasSuffix(r.URL.Path, "-linux-x64.tar.gz"):
	300	+	_, _ = w.Write(f.archive)
	301	+	case strings.HasSuffix(r.URL.Path, "tampered.tar.gz"):
	302	+	_, _ = w.Write(append([]byte(nil), f.archive[:len(f.archive)-1]...))
	303	+	default:
	304	+	http.NotFound(w, r)
	305	+	}
	306	+	}))
	307	+	t.Cleanup(f.srv.Close)
	308	+	f.dir = filepath.Join(t.TempDir(), "externals")
	309	+	f.table = []Artifact{
	310	+	{Name: "node20", Version: "1.0.0", Arch: "x64", URL: f.srv.URL + "/v1.0.0/node-v1.0.0-linux-x64.tar.gz", SHA256: f.sum},
	311	+	{Name: "node24", Version: "2.0.0", Arch: "x64", URL: f.srv.URL + "/v2.0.0/node-v2.0.0-linux-x64.tar.gz", SHA256: f.sum},
	312	+	{Name: "node20_alpine", Version: "1.0.0", Arch: "x64", URL: f.srv.URL + "/v1.0.0/alpine.tar.gz", Pending: "no checksum yet"},
	313	+	}
	314	+	f.m = &Manager{Dir: f.dir, Arch: "x64", Table: f.table, Client: f.srv.Client()}
	315	+	return f
	316	+	}
	317	+
	318	+	func byName(set []sandbox.External) map[string]sandbox.External {
	319	+	m := map[string]sandbox.External{}
	320	+	for _, e := range set {
	321	+	m[e.Name] = e
	322	+	}
	323	+	return m
	324	+	}
	325	+
	326	+	func TestSyncDownloadsVerifiesAndReuses(t *testing.T) {
	327	+	f := newFixture(t)
	328	+	if err := f.m.Sync(context.Background()); err != nil {
	329	+	t.Fatal(err)
	330	+	}
	331	+	set := byName(f.m.Externals())
	332	+	n20, n24, al := set["node20"], set["node24"], set["node20_alpine"]
	333	+	if n20.Unavailable \|\| n24.Unavailable \|\| n20.Dir != filepath.Join(f.dir, "node20-1.0.0-x64") {
	334	+	t.Fatalf("set: %+v", set)
	335	+	}
	336	+	if n20.ID == n24.ID {
	337	+	t.Error("two tools share an ID")
	338	+	}
	339	+	if b, err := os.ReadFile(filepath.Join(n24.Dir, "bin", "node")); err != nil \|\| !strings.Contains(string(b), "echo v1") {
	340	+	t.Fatalf("node24: %q %v", b, err)
	341	+	}
	342	+	// The alpine build has no checksum: not downloaded, with the reason.
	343	+	if !al.Unavailable {
	344	+	t.Fatalf("alpine: %+v", al)
	345	+	}
	346	+	reason, err := os.ReadFile(filepath.Join(al.Dir, sandbox.ExternalUnavailableFile))
	347	+	if err != nil \|\| !strings.Contains(string(reason), "not available yet") \|\| !strings.Contains(string(reason), "no checksum yet") {
	348	+	t.Fatalf("reason: %q %v", reason, err)
	349	+	}
	350	+	stub, err := os.ReadFile(filepath.Join(al.Dir, "bin", "node"))
	351	+	if err != nil \|\| !strings.Contains(string(stub), "not available yet") {
	352	+	t.Fatalf("stub: %q %v", stub, err)
	353	+	}
	354	+	hits := f.hits.Load()
	355	+	if hits != 2 {
	356	+	t.Fatalf("downloads: %d", hits)
	357	+	}
	358	+
	359	+	// A second start finds everything in place and downloads nothing.
	360	+	m2 := &Manager{Dir: f.dir, Arch: "x64", Table: f.table, Client: f.srv.Client()}
	361	+	if err := m2.Sync(context.Background()); err != nil {
	362	+	t.Fatal(err)
	363	+	}
	364	+	if f.hits.Load() != hits {
	365	+	t.Fatal("downloaded again")
	366	+	}
	367	+	if got := byName(m2.Externals())["node24"]; got.ID != n24.ID {
	368	+	t.Fatalf("ID changed: %q -> %q", n24.ID, got.ID)
	369	+	}
	370	+	}
	371	+
	372	+	func TestSyncRefusesAChecksumMismatchBeforeExtracting(t *testing.T) {
	373	+	f := newFixture(t)
	374	+	f.table[0].SHA256 = strings.Repeat("0", 64)
	375	+	f.m.Table = f.table
	376	+	err := f.m.Sync(context.Background())
	377	+	if err == nil \|\| !strings.Contains(err.Error(), "SHA-256 mismatch") {
	378	+	t.Fatalf("got %v", err)
	379	+	}
	380	+	n20 := byName(f.m.Externals())["node20"]
	381	+	if !n20.Unavailable {
	382	+	t.Fatal("a tool with a wrong checksum is available")
	383	+	}
	384	+	if _, err := os.Stat(filepath.Join(f.dir, "node20-1.0.0-x64")); err == nil {
	385	+	t.Fatal("extracted although the checksum differs")
	386	+	}
	387	+	reason, _ := os.ReadFile(filepath.Join(n20.Dir, sandbox.ExternalUnavailableFile))
	388	+	if !strings.Contains(string(reason), "SHA-256 mismatch") {
	389	+	t.Fatalf("reason: %q", reason)
	390	+	}
	391	+	// The other tool is unaffected.
	392	+	if byName(f.m.Externals())["node24"].Unavailable {
	393	+	t.Fatal("node24 lost to node20's failure")
	394	+	}
	395	+	// A checksum failure is a verdict: no retry is scheduled for it.
	396	+	if f.m.state["node20"].retry {
	397	+	t.Fatal("retry scheduled for a mismatch")
	398	+	}
	399	+	ents, _ := os.ReadDir(f.dir)
	400	+	for _, e := range ents {
	401	+	if strings.HasPrefix(e.Name(), ".tmp-") {
	402	+	t.Fatalf("leftover %s", e.Name())
	403	+	}
	404	+	}
	405	+	}
	406	+
	407	+	func TestSyncRefusesAnUnsafeArchiveWithTheRightChecksum(t *testing.T) {
	408	+	f := newFixture(t)
	409	+	bad := targz(t, ent{name: "node-v1-linux-x64/bin/node", mode: 0o755, body: "x"}, ent{name: "node-v1-linux-x64/l", typ: tar.TypeSymlink, link: "/etc"})
	410	+	f.archive = bad
	411	+	s := sha256.Sum256(bad)
	412	+	f.table[0].SHA256, f.table[1].SHA256 = hex.EncodeToString(s[:]), hex.EncodeToString(s[:])
	413	+	f.m.Table = f.table
	414	+	err := f.m.Sync(context.Background())
	415	+	if err == nil \|\| !strings.Contains(err.Error(), "absolute") {
	416	+	t.Fatalf("got %v", err)
	417	+	}
	418	+	if _, err := os.Stat(filepath.Join(f.dir, "node20-1.0.0-x64")); err == nil {
	419	+	t.Fatal("unsafe archive left in place")
	420	+	}
	421	+	}
	422	+
	423	+	func TestSyncOfflineKeepsTheRunnerGoing(t *testing.T) {
	424	+	f := newFixture(t)
	425	+	f.srv.Close() // offline
	426	+	err := f.m.Sync(context.Background())
	427	+	if err == nil {
	428	+	t.Fatal("no error offline")
	429	+	}
	430	+	for _, n := range []string{"node20", "node24"} {
	431	+	e := byName(f.m.Externals())[n]
	432	+	if !e.Unavailable {
	433	+	t.Fatalf("%s available offline", n)
	434	+	}
	435	+	stub, _ := os.ReadFile(filepath.Join(e.Dir, "bin", "node"))
	436	+	if !strings.Contains(string(stub), "is not available on this runner") \|\| !strings.HasPrefix(string(stub), "#!/bin/sh") {
	437	+	t.Fatalf("stub: %q", stub)
	438	+	}
	439	+	}
	440	+	if !f.m.state["node20"].retry {
	441	+	t.Fatal("a network failure must be retried")
	442	+	}
	443	+	}
	444	+
	445	+	func TestBadStatusAndRedirectToHTTP(t *testing.T) {
	446	+	f := newFixture(t)
	447	+	f.table[0].URL = f.srv.URL + "/missing.tar.gz"
	448	+	f.m.Table = f.table
	449	+	if err := f.m.Sync(context.Background()); err == nil \|\| !strings.Contains(err.Error(), "404") {
	450	+	t.Fatalf("got %v", err)
	451	+	}
	452	+	if !byName(f.m.Externals())["node20"].Unavailable {
	453	+	t.Fatal("404 gives a tool")
	454	+	}
	455	+
	456	+	// The default client refuses a redirect to plain http, and too many.
	457	+	check := newClient().CheckRedirect
	458	+	if err := check(&http.Request{URL: mustURL(t, "http://example.org/x")}, nil); err == nil {
	459	+	t.Error("redirect to http followed")
	460	+	}
	461	+	if err := check(&http.Request{URL: mustURL(t, "https://example.org/x")}, make([]*http.Request, 5)); err == nil {
	462	+	t.Error("redirect loop followed")
	463	+	}
	464	+	if err := check(&http.Request{URL: mustURL(t, "https://example.org/x")}, make([]*http.Request, 1)); err != nil {
	465	+	t.Error(err)
	466	+	}
	467	+	// A plain http URL in the table is refused outright.
	468	+	m2 := &Manager{Dir: filepath.Join(t.TempDir(), "e"), Arch: "x64", Client: f.srv.Client()}
	469	+	m2.Table = []Artifact{{Name: "node20", Version: "1", Arch: "x64", URL: "http://example.invalid/a.tar.gz", SHA256: f.sum}}
	470	+	if err := m2.Sync(context.Background()); err == nil \|\| !strings.Contains(err.Error(), "not an https URL") {
	471	+	t.Fatalf("got %v", err)
	472	+	}
	473	+	}
	474	+
	475	+	func mustURL(t testing.T, s string) url.URL {
	476	+	t.Helper()
	477	+	u, err := url.Parse(s)
	478	+	if err != nil {
	479	+	t.Fatal(err)
	480	+	}
	481	+	return u
	482	+	}
	483	+
	484	+	func TestSyncKeepsOnlyPinnedVersions(t *testing.T) {
	485	+	f := newFixture(t)
	486	+	if err := f.m.Sync(context.Background()); err != nil {
	487	+	t.Fatal(err)
	488	+	}
	489	+	// Leftovers: an old version, an interrupted download, a stray file, an
	490	+	// incomplete directory under a pinned name.
	491	+	for _, d := range []string{"node20-0.9.0-x64", ".tmp-123", "node24-2.0.0-arm64", "node-old"} {
	492	+	if err := os.MkdirAll(filepath.Join(f.dir, d, "bin"), 0o755); err != nil {
	493	+	t.Fatal(err)
	494	+	}
	495	+	}
	496	+	if err := os.WriteFile(filepath.Join(f.dir, "stray"), []byte("x"), 0o644); err != nil {
	497	+	t.Fatal(err)
	498	+	}
	499	+	if err := f.m.Sync(context.Background()); err != nil {
	500	+	t.Fatal(err)
	501	+	}
	502	+	var got []string
	503	+	ents, _ := os.ReadDir(f.dir)
	504	+	for _, e := range ents {
	505	+	got = append(got, e.Name())
	506	+	}
	507	+	want := ".unavailable node20-1.0.0-x64 node24-2.0.0-x64"
	508	+	if strings.Join(got, " ") != want {
	509	+	t.Fatalf("kept %v, want %s", got, want)
	510	+	}
	511	+	}
	512	+
	513	+	func TestSyncRebuildsAnIncompleteDirectory(t *testing.T) {
	514	+	f := newFixture(t)
	515	+	final := filepath.Join(f.dir, "node20-1.0.0-x64")
	516	+	if err := os.MkdirAll(filepath.Join(final, "bin"), 0o755); err != nil { // no bin/node
	517	+	t.Fatal(err)
	518	+	}
	519	+	if err := f.m.Sync(context.Background()); err != nil {
	520	+	t.Fatal(err)
	521	+	}
	522	+	if !isReady(final) {
	523	+	t.Fatal("incomplete directory kept")
	524	+	}
	525	+	}
	526	+
	527	+	func TestANewVersionChangesTheIDAndDropsTheOldDirectory(t *testing.T) {
	528	+	f := newFixture(t)
	529	+	if err := f.m.Sync(context.Background()); err != nil {
	530	+	t.Fatal(err)
	531	+	}
	532	+	old := byName(f.m.Externals())["node24"]
	533	+	f.table[1].Version = "2.1.0"
	534	+	f.table[1].URL = f.srv.URL + "/v2.1.0/node-v2.1.0-linux-x64.tar.gz"
	535	+	if err := f.m.Sync(context.Background()); err != nil {
	536	+	t.Fatal(err)
	537	+	}
	538	+	n := byName(f.m.Externals())["node24"]
	539	+	if n.ID == old.ID \|\| n.Dir == old.Dir {
	540	+	t.Fatalf("same tool after an upgrade: %+v %+v", old, n)
	541	+	}
	542	+	if _, err := os.Stat(old.Dir); err == nil {
	543	+	t.Fatal("old version kept")
	544	+	}
	545	+	}
	546	+
	547	+	func TestNoBuildForTheArchitecture(t *testing.T) {
	548	+	f := newFixture(t)
	549	+	f.m.Arch = "arm64" // the fixture table is x64 only
	550	+	if err := f.m.Sync(context.Background()); err != nil {
	551	+	t.Fatal(err)
	552	+	}
	553	+	for _, e := range f.m.Externals() {
	554	+	reason, _ := os.ReadFile(filepath.Join(e.Dir, sandbox.ExternalUnavailableFile))
	555	+	if !e.Unavailable \|\| !strings.Contains(string(reason), "no "+e.Name+" build for linux-arm64") {
	556	+	t.Errorf("%s: %+v %q", e.Name, e, reason)
	557	+	}
	558	+	}
	559	+	if f.hits.Load() != 0 {
	560	+	t.Error("downloaded for the wrong architecture")
	561	+	}
	562	+	}
	563	+
	564	+	func TestBeforeTheFirstSyncTheToolsAreBeingDownloaded(t *testing.T) {
	565	+	f := newFixture(t)
	566	+	set := f.m.Externals()
	567	+	if len(set) != 3 {
	568	+	t.Fatalf("set: %+v", set)
	569	+	}
	570	+	for _, e := range set {
	571	+	reason, _ := os.ReadFile(filepath.Join(e.Dir, sandbox.ExternalUnavailableFile))
	572	+	if !e.Unavailable \|\| !strings.Contains(string(reason), "still being downloaded") {
	573	+	t.Errorf("%s: %q", e.Name, reason)
	574	+	}
	575	+	}
	576	+	}
	577	+
	578	+	func TestFailedDownloadIsRetriedInTheBackground(t *testing.T) {
	579	+	f := newFixture(t)
	580	+	var up atomic.Bool
	581	+	srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	582	+	if !up.Load() {
	583	+	http.Error(w, "down", http.StatusServiceUnavailable)
	584	+	return
	585	+	}
	586	+	_, _ = w.Write(f.archive)
	587	+	}))
	588	+	defer srv.Close()
	589	+	m := &Manager{Dir: f.dir, Arch: "x64", Client: srv.Client(), RetryEvery: time.Millisecond}
	590	+	m.Table = []Artifact{{Name: "node24", Version: "2.0.0", Arch: "x64", URL: srv.URL + "/a-linux-x64.tar.gz", SHA256: f.sum}}
	591	+	ctx, cancel := context.WithCancel(context.Background())
	592	+	defer cancel()
	593	+	m.Start(ctx, time.Minute)
	594	+	if !m.Externals()[0].Unavailable {
	595	+	t.Fatal("available while the server is down")
	596	+	}
	597	+	up.Store(true)
	598	+	deadline := time.Now().Add(10 * time.Second)
	599	+	for time.Now().Before(deadline) {
	600	+	time.Sleep(20 * time.Millisecond)
	601	+	if !m.Externals()[0].Unavailable {
	602	+	return
	603	+	}
	604	+	}
	605	+	t.Fatal("never retried")
	606	+	}
	607	+
	608	+	func TestStartDoesNotWaitLongerThanAsked(t *testing.T) {
	609	+	block := make(chan struct{})
	610	+	srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { <-block }))
	611	+	defer srv.Close()
	612	+	m := &Manager{Dir: filepath.Join(t.TempDir(), "e"), Arch: "x64", Client: srv.Client()}
	613	+	m.Table = []Artifact{{Name: "node24", Version: "2.0.0", Arch: "x64", URL: srv.URL + "/a.tar.gz", SHA256: strings.Repeat("a", 64)}}
	614	+	ctx, cancel := context.WithCancel(context.Background())
	615	+	start := time.Now()
	616	+	m.Start(ctx, 100*time.Millisecond)
	617	+	if d := time.Since(start); d > 5*time.Second {
	618	+	t.Fatalf("waited %v", d)
	619	+	}
	620	+	// The download goes on in the background; stop it before the directory goes.
	621	+	cancel()
	622	+	close(block)
	623	+	for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(10 * time.Millisecond) {
	624	+	m.mu.Lock()
	625	+	busy := m.syncing
	626	+	m.mu.Unlock()
	627	+	if !busy {
	628	+	return
	629	+	}
	630	+	}
	631	+	t.Fatal("the background sync did not stop")
	632	+	}

A internal/externals/extract.go +223 -0

@@ -0,0 +1,223 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package externals
	4	+
	5	+	import (
	6	+	"archive/tar"
	7	+	"compress/gzip"
	8	+	"errors"
	9	+	"fmt"
	10	+	"io"
	11	+	"io/fs"
	12	+	"os"
	13	+	"path"
	14	+	"path/filepath"
	15	+	"strings"
	16	+	)
	17	+
	18	+	// Limits on what an archive may unpack to. They are far above a Node.js
	19	+	// release (about 6,500 entries and 220 MiB) and exist to bound a corrupt or
	20	+	// hostile archive that still has the right checksum.
	21	+	const (
	22	+	maxEntries = 200_000
	23	+	maxUnpack = 2 << 30
	24	+	)
	25	+
	26	+	// extractTarGz unpacks the .tar.gz at archive into the existing directory dest,
	27	+	// dropping the first path component (Node.js archives hold one top directory,
	28	+	// node-vX-linux-x64). It refuses what could write or point outside dest:
	29	+	//
	30	+	// - absolute names and names with ".." components;
	31	+	// - anything but directories, regular files, symbolic links and hard links;
	32	+	// - symbolic links that are absolute or whose target climbs with ".." anywhere
	33	+	// but at the start (so that the target cannot walk back out through another
	34	+	// link), and entries below a symbolic link;
	35	+	// - hard links to something that is not already unpacked.
	36	+	//
	37	+	// Writes go through os.Root, which also refuses to follow a link out of dest.
	38	+	// File modes are reduced to 0755 / 0644: the owner of the files is whoever runs
	39	+	// the runner, and the sandboxes only read and execute them.
	40	+	func extractTarGz(archive, dest string) error {
	41	+	f, err := os.Open(archive)
	42	+	if err != nil {
	43	+	return err
	44	+	}
	45	+	defer func() { _ = f.Close() }()
	46	+	gz, err := gzip.NewReader(f)
	47	+	if err != nil {
	48	+	return fmt.Errorf("not a gzip file: %w", err)
	49	+	}
	50	+	defer func() { _ = gz.Close() }()
	51	+	root, err := os.OpenRoot(dest)
	52	+	if err != nil {
	53	+	return err
	54	+	}
	55	+	defer func() { _ = root.Close() }()
	56	+
	57	+	tr := tar.NewReader(gz)
	58	+	links := map[string]bool{} // unpacked symbolic links
	59	+	var entries int
	60	+	var total int64
	61	+	for {
	62	+	h, err := tr.Next()
	63	+	if errors.Is(err, io.EOF) {
	64	+	return openDirs(dest)
	65	+	}
	66	+	if err != nil {
	67	+	return fmt.Errorf("read archive: %w", err)
	68	+	}
	69	+	if entries++; entries > maxEntries {
	70	+	return fmt.Errorf("archive has more than %d entries", maxEntries)
	71	+	}
	72	+	if h.Typeflag == tar.TypeXGlobalHeader {
	73	+	continue
	74	+	}
	75	+	name, err := stripTop(h.Name)
	76	+	if err != nil {
	77	+	return err
	78	+	}
	79	+	if name == "" { // the top directory itself
	80	+	continue
	81	+	}
	82	+	if under(links, name) {
	83	+	return fmt.Errorf("entry %q is below a symbolic link", h.Name)
	84	+	}
	85	+	switch h.Typeflag {
	86	+	case tar.TypeDir:
	87	+	err = root.MkdirAll(name, 0o755)
	88	+	case tar.TypeReg:
	89	+	if total += h.Size; total > maxUnpack {
	90	+	return fmt.Errorf("archive unpacks to more than %d bytes", int64(maxUnpack))
	91	+	}
	92	+	err = writeFile(root, name, tr, h.Mode)
	93	+	case tar.TypeSymlink:
	94	+	if err = checkLinkTarget(h.Name, h.Linkname); err == nil {
	95	+	if err = root.MkdirAll(path.Dir(name), 0o755); err == nil {
	96	+	if err = root.Symlink(h.Linkname, name); err == nil {
	97	+	links[name] = true
	98	+	}
	99	+	}
	100	+	}
	101	+	case tar.TypeLink:
	102	+	var old string
	103	+	if old, err = stripTop(h.Linkname); err == nil {
	104	+	if old == "" \|\| under(links, old) {
	105	+	err = fmt.Errorf("hard link %q points at %q", h.Name, h.Linkname)
	106	+	} else if err = root.MkdirAll(path.Dir(name), 0o755); err == nil {
	107	+	err = root.Link(old, name)
	108	+	}
	109	+	}
	110	+	default:
	111	+	err = fmt.Errorf("entry %q has the unsupported type %q", h.Name, string(h.Typeflag))
	112	+	}
	113	+	if err != nil {
	114	+	return fmt.Errorf("unpack %q: %w", h.Name, err)
	115	+	}
	116	+	}
	117	+	}
	118	+
	119	+	func writeFile(root *os.Root, name string, r io.Reader, mode int64) error {
	120	+	if err := root.MkdirAll(path.Dir(name), 0o755); err != nil {
	121	+	return err
	122	+	}
	123	+	perm := os.FileMode(0o644)
	124	+	if mode&0o111 != 0 {
	125	+	perm = 0o755
	126	+	}
	127	+	f, err := root.OpenFile(name, os.O_WRONLY\|os.O_CREATE\|os.O_EXCL, perm)
	128	+	if err != nil {
	129	+	return err
	130	+	}
	131	+	if _, err := io.Copy(f, r); err != nil {
	132	+	_ = f.Close()
	133	+	return err
	134	+	}
	135	+	if err := f.Chmod(perm); err != nil { // the umask may have removed bits
	136	+	_ = f.Close()
	137	+	return err
	138	+	}
	139	+	return f.Close()
	140	+	}
	141	+
	142	+	// stripTop validates an archive path and removes its first component. It
	143	+	// returns "" for the top directory itself.
	144	+	func stripTop(name string) (string, error) {
	145	+	if name == "" \|\| strings.ContainsRune(name, 0) \|\| strings.ContainsRune(name, '\\') {
	146	+	return "", fmt.Errorf("invalid entry name %q", name)
	147	+	}
	148	+	if strings.HasPrefix(name, "/") {
	149	+	return "", fmt.Errorf("entry %q has an absolute path", name)
	150	+	}
	151	+	var parts []string
	152	+	for _, p := range strings.Split(name, "/") {
	153	+	switch p {
	154	+	case "", ".":
	155	+	case "..":
	156	+	return "", fmt.Errorf("entry %q has a .. component", name)
	157	+	default:
	158	+	parts = append(parts, p)
	159	+	}
	160	+	}
	161	+	if len(parts) <= 1 {
	162	+	return "", nil
	163	+	}
	164	+	return strings.Join(parts[1:], "/"), nil
	165	+	}
	166	+
	167	+	// checkLinkTarget accepts a relative target whose ".." components all come
	168	+	// first. Below the top of the tree that lexical rule is exact, because no entry
	169	+	// is ever unpacked below a symbolic link.
	170	+	func checkLinkTarget(name, target string) error {
	171	+	if target == "" \|\| strings.ContainsRune(target, 0) \|\| strings.HasPrefix(target, "/") {
	172	+	return fmt.Errorf("symbolic link %q has the target %q: absolute or empty", name, target)
	173	+	}
	174	+	// depth is how many directories lie between the tree's top and the link.
	175	+	depth := 0
	176	+	if dir := path.Dir(mustStrip(name)); dir != "." {
	177	+	depth = strings.Count(dir, "/") + 1
	178	+	}
	179	+	up, seenName := 0, false
	180	+	for _, p := range strings.Split(target, "/") {
	181	+	switch p {
	182	+	case "", ".":
	183	+	case "..":
	184	+	if seenName {
	185	+	return fmt.Errorf("symbolic link %q: target %q climbs after a name", name, target)
	186	+	}
	187	+	up++
	188	+	default:
	189	+	seenName = true
	190	+	}
	191	+	}
	192	+	if up > depth {
	193	+	return fmt.Errorf("symbolic link %q: target %q leaves the archive", name, target)
	194	+	}
	195	+	return nil
	196	+	}
	197	+
	198	+	// mustStrip is stripTop for a name that was already validated.
	199	+	func mustStrip(name string) string {
	200	+	s, _ := stripTop(name)
	201	+	return s
	202	+	}
	203	+
	204	+	// under reports whether name, or one of its parents, is an unpacked symbolic link.
	205	+	func under(links map[string]bool, name string) bool {
	206	+	for p := name; p != "." && p != "/" && p != ""; p = path.Dir(p) {
	207	+	if links[p] {
	208	+	return true
	209	+	}
	210	+	}
	211	+	return false
	212	+	}
	213	+
	214	+	// openDirs gives every directory the mode 0755, whatever the umask of the
	215	+	// runner was when it created them: the sandboxes read the tree as another user.
	216	+	func openDirs(dest string) error {
	217	+	return filepath.WalkDir(dest, func(p string, d fs.DirEntry, err error) error {
	218	+	if err != nil \|\| !d.IsDir() {
	219	+	return err
	220	+	}
	221	+	return os.Chmod(p, 0o755)
	222	+	})
	223	+	}

@@ -0,0 +1,223 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package externals
	4	+
	5	+	import (
	6	+	"archive/tar"
	7	+	"compress/gzip"
	8	+	"errors"
	9	+	"fmt"
	10	+	"io"
	11	+	"io/fs"
	12	+	"os"
	13	+	"path"
	14	+	"path/filepath"
	15	+	"strings"
	16	+	)
	17	+
	18	+	// Limits on what an archive may unpack to. They are far above a Node.js
	19	+	// release (about 6,500 entries and 220 MiB) and exist to bound a corrupt or
	20	+	// hostile archive that still has the right checksum.
	21	+	const (
	22	+	maxEntries = 200_000
	23	+	maxUnpack = 2 << 30
	24	+	)
	25	+
	26	+	// extractTarGz unpacks the .tar.gz at archive into the existing directory dest,
	27	+	// dropping the first path component (Node.js archives hold one top directory,
	28	+	// node-vX-linux-x64). It refuses what could write or point outside dest:
	29	+	//
	30	+	// - absolute names and names with ".." components;
	31	+	// - anything but directories, regular files, symbolic links and hard links;
	32	+	// - symbolic links that are absolute or whose target climbs with ".." anywhere
	33	+	// but at the start (so that the target cannot walk back out through another
	34	+	// link), and entries below a symbolic link;
	35	+	// - hard links to something that is not already unpacked.
	36	+	//
	37	+	// Writes go through os.Root, which also refuses to follow a link out of dest.
	38	+	// File modes are reduced to 0755 / 0644: the owner of the files is whoever runs
	39	+	// the runner, and the sandboxes only read and execute them.
	40	+	func extractTarGz(archive, dest string) error {
	41	+	f, err := os.Open(archive)
	42	+	if err != nil {
	43	+	return err
	44	+	}
	45	+	defer func() { _ = f.Close() }()
	46	+	gz, err := gzip.NewReader(f)
	47	+	if err != nil {
	48	+	return fmt.Errorf("not a gzip file: %w", err)
	49	+	}
	50	+	defer func() { _ = gz.Close() }()
	51	+	root, err := os.OpenRoot(dest)
	52	+	if err != nil {
	53	+	return err
	54	+	}
	55	+	defer func() { _ = root.Close() }()
	56	+
	57	+	tr := tar.NewReader(gz)
	58	+	links := map[string]bool{} // unpacked symbolic links
	59	+	var entries int
	60	+	var total int64
	61	+	for {
	62	+	h, err := tr.Next()
	63	+	if errors.Is(err, io.EOF) {
	64	+	return openDirs(dest)
	65	+	}
	66	+	if err != nil {
	67	+	return fmt.Errorf("read archive: %w", err)
	68	+	}
	69	+	if entries++; entries > maxEntries {
	70	+	return fmt.Errorf("archive has more than %d entries", maxEntries)
	71	+	}
	72	+	if h.Typeflag == tar.TypeXGlobalHeader {
	73	+	continue
	74	+	}
	75	+	name, err := stripTop(h.Name)
	76	+	if err != nil {
	77	+	return err
	78	+	}
	79	+	if name == "" { // the top directory itself
	80	+	continue
	81	+	}
	82	+	if under(links, name) {
	83	+	return fmt.Errorf("entry %q is below a symbolic link", h.Name)
	84	+	}
	85	+	switch h.Typeflag {
	86	+	case tar.TypeDir:
	87	+	err = root.MkdirAll(name, 0o755)
	88	+	case tar.TypeReg:
	89	+	if total += h.Size; total > maxUnpack {
	90	+	return fmt.Errorf("archive unpacks to more than %d bytes", int64(maxUnpack))
	91	+	}
	92	+	err = writeFile(root, name, tr, h.Mode)
	93	+	case tar.TypeSymlink:
	94	+	if err = checkLinkTarget(h.Name, h.Linkname); err == nil {
	95	+	if err = root.MkdirAll(path.Dir(name), 0o755); err == nil {
	96	+	if err = root.Symlink(h.Linkname, name); err == nil {
	97	+	links[name] = true
	98	+	}
	99	+	}
	100	+	}
	101	+	case tar.TypeLink:
	102	+	var old string
	103	+	if old, err = stripTop(h.Linkname); err == nil {
	104	+	if old == "" \|\| under(links, old) {
	105	+	err = fmt.Errorf("hard link %q points at %q", h.Name, h.Linkname)
	106	+	} else if err = root.MkdirAll(path.Dir(name), 0o755); err == nil {
	107	+	err = root.Link(old, name)
	108	+	}
	109	+	}
	110	+	default:
	111	+	err = fmt.Errorf("entry %q has the unsupported type %q", h.Name, string(h.Typeflag))
	112	+	}
	113	+	if err != nil {
	114	+	return fmt.Errorf("unpack %q: %w", h.Name, err)
	115	+	}
	116	+	}
	117	+	}
	118	+
	119	+	func writeFile(root *os.Root, name string, r io.Reader, mode int64) error {
	120	+	if err := root.MkdirAll(path.Dir(name), 0o755); err != nil {
	121	+	return err
	122	+	}
	123	+	perm := os.FileMode(0o644)
	124	+	if mode&0o111 != 0 {
	125	+	perm = 0o755
	126	+	}
	127	+	f, err := root.OpenFile(name, os.O_WRONLY\|os.O_CREATE\|os.O_EXCL, perm)
	128	+	if err != nil {
	129	+	return err
	130	+	}
	131	+	if _, err := io.Copy(f, r); err != nil {
	132	+	_ = f.Close()
	133	+	return err
	134	+	}
	135	+	if err := f.Chmod(perm); err != nil { // the umask may have removed bits
	136	+	_ = f.Close()
	137	+	return err
	138	+	}
	139	+	return f.Close()
	140	+	}
	141	+
	142	+	// stripTop validates an archive path and removes its first component. It
	143	+	// returns "" for the top directory itself.
	144	+	func stripTop(name string) (string, error) {
	145	+	if name == "" \|\| strings.ContainsRune(name, 0) \|\| strings.ContainsRune(name, '\\') {
	146	+	return "", fmt.Errorf("invalid entry name %q", name)
	147	+	}
	148	+	if strings.HasPrefix(name, "/") {
	149	+	return "", fmt.Errorf("entry %q has an absolute path", name)
	150	+	}
	151	+	var parts []string
	152	+	for _, p := range strings.Split(name, "/") {
	153	+	switch p {
	154	+	case "", ".":
	155	+	case "..":
	156	+	return "", fmt.Errorf("entry %q has a .. component", name)
	157	+	default:
	158	+	parts = append(parts, p)
	159	+	}
	160	+	}
	161	+	if len(parts) <= 1 {
	162	+	return "", nil
	163	+	}
	164	+	return strings.Join(parts[1:], "/"), nil
	165	+	}
	166	+
	167	+	// checkLinkTarget accepts a relative target whose ".." components all come
	168	+	// first. Below the top of the tree that lexical rule is exact, because no entry
	169	+	// is ever unpacked below a symbolic link.
	170	+	func checkLinkTarget(name, target string) error {
	171	+	if target == "" \|\| strings.ContainsRune(target, 0) \|\| strings.HasPrefix(target, "/") {
	172	+	return fmt.Errorf("symbolic link %q has the target %q: absolute or empty", name, target)
	173	+	}
	174	+	// depth is how many directories lie between the tree's top and the link.
	175	+	depth := 0
	176	+	if dir := path.Dir(mustStrip(name)); dir != "." {
	177	+	depth = strings.Count(dir, "/") + 1
	178	+	}
	179	+	up, seenName := 0, false
	180	+	for _, p := range strings.Split(target, "/") {
	181	+	switch p {
	182	+	case "", ".":
	183	+	case "..":
	184	+	if seenName {
	185	+	return fmt.Errorf("symbolic link %q: target %q climbs after a name", name, target)
	186	+	}
	187	+	up++
	188	+	default:
	189	+	seenName = true
	190	+	}
	191	+	}
	192	+	if up > depth {
	193	+	return fmt.Errorf("symbolic link %q: target %q leaves the archive", name, target)
	194	+	}
	195	+	return nil
	196	+	}
	197	+
	198	+	// mustStrip is stripTop for a name that was already validated.
	199	+	func mustStrip(name string) string {
	200	+	s, _ := stripTop(name)
	201	+	return s
	202	+	}
	203	+
	204	+	// under reports whether name, or one of its parents, is an unpacked symbolic link.
	205	+	func under(links map[string]bool, name string) bool {
	206	+	for p := name; p != "." && p != "/" && p != ""; p = path.Dir(p) {
	207	+	if links[p] {
	208	+	return true
	209	+	}
	210	+	}
	211	+	return false
	212	+	}
	213	+
	214	+	// openDirs gives every directory the mode 0755, whatever the umask of the
	215	+	// runner was when it created them: the sandboxes read the tree as another user.
	216	+	func openDirs(dest string) error {
	217	+	return filepath.WalkDir(dest, func(p string, d fs.DirEntry, err error) error {
	218	+	if err != nil \|\| !d.IsDir() {
	219	+	return err
	220	+	}
	221	+	return os.Chmod(p, 0o755)
	222	+	})
	223	+	}

A internal/externals/manager.go +401 -0

@@ -0,0 +1,401 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	// Package externals provides the tools that jobs expect to find outside the
	4	+	// image, today the Node.js builds of actions/runner (node20, node24 and the
	5	+	// alpine variants for job containers), so that a runner upgrade or a new Node.js
	6	+	// version never needs an image rebuild.
	7	+	//
	8	+	// The host downloads the pinned builds (table.go) into
	9	+	// <state dir>/externals/<name>-<version>-<arch>/ at start, verifying the
	10	+	// SHA-256 of each archive before it is unpacked, and the sandbox drivers attach
	11	+	// them read-only at /home/runner/externals/<name> (GitHub's layout:
	12	+	// externals/node24/bin/node). Only the pinned versions are kept.
	13	+	//
	14	+	// A tool that cannot be provided (offline, checksum mismatch, no build for this
	15	+	// architecture, checksum not recorded) never stops the runner: the sandboxes get
	16	+	// a directory with the file UNAVAILABLE and a bin/node stub that print why, and
	17	+	// a job that needs the tool fails with that message. Failed downloads are retried in the
	18	+	// background.
	19	+	package externals
	20	+
	21	+	import (
	22	+	"context"
	23	+	"crypto/sha256"
	24	+	"encoding/hex"
	25	+	"errors"
	26	+	"fmt"
	27	+	"io"
	28	+	"net/http"
	29	+	"net/url"
	30	+	"os"
	31	+	"path/filepath"
	32	+	"strings"
	33	+	"sync"
	34	+	"time"
	35	+
	36	+	"github.com/bnema/zerowrap"
	37	+
	38	+	"bnema.dev/ermite-runner/internal/sandbox"
	39	+	)
	40	+
	41	+	// maxDownload bounds one archive (the largest build is about 60 MiB).
	42	+	const maxDownload = 512 << 20
	43	+
	44	+	// Manager downloads, verifies and keeps the externals of one architecture.
	45	+	type Manager struct {
	46	+	// Dir is <state dir>/externals.
	47	+	Dir string
	48	+	// Arch is GitHub's architecture name of the host (x64, arm64).
	49	+	Arch string
	50	+	// Table is the set of known artifacts (Pinned in production).
	51	+	Table []Artifact
	52	+	// Client downloads the archives; nil means a client with a 15 minute limit
	53	+	// that follows redirects to https locations only.
	54	+	Client *http.Client
	55	+	// RetryEvery is how long a failed download waits before the next attempt
	56	+	// (default 5 minutes).
	57	+	RetryEvery time.Duration
	58	+
	59	+	mu sync.Mutex
	60	+	state map[string]entry // by artifact name
	61	+	bg context.Context // the context of Start, for retries
	62	+	lastTry time.Time
	63	+	syncing bool
	64	+	}
	65	+
	66	+	type entry struct {
	67	+	ext sandbox.External
	68	+	retry bool // a failure worth trying again (network), not a verdict
	69	+	}
	70	+
	71	+	// New returns a Manager for the host architecture; ok is false when the runner
	72	+	// has no builds for it.
	73	+	func New(stateDir string) (m *Manager, ok bool) {
	74	+	arch := HostArch()
	75	+	return &Manager{Dir: filepath.Join(stateDir, "externals"), Arch: arch, Table: Pinned}, arch != ""
	76	+	}
	77	+
	78	+	// Start runs Sync in the background and waits for it at most wait, so that a
	79	+	// runner with a working network normally takes its first job with everything in
	80	+	// place, and one without never waits long. When the wait ends the download goes
	81	+	// on; jobs that start meanwhile see the tools that are not done yet as
	82	+	// unavailable ("still being downloaded"). Retries of failed downloads use ctx
	83	+	// too: they stop with it.
	84	+	func (m *Manager) Start(ctx context.Context, wait time.Duration) {
	85	+	log := zerowrap.FromCtx(ctx)
	86	+	m.mu.Lock()
	87	+	m.bg = ctx
	88	+	m.mu.Unlock()
	89	+	done := make(chan struct{})
	90	+	go func() {
	91	+	defer close(done)
	92	+	if err := m.Sync(ctx); err != nil && ctx.Err() == nil {
	93	+	log.Warn().Err(err).Msg("some external tools are not available: jobs that need them fail with the reason; the download is retried")
	94	+	}
	95	+	}()
	96	+	select {
	97	+	case <-done:
	98	+	case <-time.After(wait):
	99	+	log.Warn().Dur("waited", wait).Msg("external tools are still downloading: the runner takes jobs, those that need them fail until they are ready")
	100	+	case <-ctx.Done():
	101	+	}
	102	+	}
	103	+
	104	+	// Sync makes the externals match the table: it downloads what is missing,
	105	+	// removes everything else from Dir, and records the outcome per tool as it goes.
	106	+	// It returns an error describing the artifacts that could not be provided; the
	107	+	// runner goes on with them marked unavailable.
	108	+	func (m *Manager) Sync(ctx context.Context) error {
	109	+	m.mu.Lock()
	110	+	if m.syncing {
	111	+	m.mu.Unlock()
	112	+	return nil
	113	+	}
	114	+	m.syncing = true
	115	+	if m.state == nil {
	116	+	m.state = map[string]entry{}
	117	+	}
	118	+	m.mu.Unlock()
	119	+	defer func() { m.mu.Lock(); m.syncing = false; m.lastTry = time.Now(); m.mu.Unlock() }()
	120	+
	121	+	log := zerowrap.FromCtx(ctx)
	122	+	var errs []error
	123	+	keep := map[string]bool{}
	124	+	if err := os.MkdirAll(m.Dir, 0o755); err != nil {
	125	+	return err
	126	+	}
	127	+	for _, name := range m.names() {
	128	+	a, ok := m.artifact(name)
	129	+	var e entry
	130	+	switch {
	131	+	case !ok:
	132	+	e = m.unavailable(name, fmt.Sprintf("there is no %s build for linux-%s", name, m.Arch), false)
	133	+	case a.SHA256 == "":
	134	+	e = m.unavailable(name, fmt.Sprintf("%s (Node.js %s) is not available yet on this runner: %s", a.Name, a.Version, a.Pending), false)
	135	+	default:
	136	+	var err error
	137	+	if e, err = m.ensure(ctx, a); err != nil {
	138	+	errs = append(errs, fmt.Errorf("%s %s: %w", a.Name, a.Version, err))
	139	+	log.Warn().Err(err).Str("external", a.Name).Str("version", a.Version).Msg("external tool not available")
	140	+	} else {
	141	+	log.Info().Str("external", a.Name).Str("version", a.Version).Str("arch", a.Arch).Msg("external tool ready")
	142	+	}
	143	+	}
	144	+	m.mu.Lock()
	145	+	m.state[name] = e
	146	+	m.mu.Unlock()
	147	+	keep[filepath.Base(e.ext.Dir)] = true
	148	+	if e.ext.Unavailable {
	149	+	keep[unavailableDir] = true
	150	+	}
	151	+	}
	152	+	m.prune(log, keep)
	153	+	return errors.Join(errs...)
	154	+	}
	155	+
	156	+	// Externals implements sandbox.ExternalsSource. A tool that has no outcome yet
	157	+	// is reported as being downloaded. When a download failed for a reason that may
	158	+	// pass, a retry is started in the background (at most one per RetryEvery).
	159	+	func (m *Manager) Externals() []sandbox.External {
	160	+	m.mu.Lock()
	161	+	defer m.mu.Unlock()
	162	+	var out []sandbox.External
	163	+	retry := false
	164	+	for _, n := range m.names() {
	165	+	e, ok := m.state[n]
	166	+	if !ok {
	167	+	e = m.unavailable(n, n+" is still being downloaded on this runner: run the job again in a few minutes", false)
	168	+	}
	169	+	retry = retry \|\| e.retry
	170	+	out = append(out, e.ext)
	171	+	}
	172	+	every := m.RetryEvery
	173	+	if every == 0 {
	174	+	every = 5 * time.Minute
	175	+	}
	176	+	if retry && !m.syncing && m.bg != nil && m.bg.Err() == nil && time.Since(m.lastTry) >= every {
	177	+	m.lastTry = time.Now() // no second attempt before the next interval
	178	+	go func(ctx context.Context) { _ = m.Sync(ctx) }(m.bg)
	179	+	}
	180	+	return out
	181	+	}
	182	+
	183	+	// names is every artifact name in the table, in table order, once.
	184	+	func (m *Manager) names() []string {
	185	+	seen := map[string]bool{}
	186	+	var out []string
	187	+	for _, a := range m.Table {
	188	+	if !seen[a.Name] {
	189	+	seen[a.Name] = true
	190	+	out = append(out, a.Name)
	191	+	}
	192	+	}
	193	+	return out
	194	+	}
	195	+
	196	+	func (m *Manager) artifact(name string) (Artifact, bool) {
	197	+	for _, a := range m.Table {
	198	+	if a.Name == name && a.Arch == m.Arch {
	199	+	return a, true
	200	+	}
	201	+	}
	202	+	return Artifact{}, false
	203	+	}
	204	+
	205	+	const unavailableDir = ".unavailable"
	206	+
	207	+	// unavailable builds the entry of a tool that is missing: a directory with the
	208	+	// reason in ExternalUnavailableFile and a bin/node stub that prints it. Its name carries a hash of the reason, so
	209	+	// that the ID (and so a VM's disk) changes only when the reason does.
	210	+	func (m *Manager) unavailable(name, reason string, retry bool) entry {
	211	+	sum := sha256.Sum256([]byte(name + "\x00" + reason))
	212	+	id := "unavailable-" + hex.EncodeToString(sum[:8])
	213	+	dir := filepath.Join(m.Dir, unavailableDir, name+"-"+id)
	214	+	e := entry{retry: retry, ext: sandbox.External{Name: name, ID: id, Dir: dir, Unavailable: true}}
	215	+	if err := os.MkdirAll(filepath.Join(dir, "bin"), 0o755); err != nil {
	216	+	return e // still marked unavailable; the sandbox then has an empty directory
	217	+	}
	218	+	// Modes are set explicitly: the umask of the runner must not hide the files
	219	+	// from the user of the sandbox.
	220	+	for _, d := range []string{filepath.Dir(dir), dir, filepath.Join(dir, "bin")} {
	221	+	_ = os.Chmod(d, 0o755)
	222	+	}
	223	+	write := func(name, content string, mode os.FileMode) {
	224	+	p := filepath.Join(dir, name)
	225	+	if os.WriteFile(p, []byte(content), mode) == nil {
	226	+	_ = os.Chmod(p, mode)
	227	+	}
	228	+	}
	229	+	write(sandbox.ExternalUnavailableFile, reason+"\n", 0o644)
	230	+	// A stub where the executable would be, so that a job which runs it anyway
	231	+	// (a node action, a run step) fails with the reason instead of "No such file
	232	+	// or directory". The reason goes through a quoted here-document: it is data.
	233	+	stub := "#!/bin/sh\ncat >&2 <<'ERMITE_UNAVAILABLE'\n" + strings.ReplaceAll(reason, "\nERMITE_UNAVAILABLE", "") + "\nERMITE_UNAVAILABLE\nexit 1\n"
	234	+	write("bin/node", stub, 0o755)
	235	+	return e
	236	+	}
	237	+
	238	+	// ensure returns the ready entry of a, downloading it when needed. On failure it
	239	+	// returns the unavailable entry together with the error.
	240	+	func (m *Manager) ensure(ctx context.Context, a Artifact) (entry, error) {
	241	+	final := filepath.Join(m.Dir, a.dirName())
	242	+	if isReady(final) {
	243	+	return readyEntry(a, final), nil
	244	+	}
	245	+	err := m.fetch(ctx, a, final)
	246	+	if err != nil {
	247	+	reason := fmt.Sprintf("%s (Node.js %s) is not available on this runner: %v", a.Name, a.Version, err)
	248	+	var v *verdictError
	249	+	return m.unavailable(a.Name, reason, !errors.As(err, &v)), err
	250	+	}
	251	+	return readyEntry(a, final), nil
	252	+	}
	253	+
	254	+	func readyEntry(a Artifact, dir string) entry {
	255	+	return entry{ext: sandbox.External{Name: a.Name, ID: a.dirName() + "-" + a.SHA256[:16], Dir: dir}}
	256	+	}
	257	+
	258	+	// isReady reports whether dir holds a complete tool: a regular executable
	259	+	// bin/node. Directories only appear under their final name by atomic rename
	260	+	// after a verified extraction.
	261	+	func isReady(dir string) bool {
	262	+	st, err := os.Stat(filepath.Join(dir, "bin", "node"))
	263	+	return err == nil && st.Mode().IsRegular() && st.Mode().Perm()&0o111 != 0
	264	+	}
	265	+
	266	+	// verdictError is a failure that another attempt will not change (the checksum
	267	+	// differs, the archive is unsafe).
	268	+	type verdictError struct{ error }
	269	+
	270	+	func (e *verdictError) Unwrap() error { return e.error }
	271	+
	272	+	// fetch downloads a, verifies its checksum, unpacks it next to final and
	273	+	// renames it into place.
	274	+	func (m *Manager) fetch(ctx context.Context, a Artifact, final string) error {
	275	+	u, err := url.Parse(a.URL)
	276	+	if err != nil \|\| u.Scheme != "https" {
	277	+	return &verdictError{fmt.Errorf("%q is not an https URL", a.URL)}
	278	+	}
	279	+	tmp, err := os.MkdirTemp(m.Dir, ".tmp-")
	280	+	if err != nil {
	281	+	return err
	282	+	}
	283	+	defer func() { _ = os.RemoveAll(tmp) }()
	284	+
	285	+	archive := filepath.Join(tmp, "archive.tar.gz")
	286	+	if err := m.download(ctx, a, archive); err != nil {
	287	+	return err
	288	+	}
	289	+	tree := filepath.Join(tmp, "tree")
	290	+	if err := os.Mkdir(tree, 0o755); err != nil {
	291	+	return err
	292	+	}
	293	+	if err := extractTarGz(archive, tree); err != nil {
	294	+	return &verdictError{fmt.Errorf("unpack: %w", err)}
	295	+	}
	296	+	if !isReady(tree) {
	297	+	return &verdictError{errors.New("the archive has no executable bin/node")}
	298	+	}
	299	+	if err := os.RemoveAll(final); err != nil { // an incomplete directory of an older run
	300	+	return err
	301	+	}
	302	+	return os.Rename(tree, final)
	303	+	}
	304	+
	305	+	// download fetches a.URL to dest and verifies its SHA-256 as it arrives.
	306	+	func (m *Manager) download(ctx context.Context, a Artifact, dest string) error {
	307	+	client := m.Client
	308	+	if client == nil {
	309	+	client = newClient()
	310	+	}
	311	+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, a.URL, nil)
	312	+	if err != nil {
	313	+	return err
	314	+	}
	315	+	resp, err := client.Do(req)
	316	+	if err != nil {
	317	+	return fmt.Errorf("download %s: %w", a.URL, err)
	318	+	}
	319	+	defer func() { _ = resp.Body.Close() }()
	320	+	if resp.StatusCode != http.StatusOK {
	321	+	return fmt.Errorf("download %s: %s", a.URL, resp.Status)
	322	+	}
	323	+	f, err := os.OpenFile(dest, os.O_WRONLY\|os.O_CREATE\|os.O_EXCL, 0o600)
	324	+	if err != nil {
	325	+	return err
	326	+	}
	327	+	h := sha256.New()
	328	+	n, err := io.Copy(io.MultiWriter(f, h), io.LimitReader(resp.Body, maxDownload+1))
	329	+	if cerr := f.Close(); err == nil {
	330	+	err = cerr
	331	+	}
	332	+	if err != nil {
	333	+	return fmt.Errorf("download %s: %w", a.URL, err)
	334	+	}
	335	+	if n > maxDownload {
	336	+	return &verdictError{fmt.Errorf("download %s: more than %d bytes", a.URL, int64(maxDownload))}
	337	+	}
	338	+	if err := VerifySum(hex.EncodeToString(h.Sum(nil)), a.SHA256); err != nil {
	339	+	return &verdictError{fmt.Errorf("download %s: %w", a.URL, err)}
	340	+	}
	341	+	return nil
	342	+	}
	343	+
	344	+	// newClient is the default downloader: a time limit, and redirects only to
	345	+	// https locations (nodejs.org does not redirect; GitHub releases do).
	346	+	func newClient() *http.Client {
	347	+	return &http.Client{
	348	+	Timeout: 15 * time.Minute,
	349	+	CheckRedirect: func(req http.Request, via []http.Request) error {
	350	+	if req.URL.Scheme != "https" {
	351	+	return fmt.Errorf("redirect to a non-https URL %s", req.URL.Redacted())
	352	+	}
	353	+	if len(via) >= 5 {
	354	+	return errors.New("too many redirects")
	355	+	}
	356	+	return nil
	357	+	},
	358	+	}
	359	+	}
	360	+
	361	+	// VerifySum compares two hex SHA-256 values.
	362	+	func VerifySum(got, want string) error {
	363	+	if !strings.EqualFold(got, want) {
	364	+	return fmt.Errorf("SHA-256 mismatch: got %s, want %s", got, want)
	365	+	}
	366	+	return nil
	367	+	}
	368	+
	369	+	// prune removes everything in Dir that is not pinned: other versions, other
	370	+	// architectures, leftovers of an interrupted download. The reasons of
	371	+	// unavailable tools (a few bytes each) may be bind-mounted into a running
	372	+	// container, so those are only removed once they are an hour old.
	373	+	func (m *Manager) prune(log zerowrap.Logger, keep map[string]bool) {
	374	+	ents, err := os.ReadDir(m.Dir)
	375	+	if err != nil {
	376	+	return
	377	+	}
	378	+	for _, e := range ents {
	379	+	switch name := e.Name(); {
	380	+	case keep[name]:
	381	+	case name == unavailableDir:
	382	+	m.pruneUnavailable(keep)
	383	+	default:
	384	+	if err := os.RemoveAll(filepath.Join(m.Dir, name)); err != nil {
	385	+	log.Warn().Err(err).Str("path", name).Msg("removing an unpinned external tool")
	386	+	}
	387	+	}
	388	+	}
	389	+	}
	390	+
	391	+	func (m *Manager) pruneUnavailable(keep map[string]bool) {
	392	+	dir := filepath.Join(m.Dir, unavailableDir)
	393	+	ents, _ := os.ReadDir(dir)
	394	+	for _, e := range ents {
	395	+	fi, err := e.Info()
	396	+	if keep[e.Name()] \|\| err != nil \|\| time.Since(fi.ModTime()) < time.Hour {
	397	+	continue
	398	+	}
	399	+	_ = os.RemoveAll(filepath.Join(dir, e.Name()))
	400	+	}
	401	+	}

@@ -0,0 +1,401 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	// Package externals provides the tools that jobs expect to find outside the
	4	+	// image, today the Node.js builds of actions/runner (node20, node24 and the
	5	+	// alpine variants for job containers), so that a runner upgrade or a new Node.js
	6	+	// version never needs an image rebuild.
	7	+	//
	8	+	// The host downloads the pinned builds (table.go) into
	9	+	// <state dir>/externals/<name>-<version>-<arch>/ at start, verifying the
	10	+	// SHA-256 of each archive before it is unpacked, and the sandbox drivers attach
	11	+	// them read-only at /home/runner/externals/<name> (GitHub's layout:
	12	+	// externals/node24/bin/node). Only the pinned versions are kept.
	13	+	//
	14	+	// A tool that cannot be provided (offline, checksum mismatch, no build for this
	15	+	// architecture, checksum not recorded) never stops the runner: the sandboxes get
	16	+	// a directory with the file UNAVAILABLE and a bin/node stub that print why, and
	17	+	// a job that needs the tool fails with that message. Failed downloads are retried in the
	18	+	// background.
	19	+	package externals
	20	+
	21	+	import (
	22	+	"context"
	23	+	"crypto/sha256"
	24	+	"encoding/hex"
	25	+	"errors"
	26	+	"fmt"
	27	+	"io"
	28	+	"net/http"
	29	+	"net/url"
	30	+	"os"
	31	+	"path/filepath"
	32	+	"strings"
	33	+	"sync"
	34	+	"time"
	35	+
	36	+	"github.com/bnema/zerowrap"
	37	+
	38	+	"bnema.dev/ermite-runner/internal/sandbox"
	39	+	)
	40	+
	41	+	// maxDownload bounds one archive (the largest build is about 60 MiB).
	42	+	const maxDownload = 512 << 20
	43	+
	44	+	// Manager downloads, verifies and keeps the externals of one architecture.
	45	+	type Manager struct {
	46	+	// Dir is <state dir>/externals.
	47	+	Dir string
	48	+	// Arch is GitHub's architecture name of the host (x64, arm64).
	49	+	Arch string
	50	+	// Table is the set of known artifacts (Pinned in production).
	51	+	Table []Artifact
	52	+	// Client downloads the archives; nil means a client with a 15 minute limit
	53	+	// that follows redirects to https locations only.
	54	+	Client *http.Client
	55	+	// RetryEvery is how long a failed download waits before the next attempt
	56	+	// (default 5 minutes).
	57	+	RetryEvery time.Duration
	58	+
	59	+	mu sync.Mutex
	60	+	state map[string]entry // by artifact name
	61	+	bg context.Context // the context of Start, for retries
	62	+	lastTry time.Time
	63	+	syncing bool
	64	+	}
	65	+
	66	+	type entry struct {
	67	+	ext sandbox.External
	68	+	retry bool // a failure worth trying again (network), not a verdict
	69	+	}
	70	+
	71	+	// New returns a Manager for the host architecture; ok is false when the runner
	72	+	// has no builds for it.
	73	+	func New(stateDir string) (m *Manager, ok bool) {
	74	+	arch := HostArch()
	75	+	return &Manager{Dir: filepath.Join(stateDir, "externals"), Arch: arch, Table: Pinned}, arch != ""
	76	+	}
	77	+
	78	+	// Start runs Sync in the background and waits for it at most wait, so that a
	79	+	// runner with a working network normally takes its first job with everything in
	80	+	// place, and one without never waits long. When the wait ends the download goes
	81	+	// on; jobs that start meanwhile see the tools that are not done yet as
	82	+	// unavailable ("still being downloaded"). Retries of failed downloads use ctx
	83	+	// too: they stop with it.
	84	+	func (m *Manager) Start(ctx context.Context, wait time.Duration) {
	85	+	log := zerowrap.FromCtx(ctx)
	86	+	m.mu.Lock()
	87	+	m.bg = ctx
	88	+	m.mu.Unlock()
	89	+	done := make(chan struct{})
	90	+	go func() {
	91	+	defer close(done)
	92	+	if err := m.Sync(ctx); err != nil && ctx.Err() == nil {
	93	+	log.Warn().Err(err).Msg("some external tools are not available: jobs that need them fail with the reason; the download is retried")
	94	+	}
	95	+	}()
	96	+	select {
	97	+	case <-done:
	98	+	case <-time.After(wait):
	99	+	log.Warn().Dur("waited", wait).Msg("external tools are still downloading: the runner takes jobs, those that need them fail until they are ready")
	100	+	case <-ctx.Done():
	101	+	}
	102	+	}
	103	+
	104	+	// Sync makes the externals match the table: it downloads what is missing,
	105	+	// removes everything else from Dir, and records the outcome per tool as it goes.
	106	+	// It returns an error describing the artifacts that could not be provided; the
	107	+	// runner goes on with them marked unavailable.
	108	+	func (m *Manager) Sync(ctx context.Context) error {
	109	+	m.mu.Lock()
	110	+	if m.syncing {
	111	+	m.mu.Unlock()
	112	+	return nil
	113	+	}
	114	+	m.syncing = true
	115	+	if m.state == nil {
	116	+	m.state = map[string]entry{}
	117	+	}
	118	+	m.mu.Unlock()
	119	+	defer func() { m.mu.Lock(); m.syncing = false; m.lastTry = time.Now(); m.mu.Unlock() }()
	120	+
	121	+	log := zerowrap.FromCtx(ctx)
	122	+	var errs []error
	123	+	keep := map[string]bool{}
	124	+	if err := os.MkdirAll(m.Dir, 0o755); err != nil {
	125	+	return err
	126	+	}
	127	+	for _, name := range m.names() {
	128	+	a, ok := m.artifact(name)
	129	+	var e entry
	130	+	switch {
	131	+	case !ok:
	132	+	e = m.unavailable(name, fmt.Sprintf("there is no %s build for linux-%s", name, m.Arch), false)
	133	+	case a.SHA256 == "":
	134	+	e = m.unavailable(name, fmt.Sprintf("%s (Node.js %s) is not available yet on this runner: %s", a.Name, a.Version, a.Pending), false)
	135	+	default:
	136	+	var err error
	137	+	if e, err = m.ensure(ctx, a); err != nil {
	138	+	errs = append(errs, fmt.Errorf("%s %s: %w", a.Name, a.Version, err))
	139	+	log.Warn().Err(err).Str("external", a.Name).Str("version", a.Version).Msg("external tool not available")
	140	+	} else {
	141	+	log.Info().Str("external", a.Name).Str("version", a.Version).Str("arch", a.Arch).Msg("external tool ready")
	142	+	}
	143	+	}
	144	+	m.mu.Lock()
	145	+	m.state[name] = e
	146	+	m.mu.Unlock()
	147	+	keep[filepath.Base(e.ext.Dir)] = true
	148	+	if e.ext.Unavailable {
	149	+	keep[unavailableDir] = true
	150	+	}
	151	+	}
	152	+	m.prune(log, keep)
	153	+	return errors.Join(errs...)
	154	+	}
	155	+
	156	+	// Externals implements sandbox.ExternalsSource. A tool that has no outcome yet
	157	+	// is reported as being downloaded. When a download failed for a reason that may
	158	+	// pass, a retry is started in the background (at most one per RetryEvery).
	159	+	func (m *Manager) Externals() []sandbox.External {
	160	+	m.mu.Lock()
	161	+	defer m.mu.Unlock()
	162	+	var out []sandbox.External
	163	+	retry := false
	164	+	for _, n := range m.names() {
	165	+	e, ok := m.state[n]
	166	+	if !ok {
	167	+	e = m.unavailable(n, n+" is still being downloaded on this runner: run the job again in a few minutes", false)
	168	+	}
	169	+	retry = retry \|\| e.retry
	170	+	out = append(out, e.ext)
	171	+	}
	172	+	every := m.RetryEvery
	173	+	if every == 0 {
	174	+	every = 5 * time.Minute
	175	+	}
	176	+	if retry && !m.syncing && m.bg != nil && m.bg.Err() == nil && time.Since(m.lastTry) >= every {
	177	+	m.lastTry = time.Now() // no second attempt before the next interval
	178	+	go func(ctx context.Context) { _ = m.Sync(ctx) }(m.bg)
	179	+	}
	180	+	return out
	181	+	}
	182	+
	183	+	// names is every artifact name in the table, in table order, once.
	184	+	func (m *Manager) names() []string {
	185	+	seen := map[string]bool{}
	186	+	var out []string
	187	+	for _, a := range m.Table {
	188	+	if !seen[a.Name] {
	189	+	seen[a.Name] = true
	190	+	out = append(out, a.Name)
	191	+	}
	192	+	}
	193	+	return out
	194	+	}
	195	+
	196	+	func (m *Manager) artifact(name string) (Artifact, bool) {
	197	+	for _, a := range m.Table {
	198	+	if a.Name == name && a.Arch == m.Arch {
	199	+	return a, true
	200	+	}
	201	+	}
	202	+	return Artifact{}, false
	203	+	}
	204	+
	205	+	const unavailableDir = ".unavailable"
	206	+
	207	+	// unavailable builds the entry of a tool that is missing: a directory with the
	208	+	// reason in ExternalUnavailableFile and a bin/node stub that prints it. Its name carries a hash of the reason, so
	209	+	// that the ID (and so a VM's disk) changes only when the reason does.
	210	+	func (m *Manager) unavailable(name, reason string, retry bool) entry {
	211	+	sum := sha256.Sum256([]byte(name + "\x00" + reason))
	212	+	id := "unavailable-" + hex.EncodeToString(sum[:8])
	213	+	dir := filepath.Join(m.Dir, unavailableDir, name+"-"+id)
	214	+	e := entry{retry: retry, ext: sandbox.External{Name: name, ID: id, Dir: dir, Unavailable: true}}
	215	+	if err := os.MkdirAll(filepath.Join(dir, "bin"), 0o755); err != nil {
	216	+	return e // still marked unavailable; the sandbox then has an empty directory
	217	+	}
	218	+	// Modes are set explicitly: the umask of the runner must not hide the files
	219	+	// from the user of the sandbox.
	220	+	for _, d := range []string{filepath.Dir(dir), dir, filepath.Join(dir, "bin")} {
	221	+	_ = os.Chmod(d, 0o755)
	222	+	}
	223	+	write := func(name, content string, mode os.FileMode) {
	224	+	p := filepath.Join(dir, name)
	225	+	if os.WriteFile(p, []byte(content), mode) == nil {
	226	+	_ = os.Chmod(p, mode)
	227	+	}
	228	+	}
	229	+	write(sandbox.ExternalUnavailableFile, reason+"\n", 0o644)
	230	+	// A stub where the executable would be, so that a job which runs it anyway
	231	+	// (a node action, a run step) fails with the reason instead of "No such file
	232	+	// or directory". The reason goes through a quoted here-document: it is data.
	233	+	stub := "#!/bin/sh\ncat >&2 <<'ERMITE_UNAVAILABLE'\n" + strings.ReplaceAll(reason, "\nERMITE_UNAVAILABLE", "") + "\nERMITE_UNAVAILABLE\nexit 1\n"
	234	+	write("bin/node", stub, 0o755)
	235	+	return e
	236	+	}
	237	+
	238	+	// ensure returns the ready entry of a, downloading it when needed. On failure it
	239	+	// returns the unavailable entry together with the error.
	240	+	func (m *Manager) ensure(ctx context.Context, a Artifact) (entry, error) {
	241	+	final := filepath.Join(m.Dir, a.dirName())
	242	+	if isReady(final) {
	243	+	return readyEntry(a, final), nil
	244	+	}
	245	+	err := m.fetch(ctx, a, final)
	246	+	if err != nil {
	247	+	reason := fmt.Sprintf("%s (Node.js %s) is not available on this runner: %v", a.Name, a.Version, err)
	248	+	var v *verdictError
	249	+	return m.unavailable(a.Name, reason, !errors.As(err, &v)), err
	250	+	}
	251	+	return readyEntry(a, final), nil
	252	+	}
	253	+
	254	+	func readyEntry(a Artifact, dir string) entry {
	255	+	return entry{ext: sandbox.External{Name: a.Name, ID: a.dirName() + "-" + a.SHA256[:16], Dir: dir}}
	256	+	}
	257	+
	258	+	// isReady reports whether dir holds a complete tool: a regular executable
	259	+	// bin/node. Directories only appear under their final name by atomic rename
	260	+	// after a verified extraction.
	261	+	func isReady(dir string) bool {
	262	+	st, err := os.Stat(filepath.Join(dir, "bin", "node"))
	263	+	return err == nil && st.Mode().IsRegular() && st.Mode().Perm()&0o111 != 0
	264	+	}
	265	+
	266	+	// verdictError is a failure that another attempt will not change (the checksum
	267	+	// differs, the archive is unsafe).
	268	+	type verdictError struct{ error }
	269	+
	270	+	func (e *verdictError) Unwrap() error { return e.error }
	271	+
	272	+	// fetch downloads a, verifies its checksum, unpacks it next to final and
	273	+	// renames it into place.
	274	+	func (m *Manager) fetch(ctx context.Context, a Artifact, final string) error {
	275	+	u, err := url.Parse(a.URL)
	276	+	if err != nil \|\| u.Scheme != "https" {
	277	+	return &verdictError{fmt.Errorf("%q is not an https URL", a.URL)}
	278	+	}
	279	+	tmp, err := os.MkdirTemp(m.Dir, ".tmp-")
	280	+	if err != nil {
	281	+	return err
	282	+	}
	283	+	defer func() { _ = os.RemoveAll(tmp) }()
	284	+
	285	+	archive := filepath.Join(tmp, "archive.tar.gz")
	286	+	if err := m.download(ctx, a, archive); err != nil {
	287	+	return err
	288	+	}
	289	+	tree := filepath.Join(tmp, "tree")
	290	+	if err := os.Mkdir(tree, 0o755); err != nil {
	291	+	return err
	292	+	}
	293	+	if err := extractTarGz(archive, tree); err != nil {
	294	+	return &verdictError{fmt.Errorf("unpack: %w", err)}
	295	+	}
	296	+	if !isReady(tree) {
	297	+	return &verdictError{errors.New("the archive has no executable bin/node")}
	298	+	}
	299	+	if err := os.RemoveAll(final); err != nil { // an incomplete directory of an older run
	300	+	return err
	301	+	}
	302	+	return os.Rename(tree, final)
	303	+	}
	304	+
	305	+	// download fetches a.URL to dest and verifies its SHA-256 as it arrives.
	306	+	func (m *Manager) download(ctx context.Context, a Artifact, dest string) error {
	307	+	client := m.Client
	308	+	if client == nil {
	309	+	client = newClient()
	310	+	}
	311	+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, a.URL, nil)
	312	+	if err != nil {
	313	+	return err
	314	+	}
	315	+	resp, err := client.Do(req)
	316	+	if err != nil {
	317	+	return fmt.Errorf("download %s: %w", a.URL, err)
	318	+	}
	319	+	defer func() { _ = resp.Body.Close() }()
	320	+	if resp.StatusCode != http.StatusOK {
	321	+	return fmt.Errorf("download %s: %s", a.URL, resp.Status)
	322	+	}
	323	+	f, err := os.OpenFile(dest, os.O_WRONLY\|os.O_CREATE\|os.O_EXCL, 0o600)
	324	+	if err != nil {
	325	+	return err
	326	+	}
	327	+	h := sha256.New()
	328	+	n, err := io.Copy(io.MultiWriter(f, h), io.LimitReader(resp.Body, maxDownload+1))
	329	+	if cerr := f.Close(); err == nil {
	330	+	err = cerr
	331	+	}
	332	+	if err != nil {
	333	+	return fmt.Errorf("download %s: %w", a.URL, err)
	334	+	}
	335	+	if n > maxDownload {
	336	+	return &verdictError{fmt.Errorf("download %s: more than %d bytes", a.URL, int64(maxDownload))}
	337	+	}
	338	+	if err := VerifySum(hex.EncodeToString(h.Sum(nil)), a.SHA256); err != nil {
	339	+	return &verdictError{fmt.Errorf("download %s: %w", a.URL, err)}
	340	+	}
	341	+	return nil
	342	+	}
	343	+
	344	+	// newClient is the default downloader: a time limit, and redirects only to
	345	+	// https locations (nodejs.org does not redirect; GitHub releases do).
	346	+	func newClient() *http.Client {
	347	+	return &http.Client{
	348	+	Timeout: 15 * time.Minute,
	349	+	CheckRedirect: func(req http.Request, via []http.Request) error {
	350	+	if req.URL.Scheme != "https" {
	351	+	return fmt.Errorf("redirect to a non-https URL %s", req.URL.Redacted())
	352	+	}
	353	+	if len(via) >= 5 {
	354	+	return errors.New("too many redirects")
	355	+	}
	356	+	return nil
	357	+	},
	358	+	}
	359	+	}
	360	+
	361	+	// VerifySum compares two hex SHA-256 values.
	362	+	func VerifySum(got, want string) error {
	363	+	if !strings.EqualFold(got, want) {
	364	+	return fmt.Errorf("SHA-256 mismatch: got %s, want %s", got, want)
	365	+	}
	366	+	return nil
	367	+	}
	368	+
	369	+	// prune removes everything in Dir that is not pinned: other versions, other
	370	+	// architectures, leftovers of an interrupted download. The reasons of
	371	+	// unavailable tools (a few bytes each) may be bind-mounted into a running
	372	+	// container, so those are only removed once they are an hour old.
	373	+	func (m *Manager) prune(log zerowrap.Logger, keep map[string]bool) {
	374	+	ents, err := os.ReadDir(m.Dir)
	375	+	if err != nil {
	376	+	return
	377	+	}
	378	+	for _, e := range ents {
	379	+	switch name := e.Name(); {
	380	+	case keep[name]:
	381	+	case name == unavailableDir:
	382	+	m.pruneUnavailable(keep)
	383	+	default:
	384	+	if err := os.RemoveAll(filepath.Join(m.Dir, name)); err != nil {
	385	+	log.Warn().Err(err).Str("path", name).Msg("removing an unpinned external tool")
	386	+	}
	387	+	}
	388	+	}
	389	+	}
	390	+
	391	+	func (m *Manager) pruneUnavailable(keep map[string]bool) {
	392	+	dir := filepath.Join(m.Dir, unavailableDir)
	393	+	ents, _ := os.ReadDir(dir)
	394	+	for _, e := range ents {
	395	+	fi, err := e.Info()
	396	+	if keep[e.Name()] \|\| err != nil \|\| time.Since(fi.ModTime()) < time.Hour {
	397	+	continue
	398	+	}
	399	+	_ = os.RemoveAll(filepath.Join(dir, e.Name()))
	400	+	}
	401	+	}

A internal/externals/table.go +96 -0

@@ -0,0 +1,96 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package externals
	4	+
	5	+	import "runtime"
	6	+
	7	+	// Artifact is one pinned external tool: where it comes from and the SHA-256 of
	8	+	// the archive. The pins mirror actions/runner's Misc/externals.sh (node20
	9	+	// 20.20.2, node24 24.21.0), so a workflow sees the Node.js it would see on a
	10	+	// GitHub-hosted runner. The checksums of the nodejs.org builds come from the
	11	+	// official SHASUMS256.txt of each release, and the archives are the .tar.gz
	12	+	// ones (the Go standard library has no xz).
	13	+	type Artifact struct {
	14	+	// Name is the directory of the GitHub layout, below /home/runner/externals:
	15	+	// node20, node24, node20_alpine, node24_alpine.
	16	+	Name string
	17	+	// Version is the Node.js version without the leading "v".
	18	+	Version string
	19	+	// Arch is GitHub's architecture name: x64 or arm64.
	20	+	Arch string
	21	+	// URL is the https location of the .tar.gz.
	22	+	URL string
	23	+	// SHA256 is the lower-case hex SHA-256 of the archive. Empty means that the
	24	+	// checksum has not been recorded yet: the artifact is then reported as
	25	+	// unavailable (Pending says why) and nothing is downloaded.
	26	+	SHA256 string
	27	+	// Pending is the explanation shown to a job that needs an artifact without
	28	+	// a recorded checksum.
	29	+	Pending string
	30	+	}
	31	+
	32	+	// dirName is the directory of the artifact below the externals directory.
	33	+	func (a Artifact) dirName() string { return a.Name + "-" + a.Version + "-" + a.Arch }
	34	+
	35	+	// The versions of actions/runner's externals.sh.
	36	+	const (
	37	+	node20Version = "20.20.2"
	38	+	node24Version = "24.21.0"
	39	+	)
	40	+
	41	+	const (
	42	+	nodeDist = "https://nodejs.org/dist/"
	43	+	alpineDist = "https://github.com/actions/alpine_nodejs/releases/download/"
	44	+	)
	45	+
	46	+	func nodejs(name, version, arch, sum string) Artifact {
	47	+	return Artifact{
	48	+	Name: name, Version: version, Arch: arch, SHA256: sum,
	49	+	URL: nodeDist + "v" + version + "/node-v" + version + "-linux-" + arch + ".tar.gz",
	50	+	}
	51	+	}
	52	+
	53	+	func alpine(name, version, sum string) Artifact {
	54	+	a := Artifact{
	55	+	Name: name, Version: version, Arch: "x64", SHA256: sum,
	56	+	URL: alpineDist + "v" + version + "/node-v" + version + "-alpine-x64.tar.gz",
	57	+	}
	58	+	if sum == "" {
	59	+	a.Pending = "the checksum of the actions/alpine_nodejs build has not been recorded in this runner yet"
	60	+	}
	61	+	return a
	62	+	}
	63	+
	64	+	// Pinned is every artifact the runner knows, for all architectures.
	65	+	var Pinned = []Artifact{
	66	+	nodejs("node20", node20Version, "x64", "19e56f0825510207dd904f087fe52faa0a4eb6b2aab5f0ea7a33830d04888b8b"),
	67	+	nodejs("node20", node20Version, "arm64", "47ef73d543ecf6eb19435f6c03a0ac4809b3bf0dd6b26c7c571efc2a6572a74d"),
	68	+	nodejs("node24", node24Version, "x64", "6e1db87ef58b8819e5d5402eff1536491b18edd8eb7bee5ef7897876e88dc5ff"),
	69	+	nodejs("node24", node24Version, "arm64", "724282c3b43aec998aa9527380465b45d229e021b58035f5f4f63095eabfe5d5"),
	70	+	// actions/runner ships the alpine builds for x64 only.
	71	+	alpine("node20_alpine", node20Version, ""),
	72	+	alpine("node24_alpine", node24Version, ""),
	73	+	}
	74	+
	75	+	// HostArch is GitHub's name for the architecture of this host ("" when the
	76	+	// runner has no build for it).
	77	+	func HostArch() string {
	78	+	switch runtime.GOARCH {
	79	+	case "amd64":
	80	+	return "x64"
	81	+	case "arm64":
	82	+	return "arm64"
	83	+	}
	84	+	return ""
	85	+	}
	86	+
	87	+	// ForArch selects the artifacts of one architecture, in table order.
	88	+	func ForArch(all []Artifact, arch string) []Artifact {
	89	+	var out []Artifact
	90	+	for _, a := range all {
	91	+	if a.Arch == arch {
	92	+	out = append(out, a)
	93	+	}
	94	+	}
	95	+	return out
	96	+	}

@@ -0,0 +1,96 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package externals
	4	+
	5	+	import "runtime"
	6	+
	7	+	// Artifact is one pinned external tool: where it comes from and the SHA-256 of
	8	+	// the archive. The pins mirror actions/runner's Misc/externals.sh (node20
	9	+	// 20.20.2, node24 24.21.0), so a workflow sees the Node.js it would see on a
	10	+	// GitHub-hosted runner. The checksums of the nodejs.org builds come from the
	11	+	// official SHASUMS256.txt of each release, and the archives are the .tar.gz
	12	+	// ones (the Go standard library has no xz).
	13	+	type Artifact struct {
	14	+	// Name is the directory of the GitHub layout, below /home/runner/externals:
	15	+	// node20, node24, node20_alpine, node24_alpine.
	16	+	Name string
	17	+	// Version is the Node.js version without the leading "v".
	18	+	Version string
	19	+	// Arch is GitHub's architecture name: x64 or arm64.
	20	+	Arch string
	21	+	// URL is the https location of the .tar.gz.
	22	+	URL string
	23	+	// SHA256 is the lower-case hex SHA-256 of the archive. Empty means that the
	24	+	// checksum has not been recorded yet: the artifact is then reported as
	25	+	// unavailable (Pending says why) and nothing is downloaded.
	26	+	SHA256 string
	27	+	// Pending is the explanation shown to a job that needs an artifact without
	28	+	// a recorded checksum.
	29	+	Pending string
	30	+	}
	31	+
	32	+	// dirName is the directory of the artifact below the externals directory.
	33	+	func (a Artifact) dirName() string { return a.Name + "-" + a.Version + "-" + a.Arch }
	34	+
	35	+	// The versions of actions/runner's externals.sh.
	36	+	const (
	37	+	node20Version = "20.20.2"
	38	+	node24Version = "24.21.0"
	39	+	)
	40	+
	41	+	const (
	42	+	nodeDist = "https://nodejs.org/dist/"
	43	+	alpineDist = "https://github.com/actions/alpine_nodejs/releases/download/"
	44	+	)
	45	+
	46	+	func nodejs(name, version, arch, sum string) Artifact {
	47	+	return Artifact{
	48	+	Name: name, Version: version, Arch: arch, SHA256: sum,
	49	+	URL: nodeDist + "v" + version + "/node-v" + version + "-linux-" + arch + ".tar.gz",
	50	+	}
	51	+	}
	52	+
	53	+	func alpine(name, version, sum string) Artifact {
	54	+	a := Artifact{
	55	+	Name: name, Version: version, Arch: "x64", SHA256: sum,
	56	+	URL: alpineDist + "v" + version + "/node-v" + version + "-alpine-x64.tar.gz",
	57	+	}
	58	+	if sum == "" {
	59	+	a.Pending = "the checksum of the actions/alpine_nodejs build has not been recorded in this runner yet"
	60	+	}
	61	+	return a
	62	+	}
	63	+
	64	+	// Pinned is every artifact the runner knows, for all architectures.
	65	+	var Pinned = []Artifact{
	66	+	nodejs("node20", node20Version, "x64", "19e56f0825510207dd904f087fe52faa0a4eb6b2aab5f0ea7a33830d04888b8b"),
	67	+	nodejs("node20", node20Version, "arm64", "47ef73d543ecf6eb19435f6c03a0ac4809b3bf0dd6b26c7c571efc2a6572a74d"),
	68	+	nodejs("node24", node24Version, "x64", "6e1db87ef58b8819e5d5402eff1536491b18edd8eb7bee5ef7897876e88dc5ff"),
	69	+	nodejs("node24", node24Version, "arm64", "724282c3b43aec998aa9527380465b45d229e021b58035f5f4f63095eabfe5d5"),
	70	+	// actions/runner ships the alpine builds for x64 only.
	71	+	alpine("node20_alpine", node20Version, ""),
	72	+	alpine("node24_alpine", node24Version, ""),
	73	+	}
	74	+
	75	+	// HostArch is GitHub's name for the architecture of this host ("" when the
	76	+	// runner has no build for it).
	77	+	func HostArch() string {
	78	+	switch runtime.GOARCH {
	79	+	case "amd64":
	80	+	return "x64"
	81	+	case "arm64":
	82	+	return "arm64"
	83	+	}
	84	+	return ""
	85	+	}
	86	+
	87	+	// ForArch selects the artifacts of one architecture, in table order.
	88	+	func ForArch(all []Artifact, arch string) []Artifact {
	89	+	var out []Artifact
	90	+	for _, a := range all {
	91	+	if a.Arch == arch {
	92	+	out = append(out, a)
	93	+	}
	94	+	}
	95	+	return out
	96	+	}

M internal/guest/vmsetup.go +13 -0


@@ -18,6 +18,9 @@ import (
18	18		const (
19	19		devBase = "/dev/vdb" // shared read-only base image (ext4)
20	20		devScratch = "/dev/vdc" // per-job scratch disk (ext4)
	21	+	// devExternals is the host's external tools (Node.js), attached last and only
	22	+	// when the command line says ermite.externals=1.
	23	+	devExternals = "/dev/vdd"
21	24		)
22	25
23	26		// KernelArg names of the clock and DNS settings (not part of the contract

@@ -79,6 +82,16 @@ func setupVM(log zerowrap.Logger, args map[string]string) error {
79	82		return err
80	83		}
81	84		}
	85	+	// The host's external tools, read-only, in GitHub's layout. A failure is not
	86	+	// fatal: only the jobs that need Node.js are affected, and they say so.
	87	+	if args[sandbox.CmdlineExternals] == "1" {
	88	+	dst := "/mnt/newroot" + sandbox.GuestExternalsDir
	89	+	if err := os.MkdirAll(dst, 0o755); err != nil {
	90	+	log.Warn().Err(err).Msg("init: externals mountpoint")
	91	+	} else if err := mount(devExternals, dst, "ext4", unix.MS_RDONLY\|unix.MS_NOSUID\|unix.MS_NODEV, ""); err != nil {
	92	+	log.Warn().Err(err).Msg("init: mount the externals disk")
	93	+	}
	94	+	}
82	95		// Scratch and the exec disk stay reachable after the pivot.
83	96		if err := mount("/mnt/scratch", "/mnt/newroot"+sandbox.GuestScratch, "", unix.MS_BIND, ""); err != nil {
84	97		return err


@@ -18,6 +18,9 @@ import (
18	const (	18		const (
19	devBase = "/dev/vdb" // shared read-only base image (ext4)	19		devBase = "/dev/vdb" // shared read-only base image (ext4)
20	devScratch = "/dev/vdc" // per-job scratch disk (ext4)	20		devScratch = "/dev/vdc" // per-job scratch disk (ext4)
		21	+	// devExternals is the host's external tools (Node.js), attached last and only
		22	+	// when the command line says ermite.externals=1.
		23	+	devExternals = "/dev/vdd"
21	)	24		)
22		25
23	// KernelArg names of the clock and DNS settings (not part of the contract	26		// KernelArg names of the clock and DNS settings (not part of the contract

@@ -79,6 +82,16 @@ func setupVM(log zerowrap.Logger, args map[string]string) error {
79	return err	82		return err
80	}	83		}
81	}	84		}
		85	+	// The host's external tools, read-only, in GitHub's layout. A failure is not
		86	+	// fatal: only the jobs that need Node.js are affected, and they say so.
		87	+	if args[sandbox.CmdlineExternals] == "1" {
		88	+	dst := "/mnt/newroot" + sandbox.GuestExternalsDir
		89	+	if err := os.MkdirAll(dst, 0o755); err != nil {
		90	+	log.Warn().Err(err).Msg("init: externals mountpoint")
		91	+	} else if err := mount(devExternals, dst, "ext4", unix.MS_RDONLY\|unix.MS_NOSUID\|unix.MS_NODEV, ""); err != nil {
		92	+	log.Warn().Err(err).Msg("init: mount the externals disk")
		93	+	}
		94	+	}
82	// Scratch and the exec disk stay reachable after the pivot.	95		// Scratch and the exec disk stay reachable after the pivot.
83	if err := mount("/mnt/scratch", "/mnt/newroot"+sandbox.GuestScratch, "", unix.MS_BIND, ""); err != nil {	96		if err := mount("/mnt/scratch", "/mnt/newroot"+sandbox.GuestScratch, "", unix.MS_BIND, ""); err != nil {
84	return err	97		return err

M internal/sandbox/config.go +20 -0


@@ -36,6 +36,9 @@ const (
36	36		EnvJobDiskBW = "ERMITE_RUNNER_JOB_DISK_BANDWIDTH"
37	37		EnvJobDiskIOPS = "ERMITE_RUNNER_JOB_DISK_IOPS"
38	38		EnvJobNetBandwidth = "ERMITE_RUNNER_JOB_NET_BANDWIDTH"
	39	+	// EnvNode=0 turns the Node.js externals off (nothing is downloaded or
	40	+	// attached).
	41	+	EnvNode = "ERMITE_RUNNER_NODE"
39	42		)
40	43
41	44		// Defaults.

@@ -117,6 +120,14 @@ type Config struct {
117	120		GuestForge *GuestForge
118	121		// Rate limits of micro-VMs; zero disables.
119	122		DiskBandwidthMiB, DiskIOPS, NetMbit int
	123	+
	124	+	// NodeExternals is false when ERMITE_RUNNER_NODE=0: the caller then sets no
	125	+	// Externals.
	126	+	NodeExternals bool
	127	+	// Externals, when not nil, is the host's set of external tools (Node.js),
	128	+	// attached read-only to every sandbox at GuestExternalsDir. The caller sets
	129	+	// it before building the drivers, like GuestForge.
	130	+	Externals ExternalsSource
120	131		}
121	132
122	133		// LoadConfig reads the sandbox settings from getenv (os.Getenv in production).

@@ -134,6 +145,8 @@ func LoadConfig(getenv func(string) string) (Config, error) {
134	145		JobPids: DefaultJobPids,
135	146		LockFile: DefaultLockFile,
136	147
	148	+	NodeExternals: true,
	149	+
137	150		DiskBandwidthMiB: DefaultDiskBandwidthMiB,
138	151		DiskIOPS: DefaultDiskIOPS,
139	152		NetMbit: DefaultNetMbit,

@@ -203,6 +216,13 @@ func LoadConfig(getenv func(string) string) (Config, error) {
203	216		default:
204	217		return c, fmt.Errorf("%s: %q is not 0 or 1", EnvGuestForge, getenv(EnvGuestForge))
205	218		}
	219	+	switch strings.ToLower(strings.TrimSpace(getenv(EnvNode))) {
	220	+	case "", "1", "true", "yes":
	221	+	case "0", "false", "no":
	222	+	c.NodeExternals = false
	223	+	default:
	224	+	return c, fmt.Errorf("%s: %q is not 0 or 1", EnvNode, getenv(EnvNode))
	225	+	}
206	226		if v := getenv(EnvLockFile); v != "" {
207	227		if !filepath.IsAbs(v) {
208	228		return c, fmt.Errorf("%s: %q must be an absolute path", EnvLockFile, v)


@@ -36,6 +36,9 @@ const (
36	EnvJobDiskBW = "ERMITE_RUNNER_JOB_DISK_BANDWIDTH"	36		EnvJobDiskBW = "ERMITE_RUNNER_JOB_DISK_BANDWIDTH"
37	EnvJobDiskIOPS = "ERMITE_RUNNER_JOB_DISK_IOPS"	37		EnvJobDiskIOPS = "ERMITE_RUNNER_JOB_DISK_IOPS"
38	EnvJobNetBandwidth = "ERMITE_RUNNER_JOB_NET_BANDWIDTH"	38		EnvJobNetBandwidth = "ERMITE_RUNNER_JOB_NET_BANDWIDTH"
		39	+	// EnvNode=0 turns the Node.js externals off (nothing is downloaded or
		40	+	// attached).
		41	+	EnvNode = "ERMITE_RUNNER_NODE"
39	)	42		)
40		43
41	// Defaults.	44		// Defaults.

@@ -117,6 +120,14 @@ type Config struct {
117	GuestForge *GuestForge	120		GuestForge *GuestForge
118	// Rate limits of micro-VMs; zero disables.	121		// Rate limits of micro-VMs; zero disables.
119	DiskBandwidthMiB, DiskIOPS, NetMbit int	122		DiskBandwidthMiB, DiskIOPS, NetMbit int
		123	+
		124	+	// NodeExternals is false when ERMITE_RUNNER_NODE=0: the caller then sets no
		125	+	// Externals.
		126	+	NodeExternals bool
		127	+	// Externals, when not nil, is the host's set of external tools (Node.js),
		128	+	// attached read-only to every sandbox at GuestExternalsDir. The caller sets
		129	+	// it before building the drivers, like GuestForge.
		130	+	Externals ExternalsSource
120	}	131		}
121		132
122	// LoadConfig reads the sandbox settings from getenv (os.Getenv in production).	133		// LoadConfig reads the sandbox settings from getenv (os.Getenv in production).

@@ -134,6 +145,8 @@ func LoadConfig(getenv func(string) string) (Config, error) {
134	JobPids: DefaultJobPids,	145		JobPids: DefaultJobPids,
135	LockFile: DefaultLockFile,	146		LockFile: DefaultLockFile,
136		147
		148	+	NodeExternals: true,
		149	+
137	DiskBandwidthMiB: DefaultDiskBandwidthMiB,	150		DiskBandwidthMiB: DefaultDiskBandwidthMiB,
138	DiskIOPS: DefaultDiskIOPS,	151		DiskIOPS: DefaultDiskIOPS,
139	NetMbit: DefaultNetMbit,	152		NetMbit: DefaultNetMbit,

@@ -203,6 +216,13 @@ func LoadConfig(getenv func(string) string) (Config, error) {
203	default:	216		default:
204	return c, fmt.Errorf("%s: %q is not 0 or 1", EnvGuestForge, getenv(EnvGuestForge))	217		return c, fmt.Errorf("%s: %q is not 0 or 1", EnvGuestForge, getenv(EnvGuestForge))
205	}	218		}
		219	+	switch strings.ToLower(strings.TrimSpace(getenv(EnvNode))) {
		220	+	case "", "1", "true", "yes":
		221	+	case "0", "false", "no":
		222	+	c.NodeExternals = false
		223	+	default:
		224	+	return c, fmt.Errorf("%s: %q is not 0 or 1", EnvNode, getenv(EnvNode))
		225	+	}
206	if v := getenv(EnvLockFile); v != "" {	226		if v := getenv(EnvLockFile); v != "" {
207	if !filepath.IsAbs(v) {	227		if !filepath.IsAbs(v) {
208	return c, fmt.Errorf("%s: %q must be an absolute path", EnvLockFile, v)	228		return c, fmt.Errorf("%s: %q must be an absolute path", EnvLockFile, v)

M internal/sandbox/container/container_test.go +82 -0


@@ -1084,3 +1084,85 @@ func TestNoGuestForgeByDefault(t *testing.T) {
1084	1084		t.Fatalf("override %q", n)
1085	1085		}
1086	1086		}
	1087	+
	1088	+	// The host's external tools are bind-mounted read-only at
	1089	+	// /home/runner/externals/<name> (GitHub's layout), whichever the isolation.
	1090	+	func TestExternalsAreBoundReadOnly(t *testing.T) {
	1091	+	e := newCEnv(t)
	1092	+	src := &sandboxtest.Externals{}
	1093	+	n24 := sandboxtest.MakeExternal(t, "node24", "a", "v24")
	1094	+	n20 := sandboxtest.MakeExternal(t, "node20", "b", "v20")
	1095	+	gone := sandbox.External{Name: "node24_alpine", ID: "c", Dir: "/state/externals/.unavailable/x", Unavailable: true}
	1096	+	src.Set(n24, n20, gone)
	1097	+	e.cfg.Externals = src
	1098	+	d := e.driver(t)
	1099	+	run := func() containerSpec {
	1100	+	t.Helper()
	1101	+	sb, _ := d.New()
	1102	+	conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
	1103	+	if err != nil {
	1104	+	t.Fatal(err)
	1105	+	}
	1106	+	stub(t, conn)
	1107	+	e.eng.mu.Lock()
	1108	+	var spec containerSpec
	1109	+	for _, s := range e.eng.containers {
	1110	+	spec = s
	1111	+	}
	1112	+	e.eng.mu.Unlock()
	1113	+	_ = sb.Stop(context.Background())
	1114	+	return spec
	1115	+	}
	1116	+	spec := run()
	1117	+	got := map[string]bindMount{}
	1118	+	for _, b := range spec.Binds {
	1119	+	got[b.Target] = b
	1120	+	}
	1121	+	for _, w := range []sandbox.External{n24, n20, gone} {
	1122	+	b, ok := got["/home/runner/externals/"+w.Name]
	1123	+	if !ok \|\| !b.ReadOnly \|\| b.Source != w.Dir {
	1124	+	t.Errorf("%s: %+v (binds %+v)", w.Name, b, spec.Binds)
	1125	+	}
	1126	+	}
	1127	+	// The executable and the other binds are untouched.
	1128	+	if b := got["/opt/ermite/ermite-runner"]; !b.ReadOnly \|\| b.Source != e.cfg.Executable {
	1129	+	t.Errorf("executable bind: %+v", b)
	1130	+	}
	1131	+	if len(spec.Binds) != 5+3 {
	1132	+	t.Errorf("binds: %+v", spec.Binds)
	1133	+	}
	1134	+
	1135	+	// A new Node.js is picked up by the next job, without a driver restart.
	1136	+	src.Set(sandboxtest.MakeExternal(t, "node24", "d", "v24.22"))
	1137	+	spec = run()
	1138	+	n := 0
	1139	+	for _, b := range spec.Binds {
	1140	+	if strings.HasPrefix(b.Target, "/home/runner/externals/") {
	1141	+	n++
	1142	+	}
	1143	+	}
	1144	+	if n != 1 {
	1145	+	t.Errorf("externals binds after the change: %+v", spec.Binds)
	1146	+	}
	1147	+	}
	1148	+
	1149	+	func TestNoExternalsNoBinds(t *testing.T) {
	1150	+	e := newCEnv(t)
	1151	+	d := e.driver(t)
	1152	+	sb, _ := d.New()
	1153	+	conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
	1154	+	if err != nil {
	1155	+	t.Fatal(err)
	1156	+	}
	1157	+	stub(t, conn)
	1158	+	defer func() { _ = sb.Stop(context.Background()) }()
	1159	+	e.eng.mu.Lock()
	1160	+	defer e.eng.mu.Unlock()
	1161	+	for _, s := range e.eng.containers {
	1162	+	for _, b := range s.Binds {
	1163	+	if strings.Contains(b.Target, "externals") {
	1164	+	t.Errorf("unexpected bind %+v", b)
	1165	+	}
	1166	+	}
	1167	+	}
	1168	+	}


@@ -1084,3 +1084,85 @@ func TestNoGuestForgeByDefault(t *testing.T) {
1084	t.Fatalf("override %q", n)	1084		t.Fatalf("override %q", n)
1085	}	1085		}
1086	}	1086		}
		1087	+
		1088	+	// The host's external tools are bind-mounted read-only at
		1089	+	// /home/runner/externals/<name> (GitHub's layout), whichever the isolation.
		1090	+	func TestExternalsAreBoundReadOnly(t *testing.T) {
		1091	+	e := newCEnv(t)
		1092	+	src := &sandboxtest.Externals{}
		1093	+	n24 := sandboxtest.MakeExternal(t, "node24", "a", "v24")
		1094	+	n20 := sandboxtest.MakeExternal(t, "node20", "b", "v20")
		1095	+	gone := sandbox.External{Name: "node24_alpine", ID: "c", Dir: "/state/externals/.unavailable/x", Unavailable: true}
		1096	+	src.Set(n24, n20, gone)
		1097	+	e.cfg.Externals = src
		1098	+	d := e.driver(t)
		1099	+	run := func() containerSpec {
		1100	+	t.Helper()
		1101	+	sb, _ := d.New()
		1102	+	conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
		1103	+	if err != nil {
		1104	+	t.Fatal(err)
		1105	+	}
		1106	+	stub(t, conn)
		1107	+	e.eng.mu.Lock()
		1108	+	var spec containerSpec
		1109	+	for _, s := range e.eng.containers {
		1110	+	spec = s
		1111	+	}
		1112	+	e.eng.mu.Unlock()
		1113	+	_ = sb.Stop(context.Background())
		1114	+	return spec
		1115	+	}
		1116	+	spec := run()
		1117	+	got := map[string]bindMount{}
		1118	+	for _, b := range spec.Binds {
		1119	+	got[b.Target] = b
		1120	+	}
		1121	+	for _, w := range []sandbox.External{n24, n20, gone} {
		1122	+	b, ok := got["/home/runner/externals/"+w.Name]
		1123	+	if !ok \|\| !b.ReadOnly \|\| b.Source != w.Dir {
		1124	+	t.Errorf("%s: %+v (binds %+v)", w.Name, b, spec.Binds)
		1125	+	}
		1126	+	}
		1127	+	// The executable and the other binds are untouched.
		1128	+	if b := got["/opt/ermite/ermite-runner"]; !b.ReadOnly \|\| b.Source != e.cfg.Executable {
		1129	+	t.Errorf("executable bind: %+v", b)
		1130	+	}
		1131	+	if len(spec.Binds) != 5+3 {
		1132	+	t.Errorf("binds: %+v", spec.Binds)
		1133	+	}
		1134	+
		1135	+	// A new Node.js is picked up by the next job, without a driver restart.
		1136	+	src.Set(sandboxtest.MakeExternal(t, "node24", "d", "v24.22"))
		1137	+	spec = run()
		1138	+	n := 0
		1139	+	for _, b := range spec.Binds {
		1140	+	if strings.HasPrefix(b.Target, "/home/runner/externals/") {
		1141	+	n++
		1142	+	}
		1143	+	}
		1144	+	if n != 1 {
		1145	+	t.Errorf("externals binds after the change: %+v", spec.Binds)
		1146	+	}
		1147	+	}
		1148	+
		1149	+	func TestNoExternalsNoBinds(t *testing.T) {
		1150	+	e := newCEnv(t)
		1151	+	d := e.driver(t)
		1152	+	sb, _ := d.New()
		1153	+	conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
		1154	+	if err != nil {
		1155	+	t.Fatal(err)
		1156	+	}
		1157	+	stub(t, conn)
		1158	+	defer func() { _ = sb.Stop(context.Background()) }()
		1159	+	e.eng.mu.Lock()
		1160	+	defer e.eng.mu.Unlock()
		1161	+	for _, s := range e.eng.containers {
		1162	+	for _, b := range s.Binds {
		1163	+	if strings.Contains(b.Target, "externals") {
		1164	+	t.Errorf("unexpected bind %+v", b)
		1165	+	}
		1166	+	}
		1167	+	}
		1168	+	}

M internal/sandbox/container/driver.go +25 -12


@@ -13,8 +13,8 @@
13	13		// it after a crash.
14	14		//
15	15		// The executor stream is a Unix socket in a per-job directory of the runner's
16		-	// state directory, bind-mounted into the container. The runner executable is
17		-	// bind-mounted read-only. Both paths are host paths given to the Docker daemon:
	16	+	// state directory, bind-mounted into the container. The runner executable and
	17	+	// the host's external tools (Node.js) are bind-mounted read-only. Both paths are host paths given to the Docker daemon:
18	18		// the daemon must run on the same host (and mount namespace) as the runner.
19	19		package container
20	20

@@ -435,6 +435,14 @@ func (s *containerSandbox) runNetwork(ctx context.Context, netw netpolicy.Net, l
435	435		func (s *containerSandbox) runContainer(ctx context.Context, spec sandbox.JobSpec, netw netpolicy.Net, labels map[string]string) (<-chan exitResult, error) {
436	436		d := s.d
437	437		entry := sandbox.GuestExecDir + "/" + sandbox.GuestExecName
	438	+	binds := []bindMount{
	439	+	{Source: d.exe, Target: entry, ReadOnly: true},
	440	+	{Source: d.sockDir(s.id), Target: sandbox.ContainerRunDir},
	441	+	{Source: d.scratchDir(s.id), Target: sandbox.GuestScratch},
	442	+	{Source: d.workDir(s.id), Target: containerWorkspace},
	443	+	{Source: filepath.Join(d.scratchDir(s.id), "tmp"), Target: containerTmp},
	444	+	}
	445	+	binds = append(binds, externalBinds(d.cfg.ExternalsOf())...)
438	446		cid, err := d.eng.CreateContainer(ctx, containerSpec{
439	447		Name: "ermite-" + s.id,
440	448		Image: d.image,

@@ -446,16 +454,10 @@ func (s *containerSandbox) runContainer(ctx context.Context, spec sandbox.JobSpe
446	454		Network: "ermite-" + s.id,
447	455		IPv4: netw.Guest,
448	456		DNS: netw.Gateway,
449		-	Binds: []bindMount{
450		-	{Source: d.exe, Target: entry, ReadOnly: true},
451		-	{Source: d.sockDir(s.id), Target: sandbox.ContainerRunDir},
452		-	{Source: d.scratchDir(s.id), Target: sandbox.GuestScratch},
453		-	{Source: d.workDir(s.id), Target: containerWorkspace},
454		-	{Source: filepath.Join(d.scratchDir(s.id), "tmp"), Target: containerTmp},
455		-	},
456		-	MemoryMiB: spec.MemoryMiB,
457		-	CPUs: spec.CPUs,
458		-	Pids: d.cfg.JobPids,
	457	+	Binds: binds,
	458	+	MemoryMiB: spec.MemoryMiB,
	459	+	CPUs: spec.CPUs,
	460	+	Pids: d.cfg.JobPids,
459	461		})
460	462		if err != nil {
461	463		return nil, err

@@ -468,6 +470,17 @@ func (s *containerSandbox) runContainer(ctx context.Context, spec sandbox.JobSpe
468	470		return exited, nil
469	471		}
470	472
	473	+	// externalBinds are the read-only bind mounts of the host's external tools
	474	+	// (Node.js) at /home/runner/externals/<name>, GitHub's layout. A container needs
	475	+	// no more than the host path: it reads the tool directly.
	476	+	func externalBinds(set []sandbox.External) []bindMount {
	477	+	var out []bindMount
	478	+	for _, e := range set {
	479	+	out = append(out, bindMount{Source: e.Dir, Target: sandbox.GuestExternalsDir + "/" + e.Name, ReadOnly: true})
	480	+	}
	481	+	return out
	482	+	}
	483	+
471	484		// waitExecutor waits for the executor to connect. Only ctx (the caller's
472	485		// context, or Stop through the boot gate), the container exiting, and the
473	486		// timeout end the wait.

 // it after a crash.
 //
 // The executor stream is a Unix socket in a per-job directory of the runner's
-// state directory, bind-mounted into the container. The runner executable is
+// state directory, bind-mounted into the container. The runner executable and
-// bind-mounted read-only. Both paths are host paths given to the Docker daemon:
+// the host's external tools (Node.js) are bind-mounted read-only. Both paths are host paths given to the Docker daemon:
 // the daemon must run on the same host (and mount namespace) as the runner.
 package container
 func (s *containerSandbox) runContainer(ctx context.Context, spec sandbox.JobSpec, netw netpolicy.Net, labels map[string]string) (<-chan exitResult, error) {
 	d := s.d
 	entry := sandbox.GuestExecDir + "/" + sandbox.GuestExecName
+	binds := []bindMount{
+		{Source: d.exe, Target: entry, ReadOnly: true},
+		{Source: d.sockDir(s.id), Target: sandbox.ContainerRunDir},
+		{Source: d.scratchDir(s.id), Target: sandbox.GuestScratch},
+		{Source: d.workDir(s.id), Target: containerWorkspace},
+		{Source: filepath.Join(d.scratchDir(s.id), "tmp"), Target: containerTmp},
+	}
+	binds = append(binds, externalBinds(d.cfg.ExternalsOf())...)
 	cid, err := d.eng.CreateContainer(ctx, containerSpec{
 		Name:       "ermite-" + s.id,
 		Image:      d.image,
 		Network:    "ermite-" + s.id,
 		IPv4:       netw.Guest,
 		DNS:        netw.Gateway,
-		Binds: []bindMount{
+		Binds:      binds,
-			{Source: d.exe, Target: entry, ReadOnly: true},
+		MemoryMiB:  spec.MemoryMiB,
-			{Source: d.sockDir(s.id), Target: sandbox.ContainerRunDir},
+		CPUs:       spec.CPUs,
-			{Source: d.scratchDir(s.id), Target: sandbox.GuestScratch},
+		Pids:       d.cfg.JobPids,
-			{Source: d.workDir(s.id), Target: containerWorkspace},
-			{Source: filepath.Join(d.scratchDir(s.id), "tmp"), Target: containerTmp},
-		},
-		MemoryMiB: spec.MemoryMiB,
-		CPUs:      spec.CPUs,
-		Pids:      d.cfg.JobPids,
 	})
 	if err != nil {
 		return nil, err
 	return exited, nil
 }
+// externalBinds are the read-only bind mounts of the host's external tools
+// (Node.js) at /home/runner/externals/<name>, GitHub's layout. A container needs
+// no more than the host path: it reads the tool directly.
+func externalBinds(set []sandbox.External) []bindMount {
+	var out []bindMount
+	for _, e := range set {
+		out = append(out, bindMount{Source: e.Dir, Target: sandbox.GuestExternalsDir + "/" + e.Name, ReadOnly: true})
+	}
+	return out
+}
 // waitExecutor waits for the executor to connect. Only ctx (the caller's
 // context, or Stop through the boot gate), the container exiting, and the
 // timeout end the wait.

M internal/sandbox/contract.go +9 -0


@@ -16,6 +16,13 @@ const (
16	16		// ContainerSocket is the executor stream socket inside it.
17	17		ContainerRunDir = "/run/ermite"
18	18		ContainerSocket = "exec.sock"
	19	+	// GuestExternalsDir is where the host's external tools (Node.js) appear in
	20	+	// the sandbox, read-only, in GitHub's layout: <dir>/node24/bin/node.
	21	+	GuestExternalsDir = "/home/runner/externals"
	22	+	// ExternalUnavailableFile, in <GuestExternalsDir>/<name>, tells why a tool
	23	+	// is missing on this runner. A directory that has it has no tool (its
	24	+	// bin/node only prints the reason and fails).
	25	+	ExternalUnavailableFile = "UNAVAILABLE"
19	26		// GuestScratch is where the per-job scratch disk (VM) or volume (container)
20	27		// is mounted; dockerd keeps its data root there.
21	28		GuestScratch = "/scratch"

@@ -27,4 +34,6 @@ const (
27	34		CmdlineSandbox = "ermite.sandbox" // "vm"
28	35		CmdlineConnect = "ermite.connect" // "vsock:1024"
29	36		CmdlineGateway = "ermite.gw" // host side of the /30, also the DNS server
	37	+	// CmdlineExternals=1: the externals disk is attached as the fourth drive.
	38	+	CmdlineExternals = "ermite.externals"
30	39		)


@@ -16,6 +16,13 @@ const (
16	// ContainerSocket is the executor stream socket inside it.	16		// ContainerSocket is the executor stream socket inside it.
17	ContainerRunDir = "/run/ermite"	17		ContainerRunDir = "/run/ermite"
18	ContainerSocket = "exec.sock"	18		ContainerSocket = "exec.sock"
		19	+	// GuestExternalsDir is where the host's external tools (Node.js) appear in
		20	+	// the sandbox, read-only, in GitHub's layout: <dir>/node24/bin/node.
		21	+	GuestExternalsDir = "/home/runner/externals"
		22	+	// ExternalUnavailableFile, in <GuestExternalsDir>/<name>, tells why a tool
		23	+	// is missing on this runner. A directory that has it has no tool (its
		24	+	// bin/node only prints the reason and fails).
		25	+	ExternalUnavailableFile = "UNAVAILABLE"
19	// GuestScratch is where the per-job scratch disk (VM) or volume (container)	26		// GuestScratch is where the per-job scratch disk (VM) or volume (container)
20	// is mounted; dockerd keeps its data root there.	27		// is mounted; dockerd keeps its data root there.
21	GuestScratch = "/scratch"	28		GuestScratch = "/scratch"

@@ -27,4 +34,6 @@ const (
27	CmdlineSandbox = "ermite.sandbox" // "vm"	34		CmdlineSandbox = "ermite.sandbox" // "vm"
28	CmdlineConnect = "ermite.connect" // "vsock:1024"	35		CmdlineConnect = "ermite.connect" // "vsock:1024"
29	CmdlineGateway = "ermite.gw" // host side of the /30, also the DNS server	36		CmdlineGateway = "ermite.gw" // host side of the /30, also the DNS server
		37	+	// CmdlineExternals=1: the externals disk is attached as the fourth drive.
		38	+	CmdlineExternals = "ermite.externals"
30	)	39		)

A internal/sandbox/externals.go +56 -0

@@ -0,0 +1,56 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package sandbox
	4	+
	5	+	import (
	6	+	"crypto/sha256"
	7	+	"encoding/hex"
	8	+	"sort"
	9	+	)
	10	+
	11	+	// External is one tool the host provides outside the image (Node.js), attached
	12	+	// read-only at GuestExternalsDir/<Name>.
	13	+	type External struct {
	14	+	// Name is the directory below GuestExternalsDir: node20, node24,
	15	+	// node20_alpine, node24_alpine.
	16	+	Name string
	17	+	// ID identifies the content: it changes whenever the files (or the reason of
	18	+	// an unavailable tool) change, so that drivers can key caches on it.
	19	+	ID string
	20	+	// Dir is the host directory that is attached. For an unavailable tool it
	21	+	// holds the file ExternalUnavailableFile with the reason, and a bin/node
	22	+	// script that prints it and fails.
	23	+	Dir string
	24	+	// Unavailable is true when the tool is missing on this runner.
	25	+	Unavailable bool
	26	+	}
	27	+
	28	+	// ExternalsSource hands the drivers the current set of externals. It is called
	29	+	// at every sandbox start and may change between calls (a download finishing).
	30	+	type ExternalsSource interface {
	31	+	Externals() []External
	32	+	}
	33	+
	34	+	// ExternalsKey is the cache key of a set: the same tools give the same key, in
	35	+	// any order, and any change of content gives another.
	36	+	func ExternalsKey(set []External) string {
	37	+	ids := make([]string, 0, len(set))
	38	+	for _, e := range set {
	39	+	ids = append(ids, e.Name+"="+e.ID)
	40	+	}
	41	+	sort.Strings(ids)
	42	+	h := sha256.New()
	43	+	for _, id := range ids {
	44	+	h.Write([]byte(id))
	45	+	h.Write([]byte{0})
	46	+	}
	47	+	return hex.EncodeToString(h.Sum(nil))[:16]
	48	+	}
	49	+
	50	+	// ExternalsOf returns the current set of cfg (nil when there is none).
	51	+	func (c Config) ExternalsOf() []External {
	52	+	if c.Externals == nil {
	53	+	return nil
	54	+	}
	55	+	return c.Externals.Externals()
	56	+	}

@@ -0,0 +1,56 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package sandbox
	4	+
	5	+	import (
	6	+	"crypto/sha256"
	7	+	"encoding/hex"
	8	+	"sort"
	9	+	)
	10	+
	11	+	// External is one tool the host provides outside the image (Node.js), attached
	12	+	// read-only at GuestExternalsDir/<Name>.
	13	+	type External struct {
	14	+	// Name is the directory below GuestExternalsDir: node20, node24,
	15	+	// node20_alpine, node24_alpine.
	16	+	Name string
	17	+	// ID identifies the content: it changes whenever the files (or the reason of
	18	+	// an unavailable tool) change, so that drivers can key caches on it.
	19	+	ID string
	20	+	// Dir is the host directory that is attached. For an unavailable tool it
	21	+	// holds the file ExternalUnavailableFile with the reason, and a bin/node
	22	+	// script that prints it and fails.
	23	+	Dir string
	24	+	// Unavailable is true when the tool is missing on this runner.
	25	+	Unavailable bool
	26	+	}
	27	+
	28	+	// ExternalsSource hands the drivers the current set of externals. It is called
	29	+	// at every sandbox start and may change between calls (a download finishing).
	30	+	type ExternalsSource interface {
	31	+	Externals() []External
	32	+	}
	33	+
	34	+	// ExternalsKey is the cache key of a set: the same tools give the same key, in
	35	+	// any order, and any change of content gives another.
	36	+	func ExternalsKey(set []External) string {
	37	+	ids := make([]string, 0, len(set))
	38	+	for _, e := range set {
	39	+	ids = append(ids, e.Name+"="+e.ID)
	40	+	}
	41	+	sort.Strings(ids)
	42	+	h := sha256.New()
	43	+	for _, id := range ids {
	44	+	h.Write([]byte(id))
	45	+	h.Write([]byte{0})
	46	+	}
	47	+	return hex.EncodeToString(h.Sum(nil))[:16]
	48	+	}
	49	+
	50	+	// ExternalsOf returns the current set of cfg (nil when there is none).
	51	+	func (c Config) ExternalsOf() []External {
	52	+	if c.Externals == nil {
	53	+	return nil
	54	+	}
	55	+	return c.Externals.Externals()
	56	+	}

M internal/sandbox/integration/vm_test.go +97 -0


@@ -4,7 +4,9 @@ package integration
4	4
5	5		import (
6	6		"context"
	7	+	"net/http"
7	8		"os"
	9	+	"path/filepath"
8	10		"strings"
9	11		"syscall"
10	12		"testing"

@@ -12,6 +14,7 @@ import (
12	14
13	15		"bnema.dev/ermite-runner/internal/execstream"
14	16		"bnema.dev/ermite-runner/internal/execstream/execpb"
	17	+	"bnema.dev/ermite-runner/internal/externals"
15	18		"bnema.dev/ermite-runner/internal/sandbox"
16	19		"bnema.dev/ermite-runner/internal/sandbox/drivers"
17	20		)

@@ -208,3 +211,97 @@ jobs:
208	211		}
209	212		t.Logf("cancelled job ended %v after the start", time.Since(start))
210	213		}
	214	+
	215	+	// nodeCache serves the pinned Node.js archives from a local directory
	216	+	// (ERMITE_RUNNER_IT_NODE_DIR holding node-v24.21.0-linux-x64.tar.gz, say): the
	217	+	// sandbox of the VM tests has no internet, and this way the real archive is
	218	+	// still checked against the pinned SHA-256 and unpacked by the real code.
	219	+	type nodeCache string
	220	+
	221	+	func (d nodeCache) RoundTrip(r http.Request) (http.Response, error) {
	222	+	f, err := os.Open(filepath.Join(string(d), filepath.Base(r.URL.Path)))
	223	+	if err != nil {
	224	+	return &http.Response{StatusCode: http.StatusNotFound, Status: "404 Not Found", Body: http.NoBody, Request: r}, nil
	225	+	}
	226	+	return &http.Response{StatusCode: http.StatusOK, Status: "200 OK", Body: f, Request: r}, nil
	227	+	}
	228	+
	229	+	// TestVMNodeExternals runs the pinned Node.js 24, provided by the host outside
	230	+	// the image, inside a job's micro-VM: the externals disk is the fourth drive,
	231	+	// mounted read-only at /home/runner/externals, in GitHub's layout.
	232	+	//
	233	+	// Needs the archive of the pinned node24 for this architecture, either in
	234	+	// ERMITE_RUNNER_IT_NODE_DIR (offline) or downloadable from nodejs.org.
	235	+	func TestVMNodeExternals(t *testing.T) {
	236	+	if os.Getenv("ERMITE_RUNNER_IT_VM") != "1" {
	237	+	t.Skip("set ERMITE_RUNNER_IT_VM=1 (needs KVM, firecracker, a kernel and a base image; see README)")
	238	+	}
	239	+	need(t, "mkfs.ext4")
	240	+	needNetAdmin(t)
	241	+	ctx := ctxWithLog(t)
	242	+	cfg := loadConfig(t, sandbox.VM)
	243	+	if why := sandbox.VMUnavailable(cfg, sandbox.Prober{}); len(why) > 0 {
	244	+	t.Fatalf("vm isolation is not available here: %s", strings.Join(why, "; "))
	245	+	}
	246	+
	247	+	m := &externals.Manager{Dir: filepath.Join(cfg.StateDir, "externals"), Arch: externals.HostArch()}
	248	+	for _, a := range externals.ForArch(externals.Pinned, m.Arch) {
	249	+	if a.Name == "node24" {
	250	+	m.Table = append(m.Table, a)
	251	+	}
	252	+	}
	253	+	if len(m.Table) != 1 {
	254	+	t.Skipf("no pinned node24 for %q", m.Arch)
	255	+	}
	256	+	if dir := os.Getenv("ERMITE_RUNNER_IT_NODE_DIR"); dir != "" {
	257	+	m.Client = &http.Client{Transport: nodeCache(dir)}
	258	+	}
	259	+	if err := m.Sync(ctx); err != nil {
	260	+	t.Skipf("cannot get the pinned Node.js (set ERMITE_RUNNER_IT_NODE_DIR to a directory with %s): %v", filepath.Base(m.Table[0].URL), err)
	261	+	}
	262	+	cfg.Externals = m
	263	+
	264	+	drv, err := sandbox.Select(ctx, cfg, drivers.Factories(), sandbox.Prober{})
	265	+	if err != nil {
	266	+	t.Fatal(err)
	267	+	}
	268	+	t.Cleanup(func() {
	269	+	_ = drv.Close()
	270	+	if d2, err := sandbox.Select(context.Background(), cfg, drivers.Factories(), sandbox.Prober{}); err == nil {
	271	+	_ = d2.Reap(context.Background())
	272	+	_ = d2.Close()
	273	+	}
	274	+	})
	275	+	if err := drv.Reap(ctx); err != nil {
	276	+	t.Fatal(err)
	277	+	}
	278	+	sb, err := drv.New()
	279	+	if err != nil {
	280	+	t.Fatal(err)
	281	+	}
	282	+	defer func() { _ = sb.Stop(context.Background()) }()
	283	+	conn, err := sb.Start(ctx, sandbox.JobSpec{JobID: "node-externals"})
	284	+	if err != nil {
	285	+	t.Fatal(err)
	286	+	}
	287	+	r, err := runJob(ctx, conn, `
	288	+	N=/home/runner/externals/node24/bin/node
	289	+	$N -e 'console.log(process.version)'
	290	+	echo "has-node24: $(test -x /home/runner/externals/node24/bin/node && echo yes)"
	291	+	touch /home/runner/externals/node24/probe 2>/dev/null && echo "WRITABLE" \|\| echo "read-only"
	292	+	grep ' /home/runner/externals ' /proc/mounts
	293	+	`)
	294	+	if err != nil {
	295	+	t.Fatal(err)
	296	+	}
	297	+	t.Log("\n" + r.out + r.errOut)
	298	+	want := m.Table[0].Version
	299	+	for _, w := range []string{"v" + want + "\n", "has-node24: yes\n", "read-only\n", "/dev/vdd /home/runner/externals ext4 ro"} {
	300	+	if !strings.Contains(r.out, w) {
	301	+	t.Errorf("output lacks %q", w)
	302	+	}
	303	+	}
	304	+	if strings.Contains(r.out, "WRITABLE") {
	305	+	t.Error("the externals are writable from the guest")
	306	+	}
	307	+	}


@@ -4,7 +4,9 @@ package integration
4		4
5	import (	5		import (
6	"context"	6		"context"
		7	+	"net/http"
7	"os"	8		"os"
		9	+	"path/filepath"
8	"strings"	10		"strings"
9	"syscall"	11		"syscall"
10	"testing"	12		"testing"

@@ -12,6 +14,7 @@ import (
12		14
13	"bnema.dev/ermite-runner/internal/execstream"	15		"bnema.dev/ermite-runner/internal/execstream"
14	"bnema.dev/ermite-runner/internal/execstream/execpb"	16		"bnema.dev/ermite-runner/internal/execstream/execpb"
		17	+	"bnema.dev/ermite-runner/internal/externals"
15	"bnema.dev/ermite-runner/internal/sandbox"	18		"bnema.dev/ermite-runner/internal/sandbox"
16	"bnema.dev/ermite-runner/internal/sandbox/drivers"	19		"bnema.dev/ermite-runner/internal/sandbox/drivers"
17	)	20		)

@@ -208,3 +211,97 @@ jobs:
208	}	211		}
209	t.Logf("cancelled job ended %v after the start", time.Since(start))	212		t.Logf("cancelled job ended %v after the start", time.Since(start))
210	}	213		}
		214	+
		215	+	// nodeCache serves the pinned Node.js archives from a local directory
		216	+	// (ERMITE_RUNNER_IT_NODE_DIR holding node-v24.21.0-linux-x64.tar.gz, say): the
		217	+	// sandbox of the VM tests has no internet, and this way the real archive is
		218	+	// still checked against the pinned SHA-256 and unpacked by the real code.
		219	+	type nodeCache string
		220	+
		221	+	func (d nodeCache) RoundTrip(r http.Request) (http.Response, error) {
		222	+	f, err := os.Open(filepath.Join(string(d), filepath.Base(r.URL.Path)))
		223	+	if err != nil {
		224	+	return &http.Response{StatusCode: http.StatusNotFound, Status: "404 Not Found", Body: http.NoBody, Request: r}, nil
		225	+	}
		226	+	return &http.Response{StatusCode: http.StatusOK, Status: "200 OK", Body: f, Request: r}, nil
		227	+	}
		228	+
		229	+	// TestVMNodeExternals runs the pinned Node.js 24, provided by the host outside
		230	+	// the image, inside a job's micro-VM: the externals disk is the fourth drive,
		231	+	// mounted read-only at /home/runner/externals, in GitHub's layout.
		232	+	//
		233	+	// Needs the archive of the pinned node24 for this architecture, either in
		234	+	// ERMITE_RUNNER_IT_NODE_DIR (offline) or downloadable from nodejs.org.
		235	+	func TestVMNodeExternals(t *testing.T) {
		236	+	if os.Getenv("ERMITE_RUNNER_IT_VM") != "1" {
		237	+	t.Skip("set ERMITE_RUNNER_IT_VM=1 (needs KVM, firecracker, a kernel and a base image; see README)")
		238	+	}
		239	+	need(t, "mkfs.ext4")
		240	+	needNetAdmin(t)
		241	+	ctx := ctxWithLog(t)
		242	+	cfg := loadConfig(t, sandbox.VM)
		243	+	if why := sandbox.VMUnavailable(cfg, sandbox.Prober{}); len(why) > 0 {
		244	+	t.Fatalf("vm isolation is not available here: %s", strings.Join(why, "; "))
		245	+	}
		246	+
		247	+	m := &externals.Manager{Dir: filepath.Join(cfg.StateDir, "externals"), Arch: externals.HostArch()}
		248	+	for _, a := range externals.ForArch(externals.Pinned, m.Arch) {
		249	+	if a.Name == "node24" {
		250	+	m.Table = append(m.Table, a)
		251	+	}
		252	+	}
		253	+	if len(m.Table) != 1 {
		254	+	t.Skipf("no pinned node24 for %q", m.Arch)
		255	+	}
		256	+	if dir := os.Getenv("ERMITE_RUNNER_IT_NODE_DIR"); dir != "" {
		257	+	m.Client = &http.Client{Transport: nodeCache(dir)}
		258	+	}
		259	+	if err := m.Sync(ctx); err != nil {
		260	+	t.Skipf("cannot get the pinned Node.js (set ERMITE_RUNNER_IT_NODE_DIR to a directory with %s): %v", filepath.Base(m.Table[0].URL), err)
		261	+	}
		262	+	cfg.Externals = m
		263	+
		264	+	drv, err := sandbox.Select(ctx, cfg, drivers.Factories(), sandbox.Prober{})
		265	+	if err != nil {
		266	+	t.Fatal(err)
		267	+	}
		268	+	t.Cleanup(func() {
		269	+	_ = drv.Close()
		270	+	if d2, err := sandbox.Select(context.Background(), cfg, drivers.Factories(), sandbox.Prober{}); err == nil {
		271	+	_ = d2.Reap(context.Background())
		272	+	_ = d2.Close()
		273	+	}
		274	+	})
		275	+	if err := drv.Reap(ctx); err != nil {
		276	+	t.Fatal(err)
		277	+	}
		278	+	sb, err := drv.New()
		279	+	if err != nil {
		280	+	t.Fatal(err)
		281	+	}
		282	+	defer func() { _ = sb.Stop(context.Background()) }()
		283	+	conn, err := sb.Start(ctx, sandbox.JobSpec{JobID: "node-externals"})
		284	+	if err != nil {
		285	+	t.Fatal(err)
		286	+	}
		287	+	r, err := runJob(ctx, conn, `
		288	+	N=/home/runner/externals/node24/bin/node
		289	+	$N -e 'console.log(process.version)'
		290	+	echo "has-node24: $(test -x /home/runner/externals/node24/bin/node && echo yes)"
		291	+	touch /home/runner/externals/node24/probe 2>/dev/null && echo "WRITABLE" \|\| echo "read-only"
		292	+	grep ' /home/runner/externals ' /proc/mounts
		293	+	`)
		294	+	if err != nil {
		295	+	t.Fatal(err)
		296	+	}
		297	+	t.Log("\n" + r.out + r.errOut)
		298	+	want := m.Table[0].Version
		299	+	for _, w := range []string{"v" + want + "\n", "has-node24: yes\n", "read-only\n", "/dev/vdd /home/runner/externals ext4 ro"} {
		300	+	if !strings.Contains(r.out, w) {
		301	+	t.Errorf("output lacks %q", w)
		302	+	}
		303	+	}
		304	+	if strings.Contains(r.out, "WRITABLE") {
		305	+	t.Error("the externals are writable from the guest")
		306	+	}
		307	+	}

M internal/sandbox/sandbox_test.go +32 -0


@@ -269,3 +269,35 @@ func TestCheckStatic(t *testing.T) {
269	269		t.Skip("/bin/sh is static here")
270	270		}
271	271		}
	272	+
	273	+	func TestNodeSettingAndExternalsKey(t *testing.T) {
	274	+	c, err := LoadConfig(env(nil))
	275	+	if err != nil \|\| !c.NodeExternals {
	276	+	t.Fatalf("on by default: %+v %v", c, err)
	277	+	}
	278	+	for v, want := range map[string]bool{"0": false, "false": false, "NO": false, "1": true, "yes": true, "": true} {
	279	+	c, err := LoadConfig(env(map[string]string{EnvNode: v}))
	280	+	if err != nil \|\| c.NodeExternals != want {
	281	+	t.Errorf("%s=%q: %v %v", EnvNode, v, c.NodeExternals, err)
	282	+	}
	283	+	}
	284	+	if _, err := LoadConfig(env(map[string]string{EnvNode: "maybe"})); err == nil \|\| !strings.Contains(err.Error(), EnvNode) {
	285	+	t.Errorf("got %v", err)
	286	+	}
	287	+
	288	+	a := External{Name: "node24", ID: "x1", Dir: "/a"}
	289	+	b := External{Name: "node20", ID: "y1", Dir: "/b"}
	290	+	k := ExternalsKey([]External{a, b})
	291	+	if k != ExternalsKey([]External{b, a}) {
	292	+	t.Error("the key depends on the order")
	293	+	}
	294	+	if k != ExternalsKey([]External{{Name: "node24", ID: "x1", Dir: "/elsewhere"}, b}) {
	295	+	t.Error("the key depends on the host path, which does not change the content")
	296	+	}
	297	+	if k == ExternalsKey([]External{{Name: "node24", ID: "x2"}, b}) \|\| k == ExternalsKey([]External{a}) {
	298	+	t.Error("a change of content keeps the key")
	299	+	}
	300	+	if (Config{}).ExternalsOf() != nil {
	301	+	t.Error("no source, no externals")
	302	+	}
	303	+	}


@@ -269,3 +269,35 @@ func TestCheckStatic(t *testing.T) {
269	t.Skip("/bin/sh is static here")	269		t.Skip("/bin/sh is static here")
270	}	270		}
271	}	271		}
		272	+
		273	+	func TestNodeSettingAndExternalsKey(t *testing.T) {
		274	+	c, err := LoadConfig(env(nil))
		275	+	if err != nil \|\| !c.NodeExternals {
		276	+	t.Fatalf("on by default: %+v %v", c, err)
		277	+	}
		278	+	for v, want := range map[string]bool{"0": false, "false": false, "NO": false, "1": true, "yes": true, "": true} {
		279	+	c, err := LoadConfig(env(map[string]string{EnvNode: v}))
		280	+	if err != nil \|\| c.NodeExternals != want {
		281	+	t.Errorf("%s=%q: %v %v", EnvNode, v, c.NodeExternals, err)
		282	+	}
		283	+	}
		284	+	if _, err := LoadConfig(env(map[string]string{EnvNode: "maybe"})); err == nil \|\| !strings.Contains(err.Error(), EnvNode) {
		285	+	t.Errorf("got %v", err)
		286	+	}
		287	+
		288	+	a := External{Name: "node24", ID: "x1", Dir: "/a"}
		289	+	b := External{Name: "node20", ID: "y1", Dir: "/b"}
		290	+	k := ExternalsKey([]External{a, b})
		291	+	if k != ExternalsKey([]External{b, a}) {
		292	+	t.Error("the key depends on the order")
		293	+	}
		294	+	if k != ExternalsKey([]External{{Name: "node24", ID: "x1", Dir: "/elsewhere"}, b}) {
		295	+	t.Error("the key depends on the host path, which does not change the content")
		296	+	}
		297	+	if k == ExternalsKey([]External{{Name: "node24", ID: "x2"}, b}) \|\| k == ExternalsKey([]External{a}) {
		298	+	t.Error("a change of content keeps the key")
		299	+	}
		300	+	if (Config{}).ExternalsOf() != nil {
		301	+	t.Error("no source, no externals")
		302	+	}
		303	+	}

M internal/sandbox/sandboxtest/sandboxtest.go +35 -0


@@ -9,11 +9,13 @@ import (
9	9		"io"
10	10		"net/netip"
11	11		"os"
	12	+	"path/filepath"
12	13		"sync"
13	14		"testing"
14	15
15	16		"bnema.dev/ermite-runner/internal/cmdrun"
16	17		"bnema.dev/ermite-runner/internal/netpolicy"
	18	+	"bnema.dev/ermite-runner/internal/sandbox"
17	19		)
18	20
19	21		// NoMountRunner runs real commands except mount and umount, which succeed

@@ -105,3 +107,36 @@ func (p *Plumbing) Live() int {
105	107		defer p.mu.Unlock()
106	108		return len(p.Attached)
107	109		}
	110	+
	111	+	// Externals is a fixed, replaceable sandbox.ExternalsSource.
	112	+	type Externals struct {
	113	+	mu sync.Mutex
	114	+	set []sandbox.External
	115	+	}
	116	+
	117	+	// Set replaces the set the drivers see.
	118	+	func (e *Externals) Set(set ...sandbox.External) { e.mu.Lock(); e.set = set; e.mu.Unlock() }
	119	+
	120	+	// Externals implements sandbox.ExternalsSource.
	121	+	func (e *Externals) Externals() []sandbox.External {
	122	+	e.mu.Lock()
	123	+	defer e.mu.Unlock()
	124	+	return append([]sandbox.External(nil), e.set...)
	125	+	}
	126	+
	127	+	// MakeExternal creates a host directory holding bin/node with content and
	128	+	// returns it as an external called name.
	129	+	func MakeExternal(t testing.TB, name, id, content string) sandbox.External {
	130	+	t.Helper()
	131	+	dir := filepath.Join(t.TempDir(), name)
	132	+	if err := os.MkdirAll(filepath.Join(dir, "bin"), 0o755); err != nil {
	133	+	t.Fatal(err)
	134	+	}
	135	+	if err := os.WriteFile(filepath.Join(dir, "bin", "node"), []byte(content), 0o755); err != nil {
	136	+	t.Fatal(err)
	137	+	}
	138	+	if err := os.Symlink("bin/node", filepath.Join(dir, "nodejs")); err != nil {
	139	+	t.Fatal(err)
	140	+	}
	141	+	return sandbox.External{Name: name, ID: id, Dir: dir}
	142	+	}


@@ -9,11 +9,13 @@ import (
9	"io"	9		"io"
10	"net/netip"	10		"net/netip"
11	"os"	11		"os"
		12	+	"path/filepath"
12	"sync"	13		"sync"
13	"testing"	14		"testing"
14		15
15	"bnema.dev/ermite-runner/internal/cmdrun"	16		"bnema.dev/ermite-runner/internal/cmdrun"
16	"bnema.dev/ermite-runner/internal/netpolicy"	17		"bnema.dev/ermite-runner/internal/netpolicy"
		18	+	"bnema.dev/ermite-runner/internal/sandbox"
17	)	19		)
18		20
19	// NoMountRunner runs real commands except mount and umount, which succeed	21		// NoMountRunner runs real commands except mount and umount, which succeed

@@ -105,3 +107,36 @@ func (p *Plumbing) Live() int {
105	defer p.mu.Unlock()	107		defer p.mu.Unlock()
106	return len(p.Attached)	108		return len(p.Attached)
107	}	109		}
		110	+
		111	+	// Externals is a fixed, replaceable sandbox.ExternalsSource.
		112	+	type Externals struct {
		113	+	mu sync.Mutex
		114	+	set []sandbox.External
		115	+	}
		116	+
		117	+	// Set replaces the set the drivers see.
		118	+	func (e *Externals) Set(set ...sandbox.External) { e.mu.Lock(); e.set = set; e.mu.Unlock() }
		119	+
		120	+	// Externals implements sandbox.ExternalsSource.
		121	+	func (e *Externals) Externals() []sandbox.External {
		122	+	e.mu.Lock()
		123	+	defer e.mu.Unlock()
		124	+	return append([]sandbox.External(nil), e.set...)
		125	+	}
		126	+
		127	+	// MakeExternal creates a host directory holding bin/node with content and
		128	+	// returns it as an external called name.
		129	+	func MakeExternal(t testing.TB, name, id, content string) sandbox.External {
		130	+	t.Helper()
		131	+	dir := filepath.Join(t.TempDir(), name)
		132	+	if err := os.MkdirAll(filepath.Join(dir, "bin"), 0o755); err != nil {
		133	+	t.Fatal(err)
		134	+	}
		135	+	if err := os.WriteFile(filepath.Join(dir, "bin", "node"), []byte(content), 0o755); err != nil {
		136	+	t.Fatal(err)
		137	+	}
		138	+	if err := os.Symlink("bin/node", filepath.Join(dir, "nodejs")); err != nil {
		139	+	t.Fatal(err)
		140	+	}
		141	+	return sandbox.External{Name: name, ID: id, Dir: dir}
		142	+	}

M internal/sandbox/vm/bootargs.go +7 -3


@@ -19,8 +19,9 @@ import (
19	19		// - root=/dev/vda ro: the exec disk; init=/ermite-runner -- init: the runner
20	20		// executable is PID 1 and runs its init subcommand;
21	21		// - ip=…: static address, gateway and DNS from the kernel, no DHCP client;
22		-	// - ermite.*: read by init from /proc/cmdline (connect target, gateway, time).
23		-	func bootArgs(guest, gateway netip.Addr, unixSeconds int64) string {
	22	+	// - ermite.*: read by init from /proc/cmdline (connect target, gateway, time,
	23	+	// and whether the externals disk is attached as the fourth drive).
	24	+	func bootArgs(guest, gateway netip.Addr, unixSeconds int64, externals bool) string {
24	25		args := []string{
25	26		"console=ttyS0", "reboot=k", "panic=1", "pci=off", "acpi=off", "8250.nr_uarts=1",
26	27		"ipv6.disable=1", "random.trust_cpu=on", "i8042.noaux", "i8042.nomux", "i8042.nopnp", "i8042.dumbkbd",

@@ -31,8 +32,11 @@ func bootArgs(guest, gateway netip.Addr, unixSeconds int64) string {
31	32		sandbox.CmdlineConnect + "=vsock:" + strconv.Itoa(sandbox.VsockPort),
32	33		sandbox.CmdlineGateway + "=" + gateway.String(),
33	34		"ermite.time=" + strconv.FormatInt(unixSeconds, 10),
34		-	"--", "init",
35	35		}
	36	+	if externals {
	37	+	args = append(args, sandbox.CmdlineExternals+"=1")
	38	+	}
	39	+	args = append(args, "--", "init")
36	40		return strings.Join(args, " ")
37	41		}
38	42

 //   - root=/dev/vda ro: the exec disk; init=/ermite-runner -- init: the runner
 //     executable is PID 1 and runs its init subcommand;
 //   - ip=…: static address, gateway and DNS from the kernel, no DHCP client;
-//   - ermite.*: read by init from /proc/cmdline (connect target, gateway, time).
+//   - ermite.*: read by init from /proc/cmdline (connect target, gateway, time,
-func bootArgs(guest, gateway netip.Addr, unixSeconds int64) string {
+//     and whether the externals disk is attached as the fourth drive).
+func bootArgs(guest, gateway netip.Addr, unixSeconds int64, externals bool) string {
 	args := []string{
 		"console=ttyS0", "reboot=k", "panic=1", "pci=off", "acpi=off", "8250.nr_uarts=1",
 		"ipv6.disable=1", "random.trust_cpu=on", "i8042.noaux", "i8042.nomux", "i8042.nopnp", "i8042.dumbkbd",
 		sandbox.CmdlineConnect + "=vsock:" + strconv.Itoa(sandbox.VsockPort),
 		sandbox.CmdlineGateway + "=" + gateway.String(),
 		"ermite.time=" + strconv.FormatInt(unixSeconds, 10),
-		"--", "init",
 	}
+	if externals {
+		args = append(args, sandbox.CmdlineExternals+"=1")
+	}
+	args = append(args, "--", "init")
 	return strings.Join(args, " ")
 }

M internal/sandbox/vm/driver.go +42 -10


@@ -68,9 +68,12 @@ type Driver struct {
68	68		fcPath string
69	69		jailer string
70	70		execDisk string
71		-	store store
72		-	net netsetup.Plumbing
73		-	lock *sandbox.HostLock
	71	+	// extMu serialises the building of the externals disk with the linking of it
	72	+	// into a sandbox (see shareExternals).
	73	+	extMu sync.Mutex
	74	+	store store
	75	+	net netsetup.Plumbing
	76	+	lock *sandbox.HostLock
74	77
75	78		mu sync.Mutex
76	79		slots *sandbox.Slots

@@ -370,7 +373,7 @@ func (s vmSandbox) bootVMM(ctx context.Context, spec sandbox.JobSpec) (apiClie
370	373		}
371	374
372	375		// vmFiles are the paths Firecracker is given.
373		-	type vmFiles struct{ execDisk, baseDisk, kernel string }
	376	+	type vmFiles struct{ execDisk, baseDisk, kernel, externals string }
374	377
375	378		// bootFiles makes the shared files available and creates the scratch disk.
376	379		func (s *vmSandbox) bootFiles(ctx context.Context, spec sandbox.JobSpec) (vmFiles, error) {

@@ -386,6 +389,7 @@ func (s *vmSandbox) bootFiles(ctx context.Context, spec sandbox.JobSpec) (vmFile
386	389		if f.kernel, err = s.lay.share(d.cfg.KernelPath(), "kernel"); err != nil {
387	390		return f, err
388	391		}
	392	+	f.externals = s.shareExternals(ctx)
389	393		owner := -1
390	394		if s.lay.jailed {
391	395		owner = s.lay.uid

@@ -393,6 +397,29 @@ func (s *vmSandbox) bootFiles(ctx context.Context, spec sandbox.JobSpec) (vmFile
393	397		return f, disk.CreateScratch(ctx, d.opt.Run, s.lay.host("scratch.ext4"), spec.DiskMiB, owner)
394	398		}
395	399
	400	+	// shareExternals makes the read-only disk of the host's external tools available
	401	+	// to Firecracker; "" means the sandbox has none. A disk that cannot be built
	402	+	// does not stop the job: it starts without the tools and the failure is logged.
	403	+	func (s *vmSandbox) shareExternals(ctx context.Context) string {
	404	+	d := s.d
	405	+	set := d.cfg.ExternalsOf()
	406	+	if len(set) == 0 {
	407	+	return ""
	408	+	}
	409	+	log := zerowrap.FromCtx(ctx)
	410	+	d.extMu.Lock()
	411	+	defer d.extMu.Unlock()
	412	+	path, err := ensureExternalsDisk(ctx, d.opt.Run, set, d.cfg.StateDir)
	413	+	if err == nil && path != "" {
	414	+	path, err = s.lay.share(path, "externals.ext4")
	415	+	}
	416	+	if err != nil {
	417	+	log.Error().Err(err).Str("vm", s.id).Msg("external tools (Node.js) not attached to this job")
	418	+	return ""
	419	+	}
	420	+	return path
	421	+	}
	422	+
396	423		// bootVsock listens on the host side of the vsock stream: Firecracker connects
397	424		// to <uds_path>_<port> when the guest connects to the host CID.
398	425		func (s *vmSandbox) bootVsock() (<-chan acceptResult, error) {

@@ -476,14 +503,19 @@ func checkJailBaseInfo(dir string, fi fs.FileInfo, euid int) error {
476	503
477	504		func (s *vmSandbox) vmConfig(spec sandbox.JobSpec, netw netpolicy.Net, f vmFiles) vmConfig {
478	505		cfg := s.d.cfg
	506	+	// The order decides the guest device names (vda, vdb, ...): new drives go last.
	507	+	drives := []drive{
	508	+	{DriveID: "rootfs", PathOnHost: f.execDisk, IsRootDevice: true, IsReadOnly: true},
	509	+	{DriveID: "base", PathOnHost: f.baseDisk, IsReadOnly: true},
	510	+	{DriveID: "scratch", PathOnHost: s.lay.fc("scratch.ext4"), RateLimiter: diskLimiter(cfg.DiskBandwidthMiB, cfg.DiskIOPS)},
	511	+	}
	512	+	if f.externals != "" {
	513	+	drives = append(drives, drive{DriveID: "externals", PathOnHost: f.externals, IsReadOnly: true})
	514	+	}
479	515		return vmConfig{
480	516		Machine: machineConfig{VcpuCount: spec.CPUs, MemSizeMiB: spec.MemoryMiB},
481		-	Boot: bootSource{KernelImagePath: f.kernel, BootArgs: bootArgs(netw.Guest, netw.Gateway, time.Now().Unix())},
482		-	Drives: []drive{
483		-	{DriveID: "rootfs", PathOnHost: f.execDisk, IsRootDevice: true, IsReadOnly: true},
484		-	{DriveID: "base", PathOnHost: f.baseDisk, IsReadOnly: true},
485		-	{DriveID: "scratch", PathOnHost: s.lay.fc("scratch.ext4"), RateLimiter: diskLimiter(cfg.DiskBandwidthMiB, cfg.DiskIOPS)},
486		-	},
	517	+	Boot: bootSource{KernelImagePath: f.kernel, BootArgs: bootArgs(netw.Guest, netw.Gateway, time.Now().Unix(), f.externals != "")},
	518	+	Drives: drives,
487	519		Net: networkInterface{
488	520		IfaceID: "eth0", HostDevName: s.tap, GuestMAC: macFor(s.slot),
489	521		RxRateLimiter: netLimiter(cfg.NetMbit), TxRateLimiter: netLimiter(cfg.NetMbit),

 	fcPath   string
 	jailer   string
 	execDisk string
-	store    store
+	// extMu serialises the building of the externals disk with the linking of it
-	net      netsetup.Plumbing
+	// into a sandbox (see shareExternals).
-	lock     *sandbox.HostLock
+	extMu sync.Mutex
+	store store
+	net   netsetup.Plumbing
+	lock  *sandbox.HostLock
 	mu    sync.Mutex
 	slots *sandbox.Slots
 }
 // vmFiles are the paths Firecracker is given.
-type vmFiles struct{ execDisk, baseDisk, kernel string }
+type vmFiles struct{ execDisk, baseDisk, kernel, externals string }
 // bootFiles makes the shared files available and creates the scratch disk.
 func (s *vmSandbox) bootFiles(ctx context.Context, spec sandbox.JobSpec) (vmFiles, error) {
 	if f.kernel, err = s.lay.share(d.cfg.KernelPath(), "kernel"); err != nil {
 		return f, err
 	}
+	f.externals = s.shareExternals(ctx)
 	owner := -1
 	if s.lay.jailed {
 		owner = s.lay.uid
 	return f, disk.CreateScratch(ctx, d.opt.Run, s.lay.host("scratch.ext4"), spec.DiskMiB, owner)
 }
+// shareExternals makes the read-only disk of the host's external tools available
+// to Firecracker; "" means the sandbox has none. A disk that cannot be built
+// does not stop the job: it starts without the tools and the failure is logged.
+func (s *vmSandbox) shareExternals(ctx context.Context) string {
+	d := s.d
+	set := d.cfg.ExternalsOf()
+	if len(set) == 0 {
+		return ""
+	}
+	log := zerowrap.FromCtx(ctx)
+	d.extMu.Lock()
+	defer d.extMu.Unlock()
+	path, err := ensureExternalsDisk(ctx, d.opt.Run, set, d.cfg.StateDir)
+	if err == nil && path != "" {
+		path, err = s.lay.share(path, "externals.ext4")
+	}
+	if err != nil {
+		log.Error().Err(err).Str("vm", s.id).Msg("external tools (Node.js) not attached to this job")
+		return ""
+	}
+	return path
+}
 // bootVsock listens on the host side of the vsock stream: Firecracker connects
 // to <uds_path>_<port> when the guest connects to the host CID.
 func (s *vmSandbox) bootVsock() (<-chan acceptResult, error) {
 func (s *vmSandbox) vmConfig(spec sandbox.JobSpec, netw netpolicy.Net, f vmFiles) vmConfig {
 	cfg := s.d.cfg
+	// The order decides the guest device names (vda, vdb, ...): new drives go last.
+	drives := []drive{
+		{DriveID: "rootfs", PathOnHost: f.execDisk, IsRootDevice: true, IsReadOnly: true},
+		{DriveID: "base", PathOnHost: f.baseDisk, IsReadOnly: true},
+		{DriveID: "scratch", PathOnHost: s.lay.fc("scratch.ext4"), RateLimiter: diskLimiter(cfg.DiskBandwidthMiB, cfg.DiskIOPS)},
+	}
+	if f.externals != "" {
+		drives = append(drives, drive{DriveID: "externals", PathOnHost: f.externals, IsReadOnly: true})
+	}
 	return vmConfig{
 		Machine: machineConfig{VcpuCount: spec.CPUs, MemSizeMiB: spec.MemoryMiB},
-		Boot:    bootSource{KernelImagePath: f.kernel, BootArgs: bootArgs(netw.Guest, netw.Gateway, time.Now().Unix())},
+		Boot:    bootSource{KernelImagePath: f.kernel, BootArgs: bootArgs(netw.Guest, netw.Gateway, time.Now().Unix(), f.externals != "")},
-		Drives: []drive{
+		Drives:  drives,
-			{DriveID: "rootfs", PathOnHost: f.execDisk, IsRootDevice: true, IsReadOnly: true},
-			{DriveID: "base", PathOnHost: f.baseDisk, IsReadOnly: true},
-			{DriveID: "scratch", PathOnHost: s.lay.fc("scratch.ext4"), RateLimiter: diskLimiter(cfg.DiskBandwidthMiB, cfg.DiskIOPS)},
-		},
 		Net: networkInterface{
 			IfaceID: "eth0", HostDevName: s.tap, GuestMAC: macFor(s.slot),
 			RxRateLimiter: netLimiter(cfg.NetMbit), TxRateLimiter: netLimiter(cfg.NetMbit),

M internal/sandbox/vm/images.go +120 -0


@@ -8,8 +8,11 @@ import (
8	8		"encoding/hex"
9	9		"fmt"
10	10		"io"
	11	+	"io/fs"
11	12		"os"
12	13		"path/filepath"
	14	+	"strconv"
	15	+	"time"
13	16
14	17		"bnema.dev/ermite-runner/internal/cmdrun"
15	18		"bnema.dev/ermite-runner/internal/sandbox"

@@ -95,3 +98,120 @@ func hashFile(path string) (sum string, size int64, err error) {
95	98		}
96	99		return hex.EncodeToString(h.Sum(nil)), n, nil
97	100		}
	101	+
	102	+	// ensureExternalsDisk builds (or reuses) the read-only ext4 disk that carries
	103	+	// the host's external tools (Node.js): one directory per tool at its root, so
	104	+	// that mounted at /home/runner/externals it gives the GitHub layout
	105	+	// externals/node24/bin/node. It is named after the content of the set
	106	+	// (sandbox.ExternalsKey), so a new Node.js version builds a new disk and leaves
	107	+	// the exec disk and the base image alone. It returns "" for an empty set. The
	108	+	// caller holds the driver's externals lock, so that two jobs do not build the
	109	+	// same disk.
	110	+	func ensureExternalsDisk(ctx context.Context, run cmdrun.Runner, set []sandbox.External, stateDir string) (string, error) {
	111	+	if len(set) == 0 {
	112	+	return "", nil
	113	+	}
	114	+	cache := filepath.Join(stateDir, "cache")
	115	+	if err := os.MkdirAll(cache, 0o755); err != nil {
	116	+	return "", err
	117	+	}
	118	+	name := "extdisk-" + sandbox.ExternalsKey(set) + ".ext4"
	119	+	dest := filepath.Join(cache, name)
	120	+	if _, err := os.Stat(dest); err == nil {
	121	+	return dest, nil
	122	+	}
	123	+	// Drop disks of other contents, once they are an hour old: without the
	124	+	// jailer a VM that is about to start may still be about to open the previous
	125	+	// one in place (with the jailer it holds a hard link).
	126	+	olds, _ := filepath.Glob(filepath.Join(cache, "extdisk-*"))
	127	+	for _, o := range olds {
	128	+	if fi, err := os.Stat(o); err == nil && time.Since(fi.ModTime()) > time.Hour {
	129	+	_ = os.Remove(o)
	130	+	}
	131	+	}
	132	+
	133	+	tmp, err := os.MkdirTemp(cache, "extroot-")
	134	+	if err != nil {
	135	+	return "", err
	136	+	}
	137	+	defer func() { _ = os.RemoveAll(tmp) }()
	138	+	var size, files int64
	139	+	for _, e := range set {
	140	+	sz, n, err := cloneTree(e.Dir, filepath.Join(tmp, e.Name))
	141	+	if err != nil {
	142	+	return "", fmt.Errorf("stage %s: %w", e.Name, err)
	143	+	}
	144	+	size, files = size+sz, files+n
	145	+	}
	146	+
	147	+	part := dest + ".part"
	148	+	_ = os.Remove(part)
	149	+	f, err := os.OpenFile(part, os.O_WRONLY\|os.O_CREATE\|os.O_EXCL, 0o644)
	150	+	if err != nil {
	151	+	return "", err
	152	+	}
	153	+	// The files, ext4 overhead (inodes, directories, block rounding of many
	154	+	// small files) and some room; the disk is read-only and sparse.
	155	+	mib := size/(1<<20)125/100 + files4096/(1<<20) + 16
	156	+	if err := f.Truncate(mib << 20); err != nil {
	157	+	_ = f.Close()
	158	+	return "", err
	159	+	}
	160	+	_ = f.Close()
	161	+	if _, err := run.Run(ctx, nil, "mkfs.ext4", "-q", "-F", "-L", "ermite-ext", "-m", "0",
	162	+	"-O", "^has_journal", "-N", strconv.FormatInt(files*13/10+256, 10),
	163	+	"-E", "root_owner=0:0", "-d", tmp, part); err != nil {
	164	+	_ = os.Remove(part)
	165	+	return "", fmt.Errorf("build externals disk: %w", err)
	166	+	}
	167	+	if err := os.Rename(part, dest); err != nil {
	168	+	return "", err
	169	+	}
	170	+	return dest, nil
	171	+	}
	172	+
	173	+	// cloneTree recreates the tree src at dst (directories, regular files, symbolic
	174	+	// links) for mkfs.ext4 -d, hard-linking files when the filesystem allows and
	175	+	// copying them otherwise. It returns the bytes of file content and the number
	176	+	// of entries.
	177	+	func cloneTree(src, dst string) (size, entries int64, err error) {
	178	+	err = filepath.WalkDir(src, func(p string, d fs.DirEntry, werr error) error {
	179	+	if werr != nil {
	180	+	return werr
	181	+	}
	182	+	rel, err := filepath.Rel(src, p)
	183	+	if err != nil {
	184	+	return err
	185	+	}
	186	+	to := filepath.Join(dst, rel)
	187	+	entries++
	188	+	switch {
	189	+	case d.IsDir():
	190	+	if err := os.MkdirAll(to, 0o755); err != nil {
	191	+	return err
	192	+	}
	193	+	return os.Chmod(to, 0o755) // whatever the umask
	194	+	case d.Type()&fs.ModeSymlink != 0:
	195	+	target, err := os.Readlink(p)
	196	+	if err != nil {
	197	+	return err
	198	+	}
	199	+	return os.Symlink(target, to)
	200	+	case d.Type().IsRegular():
	201	+	fi, err := d.Info()
	202	+	if err != nil {
	203	+	return err
	204	+	}
	205	+	size += fi.Size()
	206	+	if err := os.Link(p, to); err == nil {
	207	+	return nil
	208	+	}
	209	+	if err := copyFile(p, to); err != nil {
	210	+	return err
	211	+	}
	212	+	return os.Chmod(to, fi.Mode().Perm())
	213	+	}
	214	+	return fmt.Errorf("%s: unsupported file type %v", p, d.Type())
	215	+	})
	216	+	return size, entries, err
	217	+	}


@@ -8,8 +8,11 @@ import (
8	"encoding/hex"	8		"encoding/hex"
9	"fmt"	9		"fmt"
10	"io"	10		"io"
		11	+	"io/fs"
11	"os"	12		"os"
12	"path/filepath"	13		"path/filepath"
		14	+	"strconv"
		15	+	"time"
13		16
14	"bnema.dev/ermite-runner/internal/cmdrun"	17		"bnema.dev/ermite-runner/internal/cmdrun"
15	"bnema.dev/ermite-runner/internal/sandbox"	18		"bnema.dev/ermite-runner/internal/sandbox"

@@ -95,3 +98,120 @@ func hashFile(path string) (sum string, size int64, err error) {
95	}	98		}
96	return hex.EncodeToString(h.Sum(nil)), n, nil	99		return hex.EncodeToString(h.Sum(nil)), n, nil
97	}	100		}
		101	+
		102	+	// ensureExternalsDisk builds (or reuses) the read-only ext4 disk that carries
		103	+	// the host's external tools (Node.js): one directory per tool at its root, so
		104	+	// that mounted at /home/runner/externals it gives the GitHub layout
		105	+	// externals/node24/bin/node. It is named after the content of the set
		106	+	// (sandbox.ExternalsKey), so a new Node.js version builds a new disk and leaves
		107	+	// the exec disk and the base image alone. It returns "" for an empty set. The
		108	+	// caller holds the driver's externals lock, so that two jobs do not build the
		109	+	// same disk.
		110	+	func ensureExternalsDisk(ctx context.Context, run cmdrun.Runner, set []sandbox.External, stateDir string) (string, error) {
		111	+	if len(set) == 0 {
		112	+	return "", nil
		113	+	}
		114	+	cache := filepath.Join(stateDir, "cache")
		115	+	if err := os.MkdirAll(cache, 0o755); err != nil {
		116	+	return "", err
		117	+	}
		118	+	name := "extdisk-" + sandbox.ExternalsKey(set) + ".ext4"
		119	+	dest := filepath.Join(cache, name)
		120	+	if _, err := os.Stat(dest); err == nil {
		121	+	return dest, nil
		122	+	}
		123	+	// Drop disks of other contents, once they are an hour old: without the
		124	+	// jailer a VM that is about to start may still be about to open the previous
		125	+	// one in place (with the jailer it holds a hard link).
		126	+	olds, _ := filepath.Glob(filepath.Join(cache, "extdisk-*"))
		127	+	for _, o := range olds {
		128	+	if fi, err := os.Stat(o); err == nil && time.Since(fi.ModTime()) > time.Hour {
		129	+	_ = os.Remove(o)
		130	+	}
		131	+	}
		132	+
		133	+	tmp, err := os.MkdirTemp(cache, "extroot-")
		134	+	if err != nil {
		135	+	return "", err
		136	+	}
		137	+	defer func() { _ = os.RemoveAll(tmp) }()
		138	+	var size, files int64
		139	+	for _, e := range set {
		140	+	sz, n, err := cloneTree(e.Dir, filepath.Join(tmp, e.Name))
		141	+	if err != nil {
		142	+	return "", fmt.Errorf("stage %s: %w", e.Name, err)
		143	+	}
		144	+	size, files = size+sz, files+n
		145	+	}
		146	+
		147	+	part := dest + ".part"
		148	+	_ = os.Remove(part)
		149	+	f, err := os.OpenFile(part, os.O_WRONLY\|os.O_CREATE\|os.O_EXCL, 0o644)
		150	+	if err != nil {
		151	+	return "", err
		152	+	}
		153	+	// The files, ext4 overhead (inodes, directories, block rounding of many
		154	+	// small files) and some room; the disk is read-only and sparse.
		155	+	mib := size/(1<<20)125/100 + files4096/(1<<20) + 16
		156	+	if err := f.Truncate(mib << 20); err != nil {
		157	+	_ = f.Close()
		158	+	return "", err
		159	+	}
		160	+	_ = f.Close()
		161	+	if _, err := run.Run(ctx, nil, "mkfs.ext4", "-q", "-F", "-L", "ermite-ext", "-m", "0",
		162	+	"-O", "^has_journal", "-N", strconv.FormatInt(files*13/10+256, 10),
		163	+	"-E", "root_owner=0:0", "-d", tmp, part); err != nil {
		164	+	_ = os.Remove(part)
		165	+	return "", fmt.Errorf("build externals disk: %w", err)
		166	+	}
		167	+	if err := os.Rename(part, dest); err != nil {
		168	+	return "", err
		169	+	}
		170	+	return dest, nil
		171	+	}
		172	+
		173	+	// cloneTree recreates the tree src at dst (directories, regular files, symbolic
		174	+	// links) for mkfs.ext4 -d, hard-linking files when the filesystem allows and
		175	+	// copying them otherwise. It returns the bytes of file content and the number
		176	+	// of entries.
		177	+	func cloneTree(src, dst string) (size, entries int64, err error) {
		178	+	err = filepath.WalkDir(src, func(p string, d fs.DirEntry, werr error) error {
		179	+	if werr != nil {
		180	+	return werr
		181	+	}
		182	+	rel, err := filepath.Rel(src, p)
		183	+	if err != nil {
		184	+	return err
		185	+	}
		186	+	to := filepath.Join(dst, rel)
		187	+	entries++
		188	+	switch {
		189	+	case d.IsDir():
		190	+	if err := os.MkdirAll(to, 0o755); err != nil {
		191	+	return err
		192	+	}
		193	+	return os.Chmod(to, 0o755) // whatever the umask
		194	+	case d.Type()&fs.ModeSymlink != 0:
		195	+	target, err := os.Readlink(p)
		196	+	if err != nil {
		197	+	return err
		198	+	}
		199	+	return os.Symlink(target, to)
		200	+	case d.Type().IsRegular():
		201	+	fi, err := d.Info()
		202	+	if err != nil {
		203	+	return err
		204	+	}
		205	+	size += fi.Size()
		206	+	if err := os.Link(p, to); err == nil {
		207	+	return nil
		208	+	}
		209	+	if err := copyFile(p, to); err != nil {
		210	+	return err
		211	+	}
		212	+	return os.Chmod(to, fi.Mode().Perm())
		213	+	}
		214	+	return fmt.Errorf("%s: unsupported file type %v", p, d.Type())
		215	+	})
		216	+	return size, entries, err
		217	+	}

M internal/sandbox/vm/vm_test.go +182 -1


@@ -621,7 +621,7 @@ func TestJailedShareHardLinks(t *testing.T) {
621	621		}
622	622
623	623		func TestBootArgs(t *testing.T) {
624		-	a := bootArgs(mustAddr("10.200.0.6"), mustAddr("10.200.0.5"), 1791818522)
	624	+	a := bootArgs(mustAddr("10.200.0.6"), mustAddr("10.200.0.5"), 1791818522, false)
625	625		for _, want := range []string{
626	626		"ipv6.disable=1", "root=/dev/vda ro", "init=/ermite-runner", "reboot=k", "panic=1",
627	627		"ip=10.200.0.6::10.200.0.5:255.255.255.252:ermite-job:eth0:off:10.200.0.5",

@@ -634,6 +634,12 @@ func TestBootArgs(t *testing.T) {
634	634		if !strings.HasSuffix(a, "-- init") {
635	635		t.Errorf("init subcommand missing: %s", a)
636	636		}
	637	+	if strings.Contains(a, "ermite.externals") {
	638	+	t.Errorf("externals announced without a disk: %s", a)
	639	+	}
	640	+	if b := bootArgs(mustAddr("10.200.0.6"), mustAddr("10.200.0.5"), 1791818522, true); !strings.Contains(b, " ermite.externals=1 -- init") {
	641	+	t.Errorf("externals not announced: %s", b)
	642	+	}
637	643		if macFor(0) != "06:00:ac:10:00:00" \|\| macFor(257) != "06:00:ac:10:01:01" {
638	644		t.Fatal(macFor(257))
639	645		}

@@ -1348,3 +1354,178 @@ func TestGuestForgeIsAttachedAndWarnedAtEveryJob(t *testing.T) {
1348	1354		t.Fatalf("the warning must be logged at every job start (1 + 2), got %d", n)
1349	1355		}
1350	1356		}
	1357	+
	1358	+	// The host's external tools ride on a fourth, read-only drive, keyed by their
	1359	+	// content: a new Node.js builds a new disk and leaves the exec disk alone.
	1360	+	func TestExternalsDiskIsAttachedReadOnlyAndKeyedByContent(t *testing.T) {
	1361	+	e := newEnv(t, "")
	1362	+	src := &sandboxtest.Externals{}
	1363	+	n24 := sandboxtest.MakeExternal(t, "node24", "node24-24.21.0-x64-aaaa", "v24.21.0")
	1364	+	n20 := sandboxtest.MakeExternal(t, "node20", "node20-20.20.2-x64-bbbb", "v20.20.2")
	1365	+	src.Set(n24, n20)
	1366	+	e.cfg.Externals = src
	1367	+	d := e.driver(t)
	1368	+	execDisk := d.execDisk
	1369	+
	1370	+	boot := func() (drives []string, log string) {
	1371	+	t.Helper()
	1372	+	_ = os.Remove(e.log)
	1373	+	sb, _ := d.New()
	1374	+	conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
	1375	+	if err != nil {
	1376	+	t.Fatal(err)
	1377	+	}
	1378	+	runStub(t, conn)
	1379	+	defer func() { _ = sb.Stop(context.Background()) }()
	1380	+	b, _ := os.ReadFile(e.log)
	1381	+	log = string(b)
	1382	+	for _, l := range strings.Split(log, "\n") {
	1383	+	if f := strings.Fields(l); len(f) >= 2 && f[0] == "PUT" && strings.HasPrefix(f[1], "/drives/") {
	1384	+	drives = append(drives, f[1])
	1385	+	}
	1386	+	}
	1387	+	return drives, log
	1388	+	}
	1389	+	drives, log := boot()
	1390	+	if strings.Join(drives, ",") != "/drives/rootfs,/drives/base,/drives/scratch,/drives/externals" {
	1391	+	t.Fatalf("drives: %v", drives)
	1392	+	}
	1393	+	if !strings.Contains(log, `"drive_id":"externals"`) \|\| !strings.Contains(log, `"is_root_device":false,"is_read_only":true`) \|\| !strings.Contains(log, "ermite.externals=1") {
	1394	+	t.Fatalf("API log:\n%s", log)
	1395	+	}
	1396	+	// The scratch drive is still the third (vdc), and the only writable one.
	1397	+	if strings.Count(log, `"is_read_only":false`) != 1 {
	1398	+	t.Errorf("only the scratch drive is writable:\n%s", log)
	1399	+	}
	1400	+	first, _ := filepath.Glob(filepath.Join(e.cfg.StateDir, "cache", "extdisk-*"))
	1401	+	if len(first) != 1 {
	1402	+	t.Fatalf("externals disks: %v", first)
	1403	+	}
	1404	+	mkfs := func() (n int) {
	1405	+	for _, c := range e.run.Calls {
	1406	+	if c.Name == "mkfs.ext4" && len(c.Args) > 1 {
	1407	+	for _, a := range c.Args {
	1408	+	if a == "ermite-ext" {
	1409	+	n++
	1410	+	}
	1411	+	}
	1412	+	}
	1413	+	}
	1414	+	return
	1415	+	}
	1416	+	if mkfs() != 1 {
	1417	+	t.Fatalf("externals disk built %d times", mkfs())
	1418	+	}
	1419	+
	1420	+	// A second job with the same tools reuses the disk.
	1421	+	boot()
	1422	+	if mkfs() != 1 {
	1423	+	t.Fatal("externals disk rebuilt for the same content")
	1424	+	}
	1425	+
	1426	+	// A new Node.js: another disk, and the exec disk is untouched.
	1427	+	src.Set(sandboxtest.MakeExternal(t, "node24", "node24-24.22.0-x64-cccc", "v24.22.0"), n20)
	1428	+	boot()
	1429	+	if mkfs() != 2 {
	1430	+	t.Fatal("externals disk not rebuilt for a new version")
	1431	+	}
	1432	+	if d.execDisk != execDisk {
	1433	+	t.Fatal("exec disk changed")
	1434	+	}
	1435	+	if _, err := os.Stat(execDisk); err != nil {
	1436	+	t.Fatal("exec disk removed")
	1437	+	}
	1438	+	if now, _ := filepath.Glob(filepath.Join(e.cfg.StateDir, "cache", "extdisk-*")); len(now) != 2 \|\| (now[0] != first[0] && now[1] != first[0]) {
	1439	+	// The previous disk may still be linked by a starting VM: kept for an hour.
	1440	+	t.Fatalf("externals disks: %v", now)
	1441	+	}
	1442	+	}
	1443	+
	1444	+	func TestNoExternalsMeansThreeDrives(t *testing.T) {
	1445	+	for _, src := range []sandbox.ExternalsSource{nil, &sandboxtest.Externals{}} {
	1446	+	e := newEnv(t, "")
	1447	+	e.cfg.Externals = src
	1448	+	d := e.driver(t)
	1449	+	sb, _ := d.New()
	1450	+	conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
	1451	+	if err != nil {
	1452	+	t.Fatal(err)
	1453	+	}
	1454	+	runStub(t, conn)
	1455	+	b, _ := os.ReadFile(e.log)
	1456	+	if strings.Contains(string(b), "externals") {
	1457	+	t.Errorf("externals attached without any:\n%s", b)
	1458	+	}
	1459	+	_ = sb.Stop(context.Background())
	1460	+	if left, _ := filepath.Glob(filepath.Join(e.cfg.StateDir, "cache", "extdisk-*")); len(left) != 0 {
	1461	+	t.Errorf("disk built for nothing: %v", left)
	1462	+	}
	1463	+	}
	1464	+	}
	1465	+
	1466	+	func TestExternalsDiskFailureDoesNotStopTheJob(t *testing.T) {
	1467	+	e := newEnv(t, "")
	1468	+	src := &sandboxtest.Externals{}
	1469	+	src.Set(sandbox.External{Name: "node24", ID: "x", Dir: filepath.Join(t.TempDir(), "missing")})
	1470	+	e.cfg.Externals = src
	1471	+	var buf bytes.Buffer
	1472	+	ctx := zerowrap.WithCtx(context.Background(), zerowrap.New(zerowrap.Config{Level: "debug", Format: "json", Output: &buf}))
	1473	+	d := e.driver(t)
	1474	+	sb, _ := d.New()
	1475	+	conn, err := sb.Start(ctx, sandbox.JobSpec{JobID: "j"})
	1476	+	if err != nil {
	1477	+	t.Fatalf("a broken externals set must not stop the job: %v", err)
	1478	+	}
	1479	+	runStub(t, conn)
	1480	+	_ = sb.Stop(ctx)
	1481	+	if !strings.Contains(buf.String(), "external tools (Node.js) not attached") {
	1482	+	t.Fatalf("failure not logged: %s", buf.String())
	1483	+	}
	1484	+	b, _ := os.ReadFile(e.log)
	1485	+	if strings.Contains(string(b), `"externals"`) {
	1486	+	t.Fatal("drive attached after a failure")
	1487	+	}
	1488	+	}
	1489	+
	1490	+	// With the real mkfs.ext4 the disk holds <name>/bin/node at its root, which the
	1491	+	// guest mounts at /home/runner/externals.
	1492	+	func TestExternalsDiskContent(t *testing.T) {
	1493	+	if _, err := exec.LookPath("mkfs.ext4"); err != nil {
	1494	+	t.Skip("mkfs.ext4 needed")
	1495	+	}
	1496	+	dbg, err := exec.LookPath("debugfs")
	1497	+	if err != nil {
	1498	+	t.Skip("debugfs needed")
	1499	+	}
	1500	+	set := []sandbox.External{
	1501	+	sandboxtest.MakeExternal(t, "node24", "a", "v24"),
	1502	+	sandboxtest.MakeExternal(t, "node20", "b", "v20"),
	1503	+	}
	1504	+	path, err := ensureExternalsDisk(context.Background(), cmdrun.Exec{}, set, t.TempDir())
	1505	+	if err != nil {
	1506	+	t.Fatal(err)
	1507	+	}
	1508	+	out := func(cmd string) string {
	1509	+	b, err := exec.Command(dbg, "-R", cmd, path).CombinedOutput()
	1510	+	if err != nil {
	1511	+	t.Fatalf("debugfs %s: %v\n%s", cmd, err, b)
	1512	+	}
	1513	+	_, body, _ := strings.Cut(string(b), "\n") // the banner line
	1514	+	return body
	1515	+	}
	1516	+	if ls := out("ls -l /"); !strings.Contains(ls, "node24") \|\| !strings.Contains(ls, "node20") {
	1517	+	t.Fatalf("root:\n%s", ls)
	1518	+	}
	1519	+	if got := out("cat /node24/bin/node"); got != "v24" {
	1520	+	t.Fatalf("node24/bin/node: %q", got)
	1521	+	}
	1522	+	if ls := out("stat /node20/bin/node"); !strings.Contains(ls, "Mode: 0755") {
	1523	+	t.Fatalf("mode:\n%s", ls)
	1524	+	}
	1525	+	if ls := out("stat /node20/nodejs"); !strings.Contains(ls, "Type: symlink") \|\| !strings.Contains(ls, `"bin/node"`) {
	1526	+	t.Fatalf("symlink:\n%s", ls)
	1527	+	}
	1528	+	if st, _ := os.Stat(path); st.Size() > 64<<20 {
	1529	+	t.Fatalf("a tiny tree gave a %d byte disk", st.Size())
	1530	+	}
	1531	+	}


@@ -621,7 +621,7 @@ func TestJailedShareHardLinks(t *testing.T) {
621		}	621		}
622			622
623		func TestBootArgs(t *testing.T) {	623		func TestBootArgs(t *testing.T) {
624	-	a := bootArgs(mustAddr("10.200.0.6"), mustAddr("10.200.0.5"), 1791818522)	624	+	a := bootArgs(mustAddr("10.200.0.6"), mustAddr("10.200.0.5"), 1791818522, false)
625		for _, want := range []string{	625		for _, want := range []string{
626		"ipv6.disable=1", "root=/dev/vda ro", "init=/ermite-runner", "reboot=k", "panic=1",	626		"ipv6.disable=1", "root=/dev/vda ro", "init=/ermite-runner", "reboot=k", "panic=1",
627		"ip=10.200.0.6::10.200.0.5:255.255.255.252:ermite-job:eth0:off:10.200.0.5",	627		"ip=10.200.0.6::10.200.0.5:255.255.255.252:ermite-job:eth0:off:10.200.0.5",

@@ -634,6 +634,12 @@ func TestBootArgs(t *testing.T) {
634		if !strings.HasSuffix(a, "-- init") {	634		if !strings.HasSuffix(a, "-- init") {
635		t.Errorf("init subcommand missing: %s", a)	635		t.Errorf("init subcommand missing: %s", a)
636		}	636		}
			637	+	if strings.Contains(a, "ermite.externals") {
			638	+	t.Errorf("externals announced without a disk: %s", a)
			639	+	}
			640	+	if b := bootArgs(mustAddr("10.200.0.6"), mustAddr("10.200.0.5"), 1791818522, true); !strings.Contains(b, " ermite.externals=1 -- init") {
			641	+	t.Errorf("externals not announced: %s", b)
			642	+	}
637		if macFor(0) != "06:00:ac:10:00:00" \|\| macFor(257) != "06:00:ac:10:01:01" {	643		if macFor(0) != "06:00:ac:10:00:00" \|\| macFor(257) != "06:00:ac:10:01:01" {
638		t.Fatal(macFor(257))	644		t.Fatal(macFor(257))
639		}	645		}

@@ -1348,3 +1354,178 @@ func TestGuestForgeIsAttachedAndWarnedAtEveryJob(t *testing.T) {
1348		t.Fatalf("the warning must be logged at every job start (1 + 2), got %d", n)	1354		t.Fatalf("the warning must be logged at every job start (1 + 2), got %d", n)
1349		}	1355		}
1350		}	1356		}
			1357	+
			1358	+	// The host's external tools ride on a fourth, read-only drive, keyed by their
			1359	+	// content: a new Node.js builds a new disk and leaves the exec disk alone.
			1360	+	func TestExternalsDiskIsAttachedReadOnlyAndKeyedByContent(t *testing.T) {
			1361	+	e := newEnv(t, "")
			1362	+	src := &sandboxtest.Externals{}
			1363	+	n24 := sandboxtest.MakeExternal(t, "node24", "node24-24.21.0-x64-aaaa", "v24.21.0")
			1364	+	n20 := sandboxtest.MakeExternal(t, "node20", "node20-20.20.2-x64-bbbb", "v20.20.2")
			1365	+	src.Set(n24, n20)
			1366	+	e.cfg.Externals = src
			1367	+	d := e.driver(t)
			1368	+	execDisk := d.execDisk
			1369	+
			1370	+	boot := func() (drives []string, log string) {
			1371	+	t.Helper()
			1372	+	_ = os.Remove(e.log)
			1373	+	sb, _ := d.New()
			1374	+	conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
			1375	+	if err != nil {
			1376	+	t.Fatal(err)
			1377	+	}
			1378	+	runStub(t, conn)
			1379	+	defer func() { _ = sb.Stop(context.Background()) }()
			1380	+	b, _ := os.ReadFile(e.log)
			1381	+	log = string(b)
			1382	+	for _, l := range strings.Split(log, "\n") {
			1383	+	if f := strings.Fields(l); len(f) >= 2 && f[0] == "PUT" && strings.HasPrefix(f[1], "/drives/") {
			1384	+	drives = append(drives, f[1])
			1385	+	}
			1386	+	}
			1387	+	return drives, log
			1388	+	}
			1389	+	drives, log := boot()
			1390	+	if strings.Join(drives, ",") != "/drives/rootfs,/drives/base,/drives/scratch,/drives/externals" {
			1391	+	t.Fatalf("drives: %v", drives)
			1392	+	}
			1393	+	if !strings.Contains(log, `"drive_id":"externals"`) \|\| !strings.Contains(log, `"is_root_device":false,"is_read_only":true`) \|\| !strings.Contains(log, "ermite.externals=1") {
			1394	+	t.Fatalf("API log:\n%s", log)
			1395	+	}
			1396	+	// The scratch drive is still the third (vdc), and the only writable one.
			1397	+	if strings.Count(log, `"is_read_only":false`) != 1 {
			1398	+	t.Errorf("only the scratch drive is writable:\n%s", log)
			1399	+	}
			1400	+	first, _ := filepath.Glob(filepath.Join(e.cfg.StateDir, "cache", "extdisk-*"))
			1401	+	if len(first) != 1 {
			1402	+	t.Fatalf("externals disks: %v", first)
			1403	+	}
			1404	+	mkfs := func() (n int) {
			1405	+	for _, c := range e.run.Calls {
			1406	+	if c.Name == "mkfs.ext4" && len(c.Args) > 1 {
			1407	+	for _, a := range c.Args {
			1408	+	if a == "ermite-ext" {
			1409	+	n++
			1410	+	}
			1411	+	}
			1412	+	}
			1413	+	}
			1414	+	return
			1415	+	}
			1416	+	if mkfs() != 1 {
			1417	+	t.Fatalf("externals disk built %d times", mkfs())
			1418	+	}
			1419	+
			1420	+	// A second job with the same tools reuses the disk.
			1421	+	boot()
			1422	+	if mkfs() != 1 {
			1423	+	t.Fatal("externals disk rebuilt for the same content")
			1424	+	}
			1425	+
			1426	+	// A new Node.js: another disk, and the exec disk is untouched.
			1427	+	src.Set(sandboxtest.MakeExternal(t, "node24", "node24-24.22.0-x64-cccc", "v24.22.0"), n20)
			1428	+	boot()
			1429	+	if mkfs() != 2 {
			1430	+	t.Fatal("externals disk not rebuilt for a new version")
			1431	+	}
			1432	+	if d.execDisk != execDisk {
			1433	+	t.Fatal("exec disk changed")
			1434	+	}
			1435	+	if _, err := os.Stat(execDisk); err != nil {
			1436	+	t.Fatal("exec disk removed")
			1437	+	}
			1438	+	if now, _ := filepath.Glob(filepath.Join(e.cfg.StateDir, "cache", "extdisk-*")); len(now) != 2 \|\| (now[0] != first[0] && now[1] != first[0]) {
			1439	+	// The previous disk may still be linked by a starting VM: kept for an hour.
			1440	+	t.Fatalf("externals disks: %v", now)
			1441	+	}
			1442	+	}
			1443	+
			1444	+	func TestNoExternalsMeansThreeDrives(t *testing.T) {
			1445	+	for _, src := range []sandbox.ExternalsSource{nil, &sandboxtest.Externals{}} {
			1446	+	e := newEnv(t, "")
			1447	+	e.cfg.Externals = src
			1448	+	d := e.driver(t)
			1449	+	sb, _ := d.New()
			1450	+	conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
			1451	+	if err != nil {
			1452	+	t.Fatal(err)
			1453	+	}
			1454	+	runStub(t, conn)
			1455	+	b, _ := os.ReadFile(e.log)
			1456	+	if strings.Contains(string(b), "externals") {
			1457	+	t.Errorf("externals attached without any:\n%s", b)
			1458	+	}
			1459	+	_ = sb.Stop(context.Background())
			1460	+	if left, _ := filepath.Glob(filepath.Join(e.cfg.StateDir, "cache", "extdisk-*")); len(left) != 0 {
			1461	+	t.Errorf("disk built for nothing: %v", left)
			1462	+	}
			1463	+	}
			1464	+	}
			1465	+
			1466	+	func TestExternalsDiskFailureDoesNotStopTheJob(t *testing.T) {
			1467	+	e := newEnv(t, "")
			1468	+	src := &sandboxtest.Externals{}
			1469	+	src.Set(sandbox.External{Name: "node24", ID: "x", Dir: filepath.Join(t.TempDir(), "missing")})
			1470	+	e.cfg.Externals = src
			1471	+	var buf bytes.Buffer
			1472	+	ctx := zerowrap.WithCtx(context.Background(), zerowrap.New(zerowrap.Config{Level: "debug", Format: "json", Output: &buf}))
			1473	+	d := e.driver(t)
			1474	+	sb, _ := d.New()
			1475	+	conn, err := sb.Start(ctx, sandbox.JobSpec{JobID: "j"})
			1476	+	if err != nil {
			1477	+	t.Fatalf("a broken externals set must not stop the job: %v", err)
			1478	+	}
			1479	+	runStub(t, conn)
			1480	+	_ = sb.Stop(ctx)
			1481	+	if !strings.Contains(buf.String(), "external tools (Node.js) not attached") {
			1482	+	t.Fatalf("failure not logged: %s", buf.String())
			1483	+	}
			1484	+	b, _ := os.ReadFile(e.log)
			1485	+	if strings.Contains(string(b), `"externals"`) {
			1486	+	t.Fatal("drive attached after a failure")
			1487	+	}
			1488	+	}
			1489	+
			1490	+	// With the real mkfs.ext4 the disk holds <name>/bin/node at its root, which the
			1491	+	// guest mounts at /home/runner/externals.
			1492	+	func TestExternalsDiskContent(t *testing.T) {
			1493	+	if _, err := exec.LookPath("mkfs.ext4"); err != nil {
			1494	+	t.Skip("mkfs.ext4 needed")
			1495	+	}
			1496	+	dbg, err := exec.LookPath("debugfs")
			1497	+	if err != nil {
			1498	+	t.Skip("debugfs needed")
			1499	+	}
			1500	+	set := []sandbox.External{
			1501	+	sandboxtest.MakeExternal(t, "node24", "a", "v24"),
			1502	+	sandboxtest.MakeExternal(t, "node20", "b", "v20"),
			1503	+	}
			1504	+	path, err := ensureExternalsDisk(context.Background(), cmdrun.Exec{}, set, t.TempDir())
			1505	+	if err != nil {
			1506	+	t.Fatal(err)
			1507	+	}
			1508	+	out := func(cmd string) string {
			1509	+	b, err := exec.Command(dbg, "-R", cmd, path).CombinedOutput()
			1510	+	if err != nil {
			1511	+	t.Fatalf("debugfs %s: %v\n%s", cmd, err, b)
			1512	+	}
			1513	+	_, body, _ := strings.Cut(string(b), "\n") // the banner line
			1514	+	return body
			1515	+	}
			1516	+	if ls := out("ls -l /"); !strings.Contains(ls, "node24") \|\| !strings.Contains(ls, "node20") {
			1517	+	t.Fatalf("root:\n%s", ls)
			1518	+	}
			1519	+	if got := out("cat /node24/bin/node"); got != "v24" {
			1520	+	t.Fatalf("node24/bin/node: %q", got)
			1521	+	}
			1522	+	if ls := out("stat /node20/bin/node"); !strings.Contains(ls, "Mode: 0755") {
			1523	+	t.Fatalf("mode:\n%s", ls)
			1524	+	}
			1525	+	if ls := out("stat /node20/nodejs"); !strings.Contains(ls, "Type: symlink") \|\| !strings.Contains(ls, `"bin/node"`) {
			1526	+	t.Fatalf("symlink:\n%s", ls)
			1527	+	}
			1528	+	if st, _ := os.Stat(path); st.Size() > 64<<20 {
			1529	+	t.Fatalf("a tiny tree gave a %d byte disk", st.Size())
			1530	+	}
			1531	+	}

M sandbox_check.go +1 -0


@@ -29,6 +29,7 @@ func sandboxCheck(ctx context.Context, args []string) int {
29	29		fmt.Fprintln(os.Stderr, "ermite-runner:", err)
30	30		return 2
31	31		}
	32	+	setupExternals(ctx, &cfg)
32	33		drv, err := sandbox.Select(ctx, cfg, drivers.Factories(), sandbox.Prober{})
33	34		if err != nil {
34	35		fmt.Fprintln(os.Stderr, "ermite-runner:", err)


@@ -29,6 +29,7 @@ func sandboxCheck(ctx context.Context, args []string) int {
29	fmt.Fprintln(os.Stderr, "ermite-runner:", err)	29		fmt.Fprintln(os.Stderr, "ermite-runner:", err)
30	return 2	30		return 2
31	}	31		}
		32	+	setupExternals(ctx, &cfg)
32	drv, err := sandbox.Select(ctx, cfg, drivers.Factories(), sandbox.Prober{})	33		drv, err := sandbox.Select(ctx, cfg, drivers.Factories(), sandbox.Prober{})
33	if err != nil {	34		if err != nil {
34	fmt.Fprintln(os.Stderr, "ermite-runner:", err)	35		fmt.Fprintln(os.Stderr, "ermite-runner:", err)