Merge pull request 'feat(runner): executor, host loop and job frames' (#2) from feat/executor into main

commit: e917abea447b9a2a1a517387e85b0644cb009c36
author: bnema · 10d ago · 2026-10-12 20:28 UTC
merge of: 9b7473e + b0c2e21 · diff against the first parent
signature: SSH UnverifiedSHA256:VOCA5lYARHcbKDZ7GhTV45VmcjWnJ6ZQIBazj17LivA The key is unknown

53 files · +12165 -87

M .forgejo/workflows/ci.yml +15 -0


@@ -16,6 +16,10 @@ jobs:
16	16		runs-on: ubuntu-latest
17	17		steps:
18	18		- uses: actions/checkout@v6
	19	+	# Job containers reach Forgejo through the Gordon edge on the host loopback,
	20	+	# in plain HTTP on :8088 (see vps-infra homelab/igor/ci-runner).
	21	+	- name: Fetch bnema.dev/ermite* modules through the runner's Forgejo route
	22	+	run: git config --global url."http://git.bnema.dev:8088/".insteadOf "https://git.bnema.dev/"
19	23		- uses: actions/setup-go@v5
20	24		with:
21	25		go-version-file: go.mod

@@ -29,6 +33,10 @@ jobs:
29	33		runs-on: ubuntu-latest
30	34		steps:
31	35		- uses: actions/checkout@v6
	36	+	# Job containers reach Forgejo through the Gordon edge on the host loopback,
	37	+	# in plain HTTP on :8088 (see vps-infra homelab/igor/ci-runner).
	38	+	- name: Fetch bnema.dev/ermite* modules through the runner's Forgejo route
	39	+	run: git config --global url."http://git.bnema.dev:8088/".insteadOf "https://git.bnema.dev/"
32	40		- uses: actions/setup-go@v5
33	41		with:
34	42		go-version-file: go.mod

@@ -38,6 +46,13 @@ jobs:
38	46		go mod tidy
39	47		git diff --exit-code -- go.mod go.sum
40	48		- run: go vet ./...
	49	+	- name: Static build
	50	+	# The runner binary is attached read-only to every sandbox, which has no
	51	+	# libc of ours: it must be a static executable (sandbox.CheckStatic
	52	+	# refuses anything else at start).
	53	+	run: \|
	54	+	CGO_ENABLED=0 go build -trimpath -o /tmp/ermite-runner .
	55	+	file /tmp/ermite-runner \| grep -q 'statically linked'
41	56		- name: Protobuf lint and drift
42	57		# buf and protoc-gen-go are pinned as `tool` directives in go.mod: the
43	58		# generated code is reproducible and must match the committed one.

M README.md +198 -0


@@ -48,6 +48,191 @@ The host sends a `Cancel{reason}` frame (`execstream.Drive` takes a cancel chann
48	48
49	49		Everything a job creates (process, TAP or bridge, nftables rules, files, loop mounts, containers, networks, cgroups) is removed by `Stop`, which also runs with a cancelled context, aborts a `Start` in progress, and by `Reap` at start for what a crash left behind. A VM leaves a state record (`<state dir>/vms/<id>.json`) before its first resource. The network policy and the TAP or bridge are never removed while the VMM or the container may still be running: when a process cannot be killed or a container cannot be removed, `Stop` returns an error wrapping `sandbox.ErrLeaked`, the sandbox's slot stays taken, and `Driver.Reap` (which returns `sandbox.ErrBusy` while a sandbox is running) finishes the job and frees the slot. The host should stop taking jobs on a leaked slot until then. `Sandbox.ID()` names a sandbox's files and Docker objects.
50	50
	51	+	## Running jobs
	52	+
	53	+	```
	54	+	printf %s "$TOKEN" \| ermite-runner register --token-stdin # once
	55	+	ermite-runner run # as a service
	56	+	```
	57	+
	58	+	`register` picks the isolation mode like `run` will, registers the runner with
	59	+	ermite (name, labels, version, isolation, architecture) and stores the runner
	60	+	token in `$ERMITE_RUNNER_STATE_DIR/runner.json` (mode 0600, in a directory made
	61	+	0700; a file readable by others is refused). `--replace` removes a runner of the
	62	+	same name first.
	63	+
	64	+	The token of `register` (scope `runner:register`: it reads the code and the
	65	+	secrets of every repository, see `runner.proto`) is looked up in this order:
	66	+	standard input with `--token-stdin`, then `ERMITE_RUNNER_REGISTER_TOKEN`, then
	67	+	`--token`. Prefer the first two: **the arguments of a process are visible to
	68	+	every user of the host** (`ps`, `/proc/<pid>/cmdline`), the environment and
	69	+	standard input are not.
	70	+
	71	+	The server URL must be `https://`. Plain `http://` is accepted for
	72	+	`localhost`, `127.0.0.0/8` and `::1` only; `ERMITE_RUNNER_INSECURE_HTTP=1` allows
	73	+	any other `http://` URL (tests, a trusted network) and sends every token in
	74	+	clear. The client never follows a redirect, so a token cannot be carried to
	75	+	another host.
	76	+
	77	+	`run` reaps what a previous run left, then polls `FetchJob` whenever a slot is
	78	+	free (`ERMITE_RUNNER_CAPACITY`), with a `poll_id` that is kept when a response
	79	+	is lost and obeying `retry_after`. For each job it sends the acknowledging
	80	+	heartbeat first, then one every 30 s (a failed one is retried every 5 s), opens
	81	+	`WatchJob` (a cancel request reaches the executor, which interrupts, terminates
	82	+	and kills like the official runner), forwards the log with `AppendLog`
	83	+	(batches of at most 1,000 lines and 64 KiB per step, numbered offsets, retried
	84	+	with the same lines), sends step snapshots with `UpdateJob` and ends with an
	85	+	idempotent `CompleteJob`. When ermite says the job ended (`JobEnded`) or revokes
	86	+	the assignment (`unauthenticated`) the sandbox is destroyed and nothing more is
	87	+	sent. On SIGINT or SIGTERM the running jobs are cancelled with
	88	+	`runner_shutdown`, reported, and the sandboxes removed. The job token stays in
	89	+	the host process; the sandbox receives the workflow and its contexts only.
	90	+
	91	+	\| Variable \| Meaning \|
	92	+	\| --- \| --- \|
	93	+	\| `ERMITE_RUNNER_URL` \| base URL of ermite \|
	94	+	\| `ERMITE_RUNNER_NAME` \| runner name (default: the host name) \|
	95	+	\| `ERMITE_RUNNER_LABELS` \| extra labels, comma separated (`ubuntu-latest,ubuntu-24.04`) \|
	96	+	\| `ERMITE_RUNNER_REGISTER_TOKEN` \| token for `register` (see above) \|
	97	+	\| `ERMITE_RUNNER_INSECURE_HTTP` \| `1` allows a non-loopback `http://` URL (tests only) \|
	98	+	\| `ERMITE_RUNNER_LOG_LEVEL` \| `debug`, `info` (default), `warn`, `error` \|
	99	+	\| `ERMITE_RUNNER_LOG_FORMAT` \| `console` (default, colours) or `json` \|
	100	+
	101	+	## Installing the service
	102	+
	103	+	`contrib/ermite-runner.service` is a systemd unit. The runner runs as root
	104	+	on the host (TAP devices, nftables, loop mounts, the jailer, containers); the
	105	+	jobs never do: they run as user `runner` in a micro-VM or a container.
	106	+
	107	+	```sh
	108	+	sudo install -m 0755 ermite-runner /usr/local/bin/ermite-runner # a static build: CGO_ENABLED=0
	109	+	sudo install -m 0644 contrib/ermite-runner.service /etc/systemd/system/
	110	+	sudo install -d -m 0700 /etc/ermite-runner
	111	+	sudo install -m 0600 /dev/stdin /etc/ermite-runner/env <<'ENV'
	112	+	ERMITE_RUNNER_URL=https://ermite.example.org
	113	+	ERMITE_RUNNER_NAME=igor
	114	+	ERMITE_RUNNER_LABELS=ubuntu-latest,ubuntu-24.04
	115	+	ERMITE_RUNNER_ISOLATION=auto
	116	+	ERMITE_RUNNER_CAPACITY=2
	117	+	ERMITE_RUNNER_JOB_CPUS=4
	118	+	ERMITE_RUNNER_JOB_MEMORY=8G
	119	+	ERMITE_RUNNER_JOB_DISK=64G
	120	+	ERMITE_RUNNER_LOG_FORMAT=json
	121	+	ENV
	122	+	# Register once, in the same environment as the service (the settings, the state
	123	+	# directory), with the token on standard input so that it is not in any
	124	+	# command line:
	125	+	printf %s "$TOKEN" \| sudo systemd-run --pipe --wait -p EnvironmentFile=/etc/ermite-runner/env \
	126	+	-p StateDirectory=ermite-runner -p StateDirectoryMode=0700 \
	127	+	/usr/local/bin/ermite-runner register --token-stdin
	128	+	sudo systemctl enable --now ermite-runner
	129	+	journalctl -u ermite-runner -f
	130	+	```
	131	+
	132	+	`systemd-run` reads the same `EnvironmentFile` as the unit, so `register` and
	133	+	`run` agree on the URL, the name, the labels, the isolation mode and the state
	134	+	directory (`/var/lib/ermite-runner`, the default of `ERMITE_RUNNER_STATE_DIR`).
	135	+
	136	+	Logs. The runner writes to standard error, which journald collects. Set
	137	+	`ERMITE_RUNNER_LOG_FORMAT=json` under journald: one JSON event per line, with
	138	+	`job_id`, `run_id` and `attempt` on every line about a job, `warn` for what is
	139	+	retried and `error` for what is refused or leaked; the colours of the `console`
	140	+	format are noise in a journal. `ERMITE_RUNNER_LOG_LEVEL=debug` adds detail.
	141	+	Tokens are never logged.
	142	+
	143	+	Stopping. `systemctl stop` sends SIGTERM: running jobs are cancelled with
	144	+	`runner_shutdown`, reported to ermite, and their sandboxes removed before the
	145	+	service exits. The budget, with the defaults, is 90 s: 30 s for the executor
	146	+	to stop its steps (SIGINT, SIGTERM after 7.5 s, SIGKILL 2.5 s later, then the
	147	+	post-steps), 30 s to destroy the sandbox, 30 s to flush the log and report; jobs
	148	+	do this in parallel, so it does not grow with `ERMITE_RUNNER_CAPACITY`. The unit
	149	+	sets `TimeoutStopSec=120`, 30 s of margin (a test in `internal/host` fails if the
	150	+	two drift apart). A job whose first heartbeat is still pending when the signal
	151	+	arrives is not started: ermite gives it to another runner after 30 s. A crash or
	152	+	a power cut leaves sandboxes behind; the next start reaps them. Only one runner
	153	+	may live on a host (a host-wide lock enforces it), because reaping removes
	154	+	everything labelled as the runner's own.
	155	+
	156	+	Sandboxing of the unit. The runner has to be root (TAP devices, nftables,
	157	+	loop mounts, the jailer, Docker), so the unit only takes away what it does not
	158	+	use, and only with options that leave it in the host's mount namespace:
	159	+	`NoNewPrivileges`, `ProtectClock`, `RestrictSUIDSGID`, `LockPersonality`,
	160	+	`RestrictRealtime`, `SystemCallArchitectures=native` and `RestrictAddressFamilies`
	161	+	(UNIX, INET, INET6, NETLINK, VSOCK).
	162	+
	163	+	Deliberately absent, and why:
	164	+
	165	+	- Every option that gives the unit a mount namespace of its own: `PrivateTmp`,
	166	+	`ProtectHome`, `ProtectSystem`, `ProtectKernelModules`, `ProtectKernelLogs`,
	167	+	`ProtectKernelTunables`, `ProtectControlGroups`, `PrivateDevices`,
	168	+	`ReadOnlyPaths`, `ReadWritePaths`, `InaccessiblePaths`, `TemporaryFileSystem`,
	169	+	`BindPaths`. In container mode the runner loop-mounts the scratch disk of each
	170	+	job and the Docker daemon, which lives in the host's namespace, binds that
	171	+	directory into the container; a mount made inside the unit's namespace is
	172	+	invisible to it and the container would get an empty directory. (`ProtectKernelModules`
	173	+	and `ProtectKernelLogs` look harmless but are implemented with mounts too.) The
	174	+	micro-VM mode would tolerate them, but one unit serves both modes, and a second
	175	+	unit for VM-only hardening is not worth maintaining. A test in `internal/host`
	176	+	fails if the unit sets any of them.
	177	+	- `Delegate=`: the runner manages no cgroup of its own (the jailer writes the
	178	+	VMs' cgroups as root, Docker has its own).
	179	+	- `RestrictNamespaces=`, `PrivateNetwork=`, `CapabilityBoundingSet=`,
	180	+	`SystemCallFilter=`: network namespaces and devices (TAP, veth), mounts,
	181	+	`net.ipv4.ip_forward`, capabilities handed to the jailer and Docker.
	182	+	- `MemoryDenyWriteExecute=`: Firecracker maps guest memory writable and executable.
	183	+	- `UMask=`: the drivers create directories that the job's user must enter, with
	184	+	explicit modes; the state directory is made 0700 by the runner.
	185	+
	186	+	The boundary of a job is its micro-VM or container, not this unit. Checked on a
	187	+	workstation (a user manager, no root): the micro-VM job test (`TestVMRunsAJob`:
	188	+	user and network namespaces, Firecracker, TAP, vsock) passes under the options
	189	+	that remain. Not checked, because they need root on a real host: the jailer, loop
	190	+	mounts, nftables in the host namespace, Docker.
	191	+
	192	+	## The image
	193	+
	194	+	A job runs in the image named by `ERMITE_RUNNER_IMAGE` (container: an OCI
	195	+	reference; vm: the directory with `vmlinux` and `base.ext4`, both built from
	196	+	`images/ubuntu-24.04/Dockerfile`). The runner binary is attached to the sandbox
	197	+	by the host and is never part of the image. What the executor needs from the
	198	+	image:
	199	+
	200	+	- a user `runner` with uid and gid 1001 and the home `/home/runner`; the
	201	+	executor starts as root (init needs it to start `dockerd`) and runs every step
	202	+	as this user, with HOME, USER and LOGNAME set. Without such a user the steps
	203	+	run as root (a test image only);
	204	+	- the directories `/home/runner/work` and `/opt/hostedtoolcache` owned by
	205	+	`runner` (`RUNNER_TOOL_CACHE`, which the executor reads from its environment
	206	+	like the official runner; each job gets `/home/runner/work/<repo>/<repo>` and
	207	+	`/home/runner/work/_temp`, cleaned at the start of the job);
	208	+	- `bash` and `sh` (the default shell is `bash -e {0}` when bash exists,
	209	+	else `sh -e {0}`; like on hosted runners `shell: bash` adds
	210	+	`--noprofile --norc -eo pipefail`), `python3` and a `python` command for
	211	+	`shell: python`; `pwsh` is not part of the CI-1 image (a step that asks for it
	212	+	fails with `pwsh: command not found`);
	213	+	- `dockerd` (and passwordless `sudo` for `runner`) for jobs that use Docker: the
	214	+	job gets its own daemon with its data root on the scratch disk;
	215	+	- what workflows commonly use: git, curl, jq, build-essential, ca-certificates,
	216	+	tzdata, zip/unzip. Full parity with the hosted images (`actions/runner-images`)
	217	+	is a later phase.
	218	+
	219	+	## Job limits
	220	+
	221	+	Each job gets its own resources, set for the runner as a whole:
	222	+
	223	+	\| Variable \| Default \| Effect \|
	224	+	\|---\|---\|---\|
	225	+	\| `ERMITE_RUNNER_CAPACITY` \| `1` \| jobs in parallel: one sandbox, one uid, one `/30` network and one disk per slot. The runner polls ermite only when a slot is free. \|
	226	+	\| `ERMITE_RUNNER_JOB_CPUS` \| `2` \| vCPUs of a micro-VM; CPU quota of a container \|
	227	+	\| `ERMITE_RUNNER_JOB_MEMORY` \| `4G` \| guest memory (`M`, `G`, `T`; at least `256M`); a jailed VMM gets this plus 256 MiB of cgroup headroom \|
	228	+	\| `ERMITE_RUNNER_JOB_DISK` \| `64G` \| size of the sparse scratch disk (at least `1G`): the overlay upper layer of a VM and the Docker data root of a container. Deleted with the job \|
	229	+	\| `ERMITE_RUNNER_JOB_PIDS` \| `32768` \| process limit of a container \|
	230	+	\| `ERMITE_RUNNER_JOB_DISK_BANDWIDTH`, `_JOB_DISK_IOPS`, `_JOB_NET_BANDWIDTH` \| `400`, `20000`, `1000` \| VM rate limits (MiB/s, IOPS, Mbit/s); `0` disables \|
	231	+
	232	+	The job timeout is not a runner setting: it is the workflow's `timeout-minutes`
	233	+	(360 by default), which ermite sends with the job and the executor enforces from
	234	+	the moment it starts.
	235	+
51	236		## Configuration (sandbox part)
52	237
53	238		\| Variable \| Default \| \|

@@ -105,6 +290,19 @@ sudo env GOWORK=off ERMITE_RUNNER_IT_CONTAINER=1 ERMITE_RUNNER_IMAGE=ermite-runn
105	290		DOCKER_HOST=unix:///var/run/docker.sock "$(command -v go)" test -count=1 -v ./internal/sandbox/integration -run Container
106	291		```
107	292
	293	+	Jobs end to end (`ERMITE_RUNNER_IT_E2E=1`, `internal/e2e`): the real executor in the real image, started through the container driver. They also run on a workstation with a rootless Docker daemon (the test-only option `UnsafeAllowRootlessForTests`; the nested `dockerd` of the image cannot start there, and the jobs do not need it). `mount` and `umount` are replaced by stand-ins, since loop mounts need root.
	294	+
	295	+	```sh
	296	+	IMAGE_TAG=ermite-runner-ubuntu-24.04:dev scripts/build-image.sh tmp/image # or: docker build -t ermite-runner-ubuntu-24.04:dev images/ubuntu-24.04
	297	+	ERMITE_RUNNER_IT_E2E=1 ERMITE_RUNNER_TEST_IMAGE=ermite-runner-ubuntu-24.04:dev GOWORK=off go test -race -count=1 ./internal/e2e
	298	+	```
	299	+
	300	+	- `TestContainerRunsAJobAsTheRunnerUser` and `TestContainerCancelStopsTheJobAndOrphans` drive a job over the stream: steps run as `runner` (1001), file ownership, `GITHUB_PATH` and `GITHUB_ENV`, `python`, cancellation by SIGINT, the `always()` step, and the orphan cleanup of "Complete job".
	301	+	- `TestChainCorpus` runs the whole chain: a fake `RunnerService` (`internal/host/hosttest`, which checks the offsets, limits and tokens of `runner.proto`) hands out 22 jobs of the ermite-conformance corpus, the real host loop starts them in containers (two at a time), and what the fake stored (steps, conclusions, outputs, log lines) is compared with the recordings made on github.com. It reads `ERMITE_CONFORMANCE_DIR` (default: a sibling checkout of ermite-conformance) and never writes to it.
	302	+	- `TestVMRunsAJob` (`internal/sandbox/integration`, run by `scripts/it-vm-userns.sh`) does the first of these in a micro-VM.
	303	+
	304	+	Not covered anywhere yet, because they need root on a real host: the jailer (uid drop, chroot, cgroups), a rootful Docker daemon or sysbox with the nested `dockerd`, and the firewall of container mode.
	305	+
108	306		`ERMITE_RUNNER_IT_DOCKER_API=1 ERMITE_RUNNER_TEST_IMAGE=<image> go test -run DockerAPIWiring ./internal/sandbox/container` exercises the real Docker Engine calls (labels, networks, volumes, reaping) against any daemon, including a rootless one; it does not test isolation.
109	307
110	308		The firewall policy has a packet-level test (`ERMITE_RUNNER_IT_NET=1`, run by `scripts/it-vm-userns.sh`): two guests and an outside host on veth pairs in separate network namespaces, with a forged source and a forged "reply" between guests.

A contrib/ermite-runner.service +109 -0

@@ -0,0 +1,109 @@
	1	+	# SPDX-License-Identifier: MIT
	2	+	#
	3	+	# systemd unit of the ermite runner. Install the static binary as
	4	+	# /usr/local/bin/ermite-runner, write the settings to /etc/ermite-runner/env
	5	+	# (mode 0600) and register the runner once; the README has the commands.
	6	+	#
	7	+	# The runner runs as root: it creates TAP devices and nftables rules (vm), loop
	8	+	# mounts and containers (container), and starts the jailer. Jobs never run as
	9	+	# root on the host: they run in a micro-VM or a container as user "runner".
	10	+	#
	11	+	# The sandboxing below is what can be added without breaking those jobs.
	12	+	# What is deliberately NOT set, and why, is listed at the end of the file and
	13	+	# in the README.
	14	+	[Unit]
	15	+	Description=ermite CI runner
	16	+	Documentation=https://git.bnema.dev/bnema/ermite-runner
	17	+	Wants=network-online.target
	18	+	After=network-online.target docker.service
	19	+
	20	+	[Service]
	21	+	Type=simple
	22	+	EnvironmentFile=/etc/ermite-runner/env
	23	+	ExecStart=/usr/local/bin/ermite-runner run
	24	+	Restart=on-failure
	25	+	RestartSec=5
	26	+
	27	+	# SIGTERM cancels the running jobs with runner_shutdown, reports them and removes
	28	+	# their sandboxes. With the defaults that takes at most 90 s (30 s for the
	29	+	# executor to stop its steps, 30 s to destroy the sandbox, 30 s to report; the
	30	+	# jobs do this in parallel). A test checks that this value leaves 15 s of margin
	31	+	# over host.Timing.ShutdownBudget.
	32	+	KillMode=mixed
	33	+	KillSignal=SIGTERM
	34	+	TimeoutStopSec=120
	35	+
	36	+	# The state directory holds the runner token (0600), the VM images and the
	37	+	# per-job files; RuntimeDirectory is for the lock and sockets.
	38	+	StateDirectory=ermite-runner
	39	+	StateDirectoryMode=0700
	40	+	RuntimeDirectory=ermite-runner
	41	+
	42	+	# Jobs are many processes.
	43	+	TasksMax=infinity
	44	+	LimitNOFILE=1048576
	45	+
	46	+	# ---- sandboxing that leaves the unit in the host's mount namespace ----------
	47	+	# Only options that do not give the unit a mount namespace of its own: in
	48	+	# container mode the runner loop-mounts the scratch disk of a job, and the Docker
	49	+	# daemon, which lives in the host's namespace, must see that mount to bind it
	50	+	# into the container. See the end of the file.
	51	+	NoNewPrivileges=yes
	52	+	# The runner does not set the clock.
	53	+	ProtectClock=yes
	54	+	# No setuid/setgid files created by the runner or its children.
	55	+	RestrictSUIDSGID=yes
	56	+	LockPersonality=yes
	57	+	# The runner does not need real-time scheduling.
	58	+	RestrictRealtime=yes
	59	+	# Syscall architectures: native only.
	60	+	SystemCallArchitectures=native
	61	+	# Address families the runner and its children use: Unix sockets (Docker, the
	62	+	# executor stream), IPv4/IPv6 (the server, the DNS forwarder), netlink (ip, nft,
	63	+	# TAP devices), packet (none needed: left out), vsock (the executor stream in a
	64	+	# micro-VM).
	65	+	RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 AF_NETLINK AF_VSOCK
	66	+
	67	+	[Install]
	68	+	WantedBy=multi-user.target
	69	+
	70	+	# ---- deliberately absent -----------------------------------------------------
	71	+	#
	72	+	# PrivateTmp=, ProtectHome=, ProtectSystem=, ProtectKernelModules=,
	73	+	# ProtectKernelLogs=, ProtectKernelTunables=, ProtectControlGroups=,
	74	+	# PrivateDevices=, ReadOnlyPaths=, ReadWritePaths=, InaccessiblePaths=,
	75	+	# TemporaryFileSystem=, BindPaths=
	76	+	# Each of these puts the unit in a mount namespace of its own
	77	+	# (systemd sets one up for ProtectKernelModules and
	78	+	# ProtectKernelLogs too: it hides /usr/lib/modules and
	79	+	# /proc/kmsg with mounts). In container mode the runner
	80	+	# loop-mounts the scratch disk of a job below the state
	81	+	# directory, and the Docker daemon, in the host's namespace,
	82	+	# cannot see a mount made inside the unit's: the bind
	83	+	# sources of the container would be empty directories. The
	84	+	# micro-VM mode would tolerate them, but one unit serves both
	85	+	# modes and a second unit for VM-only hardening is not worth
	86	+	# it. test: internal/host TestUnitHasNoMountNamespaceOptions.
	87	+	#
	88	+	# Delegate=yes The runner does not manage cgroups of its own. The jailer
	89	+	# creates the cgroups of the VMs below /sys/fs/cgroup itself
	90	+	# (it is root and writes there directly), and Docker has its
	91	+	# own. Delegation would hand a subtree to this unit that
	92	+	# nothing uses.
	93	+	# PrivateNetwork=, RestrictNamespaces=,
	94	+	# CapabilityBoundingSet=, MemoryDenyWriteExecute=, SystemCallFilter=
	95	+	# The runner creates network namespaces and devices (TAP,
	96	+	# veth, bridges) and mounts file systems, enables
	97	+	# net.ipv4.ip_forward (the jailer also writes cgroups, /dev/kvm, /dev/net/tun and
	98	+	# /dev/loop* are needed), hands capabilities to the jailer and
	99	+	# to Docker, and starts Firecracker (which maps guest memory
	100	+	# writable and executable). Each of these would break a path
	101	+	# that cannot be tested without a rootful host; the
	102	+	# boundary of a job is the micro-VM or the container, not
	103	+	# this unit.
	104	+	# UMask=0077 The state directory is made 0700 by the runner itself, and the
	105	+	# drivers create directories that the job's user must be able
	106	+	# to enter (the scratch disk's work and tmp directories are
	107	+	# made with mode 0755): a restrictive umask would turn them
	108	+	# into root-only directories, which cannot be tested without
	109	+	# a rootful host.

M go.mod +4 -2


@@ -3,8 +3,12 @@ module bnema.dev/ermite-runner
3	3		go 1.27.1
4	4
5	5		require (
	6	+	bnema.dev/ermite-actions v0.0.0-20261012165742-b6c4c3a2f7ff
	7	+	bnema.dev/ermite-api v0.0.0-20261012155218-f8f64413e63c
	8	+	connectrpc.com/connect v1.21.0
6	9		github.com/bnema/zerowrap v1.4.1
7	10		github.com/containerd/errdefs v1.0.0
	11	+	github.com/google/uuid v1.6.0
8	12		github.com/moby/moby/api v1.56.1
9	13		github.com/moby/moby/client v0.6.2
10	14		golang.org/x/sys v0.49.0

@@ -28,7 +32,6 @@ require (
28	32		buf.build/go/standard v0.1.1-0.20260325175353-2b287e071df5 // indirect
29	33		cel.dev/cel-go v0.32.0 // indirect
30	34		cel.dev/expr v0.25.3 // indirect
31		-	connectrpc.com/connect v1.20.0 // indirect
32	35		connectrpc.com/otelconnect v0.9.0 // indirect
33	36		github.com/Microsoft/go-winio v0.6.2 // indirect
34	37		github.com/antlr4-go/antlr/v4 v4.13.1 // indirect

@@ -49,7 +52,6 @@ require (
49	52		github.com/go-logr/stdr v1.2.2 // indirect
50	53		github.com/gofrs/flock v0.13.1 // indirect
51	54		github.com/google/go-containerregistry v0.22.1 // indirect
52		-	github.com/google/uuid v1.6.0 // indirect
53	55		github.com/inconshreveable/mousetrap v1.1.0 // indirect
54	56		github.com/jdx/go-netrc v1.0.0 // indirect
55	57		github.com/klauspost/compress v1.20.0 // indirect

M go.sum +6 -2 generated file: collapsed

load diff

A host_cmd.go +134 -0

@@ -0,0 +1,134 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package main
	4	+
	5	+	import (
	6	+	"context"
	7	+	"errors"
	8	+	"flag"
	9	+	"fmt"
	10	+	"os"
	11	+
	12	+	"github.com/bnema/zerowrap"
	13	+
	14	+	"bnema.dev/ermite-api/gen/ermite/ermiteconnect"
	15	+
	16	+	"bnema.dev/ermite-runner/internal/host"
	17	+	"bnema.dev/ermite-runner/internal/sandbox"
	18	+	"bnema.dev/ermite-runner/internal/sandbox/drivers"
	19	+	)
	20	+
	21	+	// loadHostConfig reads the sandbox settings and the host settings together, so
	22	+	// that the state directory and the capacity are the same for both.
	23	+	func loadHostConfig() (sandbox.Config, host.Config, error) {
	24	+	scfg, err := sandbox.LoadConfig(os.Getenv)
	25	+	if err != nil {
	26	+	return scfg, host.Config{}, err
	27	+	}
	28	+	name, _ := os.Hostname()
	29	+	hcfg, err := host.LoadConfig(os.Getenv, name, scfg.StateDir, scfg.Capacity, version)
	30	+	return scfg, hcfg, err
	31	+	}
	32	+
	33	+	func buildDriver(ctx context.Context, scfg sandbox.Config) (sandbox.Driver, error) {
	34	+	return sandbox.Select(ctx, scfg, drivers.Factories(), sandbox.Prober{})
	35	+	}
	36	+
	37	+	// registerCmd is `ermite-runner register [--token-stdin] [--token T] [--replace]`:
	38	+	// it chooses the isolation mode like `run` will, registers the runner with it,
	39	+	// and stores the runner token (mode 0600) in the state directory. The register
	40	+	// token is read from standard input (--token-stdin) or from
	41	+	// ERMITE_RUNNER_REGISTER_TOKEN; --token is accepted but the arguments of a
	42	+	// process are visible to every user of the host.
	43	+	func registerCmd(ctx context.Context, args []string) int {
	44	+	fs := flag.NewFlagSet("register", flag.ContinueOnError)
	45	+	token := fs.String("token", "", "token with the runner:register scope (visible in the process list: prefer $"+host.EnvRegisterToken+" or --token-stdin)")
	46	+	fromStdin := fs.Bool("token-stdin", false, "read the token from standard input")
	47	+	replace := fs.Bool("replace", false, "replace a runner of the same name")
	48	+	if err := fs.Parse(args); err != nil {
	49	+	return 2
	50	+	}
	51	+	tok, err := host.ResolveRegisterToken(token, fromStdin, os.Stdin, os.Getenv)
	52	+	if err != nil {
	53	+	fmt.Fprintln(os.Stderr, "ermite-runner:", err)
	54	+	return 2
	55	+	}
	56	+	*token = tok
	57	+	scfg, hcfg, err := loadHostConfig()
	58	+	if err != nil {
	59	+	fmt.Fprintln(os.Stderr, "ermite-runner:", err)
	60	+	return 2
	61	+	}
	62	+	if hcfg.URL == "" {
	63	+	fmt.Fprintf(os.Stderr, "ermite-runner: %s is not set\n", host.EnvURL)
	64	+	return 2
	65	+	}
	66	+	drv, err := buildDriver(ctx, scfg)
	67	+	if err != nil {
	68	+	fmt.Fprintln(os.Stderr, "ermite-runner:", err)
	69	+	return 1
	70	+	}
	71	+	iso := drv.Isolation()
	72	+	_ = drv.Close()
	73	+
	74	+	client := hcfg.Client(hcfg.URL, *token)
	75	+	st, err := host.Register(ctx, hcfg, client, iso, *replace)
	76	+	if err != nil {
	77	+	fmt.Fprintln(os.Stderr, "ermite-runner:", err)
	78	+	return 1
	79	+	}
	80	+	fmt.Printf("registered runner %q (%s) with isolation %s; token stored in %s\n", st.Name, st.RunnerID, iso, host.StateFile(hcfg.StateDir))
	81	+	return 0
	82	+	}
	83	+
	84	+	// runCmd is `ermite-runner run`: poll ermite for jobs and run each one in its
	85	+	// own sandbox until SIGINT or SIGTERM, then cancel the running jobs with
	86	+	// runner_shutdown and report them.
	87	+	func runCmd(ctx context.Context) int {
	88	+	log := zerowrap.FromCtx(ctx)
	89	+	scfg, hcfg, err := loadHostConfig()
	90	+	if err != nil {
	91	+	fmt.Fprintln(os.Stderr, "ermite-runner:", err)
	92	+	return 2
	93	+	}
	94	+	st, err := host.LoadState(hcfg.StateDir)
	95	+	if err != nil {
	96	+	fmt.Fprintln(os.Stderr, "ermite-runner:", err)
	97	+	return 2
	98	+	}
	99	+	url := hcfg.URL
	100	+	if url == "" {
	101	+	url = st.URL
	102	+	}
	103	+	if url == "" {
	104	+	fmt.Fprintf(os.Stderr, "ermite-runner: %s is not set\n", host.EnvURL)
	105	+	return 2
	106	+	}
	107	+	drv, err := buildDriver(ctx, scfg)
	108	+	if err != nil {
	109	+	fmt.Fprintln(os.Stderr, "ermite-runner:", err)
	110	+	return 1
	111	+	}
	112	+	defer func() { _ = drv.Close() }()
	113	+	log.Info().Str("runner", st.Name).Str("isolation", string(drv.Isolation())).Int("capacity", scfg.Capacity).Msg("runner started")
	114	+
	115	+	r := &host.Runner{
	116	+	Cfg: hcfg,
	117	+	Driver: drv,
	118	+	Poll: hcfg.Client(url, st.RunnerToken),
	119	+	Job: func(tok string) ermiteconnect.RunnerServiceClient {
	120	+	return hcfg.Client(url, tok)
	121	+	},
	122	+	}
	123	+	r.Cfg.URL = url
	124	+	if err := r.Run(ctx); err != nil {
	125	+	if errors.Is(err, host.ErrRemoved) {
	126	+	fmt.Fprintln(os.Stderr, "ermite-runner:", err)
	127	+	return 3
	128	+	}
	129	+	fmt.Fprintln(os.Stderr, "ermite-runner:", err)
	130	+	return 1
	131	+	}
	132	+	log.Info().Msg("runner stopped")
	133	+	return 0
	134	+	}

M images/ubuntu-24.04/Dockerfile +1 -1


@@ -12,7 +12,7 @@ ARG DEBIAN_FRONTEND=noninteractive
12	12		RUN apt-get update \
13	13		&& apt-get install -y --no-install-recommends \
14	14		bash build-essential ca-certificates curl e2fsprogs git gnupg iproute2 \
15		-	iptables jq kmod procps python3 sudo tzdata unzip xz-utils zip \
	15	+	iptables jq kmod procps python-is-python3 python3 sudo tzdata unzip xz-utils zip \
16	16		&& rm -rf /var/lib/apt/lists/*
17	17
18	18		# Docker Engine from Docker's apt repository (the job gets its own daemon).

A internal/e2e/chain_test.go +328 -0

@@ -0,0 +1,328 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package e2e
	4	+
	5	+	import (
	6	+	"context"
	7	+	"encoding/json"
	8	+	"fmt"
	9	+	"net/http"
	10	+	"os"
	11	+	"path/filepath"
	12	+	"regexp"
	13	+	"sort"
	14	+	"strings"
	15	+	"testing"
	16	+	"time"
	17	+
	18	+	"github.com/bnema/zerowrap"
	19	+
	20	+	ermite "bnema.dev/ermite-api/gen/ermite"
	21	+	"bnema.dev/ermite-api/gen/ermite/ermiteconnect"
	22	+
	23	+	"bnema.dev/ermite-runner/internal/host"
	24	+	"bnema.dev/ermite-runner/internal/host/hosttest"
	25	+	)
	26	+
	27	+	// The chain test runs the whole path of a job: a fake RunnerService hands out a
	28	+	// corpus job, the real host loop starts a real sandbox (the runner image in a
	29	+	// container), the real executor runs the steps, and the host reports to the
	30	+	// fake with AppendLog, UpdateJob and CompleteJob. What the fake stored is
	31	+	// compared with the recording of the same job made on github.com.
	32	+	//
	33	+	// It needs a checkout of ermite-conformance (corpus/ and recordings/), found
	34	+	// through ERMITE_CONFORMANCE_DIR or as a sibling of this repository, and is
	35	+	// read-only on it.
	36	+
	37	+	func conformanceDir(t *testing.T) string {
	38	+	t.Helper()
	39	+	cands := []string{os.Getenv("ERMITE_CONFORMANCE_DIR")}
	40	+	wd, _ := os.Getwd()
	41	+	for d := wd; d != "/" && d != "."; d = filepath.Dir(d) {
	42	+	cands = append(cands, filepath.Join(d, "ermite-conformance", ".worktrees", "corpus"), filepath.Join(d, "ermite-conformance"))
	43	+	}
	44	+	for _, c := range cands {
	45	+	if c == "" {
	46	+	continue
	47	+	}
	48	+	if st, err := os.Stat(filepath.Join(c, "recordings")); err == nil && st.IsDir() {
	49	+	return c
	50	+	}
	51	+	}
	52	+	t.Skip("ermite-conformance checkout not found (set ERMITE_CONFORMANCE_DIR)")
	53	+	return ""
	54	+	}
	55	+
	56	+	type recStep struct {
	57	+	Name string `json:"name"`
	58	+	Conclusion string `json:"conclusion"`
	59	+	Log []string `json:"log"`
	60	+	}
	61	+
	62	+	type recJob struct {
	63	+	Name string `json:"name"`
	64	+	Conclusion string `json:"conclusion"`
	65	+	Steps []recStep `json:"steps"`
	66	+	}
	67	+
	68	+	type recording struct {
	69	+	Pushes []struct {
	70	+	Runs []struct {
	71	+	Jobs []recJob `json:"jobs"`
	72	+	} `json:"runs"`
	73	+	} `json:"pushes"`
	74	+	}
	75	+
	76	+	var (
	77	+	uuidRe = regexp.MustCompile(`[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}`)
	78	+	// Machine-dependent parts of the recordings, like the normalisation rules of
	79	+	// ermite-conformance: random script names and the repository name.
	80	+	repoRe = regexp.MustCompile(`/work/hello/hello`)
	81	+	)
	82	+
	83	+	func normalize(l string) string {
	84	+	l = repoRe.ReplaceAllString(l, "/work/<repo>/<repo>")
	85	+	return uuidRe.ReplaceAllString(l, "<uuid>")
	86	+	}
	87	+
	88	+	func chainContexts(job string) string {
	89	+	return fmt.Sprintf(`{"github":{"repository":"octo/hello","repository_owner":"octo","run_id":"1","run_number":"1","run_attempt":"1",
	90	+	"ref":"refs/heads/scn/x/r1","ref_name":"scn/x/r1","ref_type":"branch","ref_protected":false,"sha":"0123456789abcdef0123456789abcdef01234567",
	91	+	"event_name":"push","event":{"ref":"refs/heads/scn/x/r1"},"job":%q,"workflow":"w","actor":"octo","triggering_actor":"octo",
	92	+	"server_url":"https://ermite.example","retention_days":"90"}}`, job)
	93	+	}
	94	+
	95	+	func TestChainCorpus(t *testing.T) {
	96	+	image := needE2E(t)
	97	+	dir := conformanceDir(t)
	98	+	drv := newDriver(t, image)
	99	+
	100	+	// scenario -> jobs to run (jobs with `needs` depend on the server and are left out)
	101	+	scenarios := []struct {
	102	+	name string
	103	+	jobs []string
	104	+	}{
	105	+	{"env-precedence", []string{"main", "other"}},
	106	+	{"env-expressions", []string{"main"}},
	107	+	{"out-github-env", []string{"main"}},
	108	+	{"out-github-env-reserved", []string{"node-options", "github-prefix"}},
	109	+	{"out-github-path", []string{"main"}},
	110	+	{"out-step-outputs", []string{"main"}},
	111	+	{"ctx-steps", []string{"main"}},
	112	+	{"if-step-conditions", []string{"main"}},
	113	+	{"shell-builtin", []string{"main"}},
	114	+	{"shell-custom", []string{"main"}},
	115	+	{"shell-exit-codes", nil}, // every job of the workflow
	116	+	{"shell-working-directory", nil},
	117	+	{"out-job-outputs", []string{"producer"}},
	118	+	}
	119	+
	120	+	srv := hosttest.New()
	121	+	t.Cleanup(srv.Close)
	122	+
	123	+	type want struct {
	124	+	scenario string
	125	+	job recJob
	126	+	}
	127	+	wants := map[string]want{}
	128	+	var queue []*ermite.Job
	129	+	for _, sc := range scenarios {
	130	+	files, _ := filepath.Glob(filepath.Join(dir, "corpus", sc.name, "tree", ".github", "workflows", "*.yml"))
	131	+	if len(files) != 1 {
	132	+	t.Fatalf("%s: workflow files %v", sc.name, files)
	133	+	}
	134	+	yaml, err := os.ReadFile(files[0])
	135	+	if err != nil {
	136	+	t.Fatal(err)
	137	+	}
	138	+	raw, err := os.ReadFile(filepath.Join(dir, "recordings", sc.name+".json"))
	139	+	if err != nil {
	140	+	t.Fatal(err)
	141	+	}
	142	+	var rec recording
	143	+	if err := json.Unmarshal(raw, &rec); err != nil {
	144	+	t.Fatal(err)
	145	+	}
	146	+	for _, rj := range rec.Pushes[0].Runs[0].Jobs {
	147	+	if sc.jobs != nil && !contains(sc.jobs, rj.Name) {
	148	+	continue
	149	+	}
	150	+	id := sc.name + "/" + rj.Name
	151	+	queue = append(queue, &ermite.Job{
	152	+	JobId: id, RunId: 1, Attempt: 1, JobToken: "ermitejob_" + strings.ReplaceAll(id, "/", "_"),
	153	+	WorkflowPath: ".github/workflows/ci.yml", WorkflowYaml: string(yaml), JobKey: rj.Name, Matrix: "{}",
	154	+	Contexts: chainContexts(rj.Name), Repo: "hello", TimeoutMinutes: 10,
	155	+	})
	156	+	wants[id] = want{sc.name, rj}
	157	+	}
	158	+	}
	159	+	srv.Queue(queue...)
	160	+
	161	+	r := &host.Runner{
	162	+	Cfg: host.Config{URL: srv.URL, Name: "chain", Capacity: 2, Version: "test"},
	163	+	Driver: drv,
	164	+	Poll: host.NewClient(srv.URL, hosttest.RunnerToken, http.DefaultTransport),
	165	+	Job: func(tok string) ermiteconnect.RunnerServiceClient {
	166	+	return host.NewClient(srv.URL, tok, http.DefaultTransport)
	167	+	},
	168	+	Timing: host.Timing{LogFlush: 20 * time.Millisecond, NoJobPause: 100 * time.Millisecond},
	169	+	}
	170	+	ctx, cancel := context.WithCancel(zerowrap.WithCtx(context.Background(), zerowrap.New(zerowrap.Config{Level: "info", Format: "console", Output: testWriter{t}})))
	171	+	done := make(chan error, 1)
	172	+	go func() { done <- r.Run(ctx) }()
	173	+	defer func() {
	174	+	cancel()
	175	+	select {
	176	+	case err := <-done:
	177	+	if err != nil {
	178	+	t.Errorf("Run: %v", err)
	179	+	}
	180	+	case <-time.After(60 * time.Second):
	181	+	t.Error("Run did not stop")
	182	+	}
	183	+	}()
	184	+
	185	+	deadline := time.After(5 * time.Minute)
	186	+	for n := 0; n < len(queue); n++ {
	187	+	select {
	188	+	case <-srv.Done():
	189	+	case <-deadline:
	190	+	t.Fatalf("only %d of %d jobs finished", n, len(queue))
	191	+	}
	192	+	}
	193	+
	194	+	var ids []string
	195	+	for id := range wants {
	196	+	ids = append(ids, id)
	197	+	}
	198	+	sort.Strings(ids)
	199	+	for _, id := range ids {
	200	+	w := wants[id]
	201	+	t.Run(id, func(t *testing.T) { compareJob(t, srv.Result(id), w.job, w.scenario) })
	202	+	}
	203	+	for _, e := range srv.Violations() {
	204	+	t.Errorf("the fake server saw a contract violation: %s", e)
	205	+	}
	206	+	}
	207	+
	208	+	func contains(l []string, s string) bool {
	209	+	for _, x := range l {
	210	+	if x == s {
	211	+	return true
	212	+	}
	213	+	}
	214	+	return false
	215	+	}
	216	+
	217	+	// compareJob checks the steps, conclusions, outputs and log lines reported for
	218	+	// a job against the recording. "Set up job" (runner and image details) and the
	219	+	// ermite-dump step that the harness appends to every job are not compared.
	220	+	func compareJob(t *testing.T, got hosttest.Result, want recJob, scenario string) {
	221	+	if got.Complete == nil {
	222	+	t.Fatal("the job did not complete")
	223	+	}
	224	+	// The conclusion of the job as GitHub recorded it, except where the dump
	225	+	// step failed on GitHub (it only fails after a failed step in some
	226	+	// scenarios and is not part of the job here).
	227	+	wantConcl := map[string]ermite.CiConclusion{
	228	+	"success": ermite.CiConclusion_CI_CONCLUSION_SUCCESS, "failure": ermite.CiConclusion_CI_CONCLUSION_FAILURE,
	229	+	"cancelled": ermite.CiConclusion_CI_CONCLUSION_CANCELLED,
	230	+	}
	231	+	jobWant := wantConcl[want.Conclusion]
	232	+	for _, s := range want.Steps {
	233	+	if s.Name == "ermite-dump" && s.Conclusion != "success" && want.Conclusion == "failure" {
	234	+	// the job failed only because of the dump step
	235	+	jobWant = ermite.CiConclusion_CI_CONCLUSION_SUCCESS
	236	+	for _, o := range want.Steps {
	237	+	if o.Name != "ermite-dump" && o.Conclusion == "failure" {
	238	+	jobWant = ermite.CiConclusion_CI_CONCLUSION_FAILURE
	239	+	}
	240	+	}
	241	+	}
	242	+	}
	243	+	if got.Complete.Conclusion != jobWant {
	244	+	t.Errorf("job conclusion %v, want %v", got.Complete.Conclusion, jobWant)
	245	+	}
	246	+
	247	+	// Steps by number, with their lines.
	248	+	byStep := map[int32][]string{}
	249	+	for _, l := range got.Lines {
	250	+	byStep[l.Step] = append(byStep[l.Step], normalize(l.Text))
	251	+	}
	252	+	var steps []int32
	253	+	for n := range got.Steps {
	254	+	steps = append(steps, n)
	255	+	}
	256	+	sort.Slice(steps, func(i, j int) bool { return steps[i] < steps[j] })
	257	+
	258	+	type es = recStep
	259	+	var wantSteps, gotSteps []es
	260	+	for _, s := range want.Steps {
	261	+	if s.Name == "Set up job" \|\| s.Name == "ermite-dump" {
	262	+	continue
	263	+	}
	264	+	wantSteps = append(wantSteps, s)
	265	+	}
	266	+	conc := map[ermite.CiConclusion]string{
	267	+	ermite.CiConclusion_CI_CONCLUSION_SUCCESS: "success", ermite.CiConclusion_CI_CONCLUSION_FAILURE: "failure",
	268	+	ermite.CiConclusion_CI_CONCLUSION_CANCELLED: "cancelled", ermite.CiConclusion_CI_CONCLUSION_SKIPPED: "skipped",
	269	+	}
	270	+	for _, n := range steps {
	271	+	u := got.Steps[n]
	272	+	if u.Name == "Set up job" {
	273	+	continue
	274	+	}
	275	+	if u.State != ermite.StepState_STEP_STATE_COMPLETED \|\| u.StartedAt == nil \|\| u.CompletedAt == nil {
	276	+	t.Errorf("step %d %q is not a complete snapshot: %v", n, u.Name, u)
	277	+	}
	278	+	gotSteps = append(gotSteps, es{Name: u.Name, Conclusion: conc[u.Conclusion], Log: byStep[n]})
	279	+	}
	280	+	if len(gotSteps) != len(wantSteps) {
	281	+	var names []string
	282	+	for _, s := range gotSteps {
	283	+	names = append(names, s.Name)
	284	+	}
	285	+	t.Fatalf("steps %q, want %d steps", names, len(wantSteps))
	286	+	}
	287	+	for i := range wantSteps {
	288	+	w, g := wantSteps[i], gotSteps[i]
	289	+	if g.Name != w.Name \|\| g.Conclusion != w.Conclusion {
	290	+	t.Errorf("step %d: %q (%s), want %q (%s)", i+1, g.Name, g.Conclusion, w.Name, w.Conclusion)
	291	+	continue
	292	+	}
	293	+	if strings.Join(g.Log, "\n") != strings.Join(w.Log, "\n") {
	294	+	t.Errorf("step %q log differs:\n--- got\n%s\n--- want\n%s", w.Name, strings.Join(g.Log, "\n"), strings.Join(w.Log, "\n"))
	295	+	}
	296	+	}
	297	+
	298	+	if scenario == "out-job-outputs" {
	299	+	checkProducerOutputs(t, got)
	300	+	}
	301	+	}
	302	+
	303	+	// checkProducerOutputs compares the outputs of the producer job of
	304	+	// out-job-outputs with what the recorded consumer printed from `needs`.
	305	+	func checkProducerOutputs(t *testing.T, got hosttest.Result) {
	306	+	t.Helper()
	307	+	// The values the consumer printed on GitHub (recordings/out-job-outputs.json,
	308	+	// step "print needs outputs"): empty outputs are absent.
	309	+	want := map[string]string{
	310	+	"composed": "hello-push", "from-env": "env-value", "from-failed": "written-before-failure",
	311	+	"overwritten": "second", "simple": "hello", "multi": "line one\nline two", "literal": "fixed-text",
	312	+	}
	313	+	if len(got.Complete.Outputs) != len(want) {
	314	+	t.Errorf("outputs %v, want %v", got.Complete.Outputs, want)
	315	+	}
	316	+	for k, v := range want {
	317	+	if got.Complete.Outputs[k] != v {
	318	+	t.Errorf("output %q = %q, want %q", k, got.Complete.Outputs[k], v)
	319	+	}
	320	+	}
	321	+	}
	322	+
	323	+	type testWriter struct{ t *testing.T }
	324	+
	325	+	func (w testWriter) Write(p []byte) (int, error) {
	326	+	w.t.Log(strings.TrimRight(string(p), "\n"))
	327	+	return len(p), nil
	328	+	}

A internal/e2e/e2e_test.go +271 -0

@@ -0,0 +1,271 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	// Package e2e runs jobs through the real executor in a real sandbox. The tests
	4	+	// are opt-in because they need Docker (and, for the VM, KVM and Firecracker):
	5	+	//
	6	+	// ERMITE_RUNNER_IT_E2E=1 ERMITE_RUNNER_TEST_IMAGE=ermite-runner-ubuntu-24.04:dev go test ./internal/e2e
	7	+	//
	8	+	// On a workstation with a rootless Docker daemon the container driver is used
	9	+	// with UnsafeAllowRootlessForTests (a test-only option): the jobs run as in production except
	10	+	// that the nested dockerd of the image cannot start, which these tests do not
	11	+	// need. The production paths that need root (jailer, rootful Docker, sysbox)
	12	+	// are not covered here.
	13	+	package e2e
	14	+
	15	+	import (
	16	+	"context"
	17	+	"fmt"
	18	+	"io"
	19	+	"os"
	20	+	"os/exec"
	21	+	"path/filepath"
	22	+	"strings"
	23	+	"sync"
	24	+	"testing"
	25	+	"time"
	26	+
	27	+	"bnema.dev/ermite-runner/internal/execstream"
	28	+	"bnema.dev/ermite-runner/internal/execstream/execpb"
	29	+	"bnema.dev/ermite-runner/internal/sandbox"
	30	+	"bnema.dev/ermite-runner/internal/sandbox/container"
	31	+	"bnema.dev/ermite-runner/internal/sandbox/disk"
	32	+	"bnema.dev/ermite-runner/internal/sandbox/sandboxtest"
	33	+	)
	34	+
	35	+	func needE2E(t *testing.T) string {
	36	+	t.Helper()
	37	+	if os.Getenv("ERMITE_RUNNER_IT_E2E") != "1" {
	38	+	t.Skip("set ERMITE_RUNNER_IT_E2E=1 and ERMITE_RUNNER_TEST_IMAGE (needs Docker; see the package comment)")
	39	+	}
	40	+	image := os.Getenv("ERMITE_RUNNER_TEST_IMAGE")
	41	+	if image == "" {
	42	+	t.Fatal("set ERMITE_RUNNER_TEST_IMAGE to the runner image")
	43	+	}
	44	+	if _, err := exec.LookPath("docker"); err != nil {
	45	+	t.Skip("docker CLI not found")
	46	+	}
	47	+	return image
	48	+	}
	49	+
	50	+	// buildRunner compiles the static runner binary that is attached to the sandbox.
	51	+	func buildRunner(t *testing.T, dir string) string {
	52	+	t.Helper()
	53	+	exe := filepath.Join(dir, "ermite-runner")
	54	+	root, err := exec.Command("go", "list", "-m", "-f", "{{.Dir}}").Output()
	55	+	if err != nil {
	56	+	t.Fatal(err)
	57	+	}
	58	+	cmd := exec.Command("go", "build", "-o", exe, "bnema.dev/ermite-runner")
	59	+	cmd.Env = append(os.Environ(), "CGO_ENABLED=0")
	60	+	cmd.Dir = strings.TrimSpace(string(root))
	61	+	if out, err := cmd.CombinedOutput(); err != nil {
	62	+	t.Fatalf("build the runner: %v\n%s", err, out)
	63	+	}
	64	+	return exe
	65	+	}
	66	+
	67	+	// newDriver starts the container driver with test-only settings: no firewall
	68	+	// plumbing, no loop mounts, a rootless daemon allowed.
	69	+	func newDriver(t *testing.T, image string) sandbox.Driver {
	70	+	t.Helper()
	71	+	dir := t.TempDir()
	72	+	cfg := sandbox.Config{
	73	+	Isolation: sandbox.Container, StateDir: filepath.Join(dir, "state"), Capacity: 2, Image: image,
	74	+	JobCPUs: 2, JobMemoryMiB: 1024, JobDiskMiB: 2048, UIDBase: 900000,
	75	+	Network: netPrefix("10.214.0.0/16"), Executable: buildRunner(t, dir), DNSPort: 0,
	76	+	LockFile: filepath.Join(dir, "host.lock"), JobPids: 4096,
	77	+	}
	78	+	empty := filepath.Join(dir, "mountinfo")
	79	+	_ = os.WriteFile(empty, nil, 0o644)
	80	+	run := rootlessMountRunner{image: image, mountinfo: empty, mu: &sync.Mutex{}}
	81	+	d, err := container.New(context.Background(), cfg, container.Options{
	82	+	Net: sandboxtest.NewPlumbing(), UnsafeAllowRootlessForTests: true, ConnectTimeout: 90 * time.Second, Run: run,
	83	+	Mounter: &disk.Mounter{Run: run, Mountinfo: empty},
	84	+	})
	85	+	if err != nil {
	86	+	t.Fatal(err)
	87	+	}
	88	+	t.Cleanup(func() { _ = d.Close() })
	89	+	if err := d.Reap(context.Background()); err != nil {
	90	+	t.Fatal(err)
	91	+	}
	92	+	return d
	93	+	}
	94	+
	95	+	// rootlessMountRunner stands in for mount and umount, which need root. The
	96	+	// scratch directory stays a plain directory; "mount" records it in a fake
	97	+	// mountinfo file so that the driver believes it is mounted and calls "umount"
	98	+	// at the end. On a rootless daemon the files that the runner user (uid 1001 in
	99	+	// the container) creates belong to a subordinate uid that the host user cannot
	100	+	// delete, so "umount" empties the directory from inside a container, where
	101	+	// root can. With the real loop mount the scratch image is simply unmounted and
	102	+	// deleted.
	103	+	type rootlessMountRunner struct {
	104	+	image string
	105	+	mountinfo string
	106	+	mu *sync.Mutex
	107	+	}
	108	+
	109	+	func (r rootlessMountRunner) Run(ctx context.Context, in io.Reader, name string, args ...string) ([]byte, error) {
	110	+	switch name {
	111	+	case "mount":
	112	+	r.mu.Lock()
	113	+	defer r.mu.Unlock()
	114	+	f, err := os.OpenFile(r.mountinfo, os.O_APPEND\|os.O_WRONLY, 0o644)
	115	+	if err != nil {
	116	+	return nil, err
	117	+	}
	118	+	defer func() { _ = f.Close() }()
	119	+	_, err = fmt.Fprintf(f, "1 1 0:1 / %s rw - ext4 fake rw\n", args[len(args)-1])
	120	+	return nil, err
	121	+	case "umount":
	122	+	dir := args[len(args)-1]
	123	+	out, err := exec.CommandContext(ctx, "docker", "run", "--rm", "--entrypoint", "sh", "-v", dir+":/d", r.image,
	124	+	"-c", "rm -rf /d/* /d/.[!.]* 2>/dev/null; true").CombinedOutput()
	125	+	r.mu.Lock()
	126	+	defer r.mu.Unlock()
	127	+	b, _ := os.ReadFile(r.mountinfo)
	128	+	var keep []string
	129	+	for _, l := range strings.Split(string(b), "\n") {
	130	+	if l != "" && !strings.Contains(l, " "+dir+" ") {
	131	+	keep = append(keep, l)
	132	+	}
	133	+	}
	134	+	_ = os.WriteFile(r.mountinfo, []byte(strings.Join(keep, "\n")+"\n"), 0o644)
	135	+	return out, err
	136	+	}
	137	+	return sandboxtest.NoMountRunner{}.Run(ctx, in, name, args...)
	138	+	}
	139	+
	140	+	// session is the outcome of one job driven through the stream.
	141	+	type session struct {
	142	+	Steps []string
	143	+	Concl map[string]execpb.StepConclusion
	144	+	Logs []string // text of every log line, in order
	145	+	LogStep []int32
	146	+	Result *execpb.JobResult
	147	+	Exit *execpb.Exit
	148	+	}
	149	+
	150	+	// runJob starts a sandbox, drives the job and stops the sandbox.
	151	+	func runJob(t testing.T, d sandbox.Driver, spec execpb.JobSpec, cancel <-chan execpb.Cancel) session {
	152	+	t.Helper()
	153	+	sb, err := d.New()
	154	+	if err != nil {
	155	+	t.Fatal(err)
	156	+	}
	157	+	defer func() {
	158	+	if err := sb.Stop(context.Background()); err != nil {
	159	+	t.Errorf("stopping the sandbox: %v", err)
	160	+	}
	161	+	}()
	162	+	conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "e2e"})
	163	+	if err != nil {
	164	+	t.Fatal(err)
	165	+	}
	166	+	s := &session{Concl: map[string]execpb.StepConclusion{}}
	167	+	_, exit, err := execstream.DriveJob(conn, &execpb.Run{Job: spec}, cancel, execstream.JobHandler{
	168	+	StepStarted: func(x *execpb.StepStarted) { s.Steps = append(s.Steps, x.Name) },
	169	+	StepCompleted: func(x *execpb.StepCompleted) {
	170	+	s.Concl[x.Name] = x.Conclusion
	171	+	},
	172	+	Log: func(l *execpb.Log) { s.Logs = append(s.Logs, l.Text); s.LogStep = append(s.LogStep, l.Step) },
	173	+	JobResult: func(r *execpb.JobResult) { s.Result = r },
	174	+	})
	175	+	if err != nil {
	176	+	t.Fatalf("driving the job: %v", err)
	177	+	}
	178	+	s.Exit = exit
	179	+	return s
	180	+	}
	181	+
	182	+	func jobSpec(yaml string) *execpb.JobSpec {
	183	+	return &execpb.JobSpec{
	184	+	WorkflowPath: ".github/workflows/ci.yml", WorkflowYaml: yaml, JobKey: "j", Repo: "hello",
	185	+	Contexts: `{"github":{"repository":"octo/hello","repository_owner":"octo","event_name":"push","job":"j","ref":"refs/heads/main","ref_name":"main","ref_type":"branch","sha":"0123456789abcdef0123456789abcdef01234567","run_id":"1","run_number":"1","run_attempt":"1","server_url":"https://ermite.example","workflow":"w","actor":"octo","event":{"ref":"refs/heads/main"}}}`,
	186	+	Matrix: "{}", RunnerName: "e2e", RunnerVersion: "test", RunnerEnvironment: "self-hosted", TimeoutMinutes: 10,
	187	+	}
	188	+	}
	189	+
	190	+	func TestContainerRunsAJobAsTheRunnerUser(t *testing.T) {
	191	+	image := needE2E(t)
	192	+	d := newDriver(t, image)
	193	+	s := runJob(t, d, jobSpec(`on: push
	194	+	jobs:
	195	+	j:
	196	+	runs-on: x
	197	+	outputs:
	198	+	who: ${{ steps.id.outputs.who }}
	199	+	steps:
	200	+	- id: id
	201	+	run: \|
	202	+	echo "who=$(id -un):$(id -u):$(id -g)" >> "$GITHUB_OUTPUT"
	203	+	echo "home=$HOME pwd=$PWD"
	204	+	echo "ws=$GITHUB_WORKSPACE temp=$RUNNER_TEMP tool=$RUNNER_TOOL_CACHE"
	205	+	touch "$RUNNER_TEMP/t" "$GITHUB_WORKSPACE/w" "$HOME/h"
	206	+	echo "script=$0"
	207	+	- name: env and path
	208	+	run: \|
	209	+	mkdir -p "$RUNNER_TEMP/bin" && printf '#!/bin/sh\necho from-path\n' > "$RUNNER_TEMP/bin/mytool" && chmod +x "$RUNNER_TEMP/bin/mytool"
	210	+	echo "$RUNNER_TEMP/bin" >> "$GITHUB_PATH"
	211	+	echo "FOO=bar" >> "$GITHUB_ENV"
	212	+	- run: mytool; echo "FOO=$FOO CI=$CI GITHUB_ACTIONS=$GITHUB_ACTIONS ERMITE_CONNECT=${ERMITE_CONNECT-unset}"
	213	+	- name: python
	214	+	shell: python3 {0}
	215	+	run: print("py", 6*7)
	216	+	`), nil)
	217	+	if s.Result == nil \|\| s.Result.Conclusion != execpb.StepConclusion_STEP_CONCLUSION_SUCCESS \|\| s.Exit.Code != 0 {
	218	+	t.Fatalf("result %v exit %v\n%s", s.Result, s.Exit, strings.Join(s.Logs, "\n"))
	219	+	}
	220	+	if got := s.Result.Outputs["who"]; got != "runner:1001:1001" {
	221	+	t.Errorf("steps run as %q, want runner:1001:1001", got)
	222	+	}
	223	+	all := strings.Join(s.Logs, "\n")
	224	+	for _, want := range []string{
	225	+	"home=/home/runner pwd=/home/runner/work/hello/hello",
	226	+	"ws=/home/runner/work/hello/hello temp=/home/runner/work/_temp tool=/opt/hostedtoolcache",
	227	+	"from-path", "FOO=bar CI=true GITHUB_ACTIONS=true ERMITE_CONNECT=unset", "py 42",
	228	+	} {
	229	+	if !strings.Contains(all, want) {
	230	+	t.Errorf("log lacks %q\n%s", want, all)
	231	+	}
	232	+	}
	233	+	if !strings.HasPrefix(strings.Join(s.Steps, "\|"), "Set up job\|Run echo") \|\| s.Steps[len(s.Steps)-1] != "Complete job" {
	234	+	t.Errorf("steps %v", s.Steps)
	235	+	}
	236	+	}
	237	+
	238	+	func TestContainerCancelStopsTheJobAndOrphans(t *testing.T) {
	239	+	image := needE2E(t)
	240	+	d := newDriver(t, image)
	241	+	cancel := make(chan *execpb.Cancel, 1)
	242	+	time.AfterFunc(4*time.Second, func() { cancel <- &execpb.Cancel{Reason: "test", Kind: execpb.CancelKind_CANCEL_KIND_USER} })
	243	+	start := time.Now()
	244	+	s := runJob(t, d, jobSpec(`on: push
	245	+	jobs:
	246	+	j:
	247	+	runs-on: x
	248	+	steps:
	249	+	- run: \|
	250	+	(sleep 300 &)
	251	+	trap 'echo got-int; exit 130' INT
	252	+	echo ready
	253	+	while :; do sleep 0.2; done
	254	+	- if: always()
	255	+	run: echo cleanup-ran
	256	+	`), cancel)
	257	+	if s.Result == nil \|\| s.Result.Conclusion != execpb.StepConclusion_STEP_CONCLUSION_CANCELLED {
	258	+	t.Fatalf("result %v\n%s", s.Result, strings.Join(s.Logs, "\n"))
	259	+	}
	260	+	all := strings.Join(s.Logs, "\n")
	261	+	if !strings.Contains(all, "got-int") \|\| !strings.Contains(all, "cleanup-ran") {
	262	+	t.Errorf("SIGINT or the always() step missing:\n%s", all)
	263	+	}
	264	+	// "Complete job" kills the background sleep that the step left behind.
	265	+	if !strings.Contains(all, "Terminate orphan process: pid (") \|\| !strings.Contains(all, ") (sleep)") {
	266	+	t.Errorf("orphan cleanup missing:\n%s", all)
	267	+	}
	268	+	if time.Since(start) > 40*time.Second {
	269	+	t.Errorf("took %v", time.Since(start))
	270	+	}
	271	+	}

A internal/e2e/util_test.go +7 -0

@@ -0,0 +1,7 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package e2e
	4	+
	5	+	import "net/netip"
	6	+
	7	+	func netPrefix(s string) netip.Prefix { return netip.MustParsePrefix(s) }

M internal/execstream/exec.proto +85 -2


@@ -21,6 +21,10 @@ message Frame {
21	21		Output output = 3; // executor -> host
22	22		Exit exit = 4; // executor -> host, last frame
23	23		Cancel cancel = 5; // host -> executor
	24	+	StepStarted step_started = 6; // executor -> host
	25	+	StepCompleted step_completed = 7; // executor -> host
	26	+	Log log = 8; // executor -> host
	27	+	JobResult job_result = 9; // executor -> host, before Exit
24	28		}
25	29		}
26	30

@@ -30,11 +34,38 @@ message Hello {
30	34		string hostname = 3;
31	35		}
32	36
33		-	// Run asks the executor to run a script. CI-1 stub executor: the script is run
34		-	// with `sh -c`; an empty script means the built-in diagnostics script.
	37	+	// Run asks the executor to run a script or a job. Without job it is the stub
	38	+	// executor of sandbox-check: the script is run with `sh -c`; an empty script
	39	+	// means the built-in diagnostics script. With job the executor runs the job
	40	+	// like a GitHub Actions runner and reports it with StepStarted, Log,
	41	+	// StepCompleted and JobResult frames, then Exit.
35	42		message Run {
36	43		string script = 1;
37	44		map<string, string> env = 2;
	45	+	JobSpec job = 3;
	46	+	}
	47	+
	48	+	// JobSpec is the job to run: the fields of the job that ermite handed to the
	49	+	// host process, minus the tokens (the sandbox never sees them).
	50	+	message JobSpec {
	51	+	string workflow_path = 1;
	52	+	string workflow_yaml = 2;
	53	+	string job_key = 3;
	54	+	string matrix = 4; // JSON object
	55	+	string contexts = 5; // JSON object
	56	+	string repo = 6; // repository name
	57	+	int32 timeout_minutes = 7;
	58	+	string runner_name = 8;
	59	+	string runner_version = 9;
	60	+	// runner_environment is the runner.environment value ("self-hosted").
	61	+	string runner_environment = 10;
	62	+	bool debug = 11; // ACTIONS_STEP_DEBUG
	63	+	// deadline_ms is what is left of the job timeout when the host sends this
	64	+	// Run, in milliseconds: the timeout counts from the moment the host received
	65	+	// the job (runner.proto: Job.timeout_minutes), not from the start of the
	66	+	// executor, and booting the sandbox takes time. 0 means "use
	67	+	// timeout_minutes from now".
	68	+	int64 deadline_ms = 12;
38	69		}
39	70
40	71		enum StreamKind {

@@ -61,4 +92,56 @@ message Cancel {
61	92		// reason is for the job log and diagnostics ("cancelled by the user",
62	93		// "job timeout", ...); it never changes the behaviour.
63	94		string reason = 1;
	95	+	// kind tells the job executor how the job ends: cancelled, timed out, or
	96	+	// cancelled because the runner shuts down. The stub executor ignores it.
	97	+	CancelKind kind = 2;
	98	+	}
	99	+
	100	+	enum CancelKind {
	101	+	CANCEL_KIND_UNSPECIFIED = 0; // same as USER
	102	+	CANCEL_KIND_USER = 1;
	103	+	CANCEL_KIND_TIMEOUT = 2;
	104	+	CANCEL_KIND_SHUTDOWN = 3;
	105	+	}
	106	+
	107	+	// Steps are numbered like GitHub's log: "Set up job" is 1, the steps of the
	108	+	// workflow follow, "Complete job" is last.
	109	+	message StepStarted {
	110	+	int32 number = 1;
	111	+	string name = 2;
	112	+	int64 unix_nano = 3;
	113	+	}
	114	+
	115	+	message StepCompleted {
	116	+	int32 number = 1;
	117	+	string name = 2;
	118	+	StepConclusion conclusion = 3;
	119	+	int64 unix_nano = 4;
	120	+	}
	121	+
	122	+	enum StepConclusion {
	123	+	STEP_CONCLUSION_UNSPECIFIED = 0;
	124	+	STEP_CONCLUSION_SUCCESS = 1;
	125	+	STEP_CONCLUSION_FAILURE = 2;
	126	+	STEP_CONCLUSION_CANCELLED = 3;
	127	+	STEP_CONCLUSION_SKIPPED = 4;
	128	+	STEP_CONCLUSION_TIMED_OUT = 5; // jobs only
	129	+	}
	130	+
	131	+	// Log is one line of the job log, without terminator, valid UTF-8. A line of
	132	+	// more than 64 KiB is cut by the executor into pieces at character boundaries;
	133	+	// every piece but the first has continued set (the AppendLog rule).
	134	+	message Log {
	135	+	int32 step = 1;
	136	+	int64 unix_nano = 2;
	137	+	string text = 3;
	138	+	bool continued = 4;
	139	+	}
	140	+
	141	+	// JobResult is the outcome of the job; Exit follows.
	142	+	message JobResult {
	143	+	StepConclusion conclusion = 1;
	144	+	map<string, string> outputs = 2;
	145	+	string error = 3;
	146	+	bool runner_shutdown = 4;
64	147		}

M internal/execstream/execpb/exec.pb.go +732 -44 generated file: collapsed

load diff

M internal/execstream/frame.go +42 -3


@@ -25,7 +25,9 @@ import (
25	25		)
26	26
27	27		// MaxFrame is the largest accepted serialized frame.
28		-	const MaxFrame = 1 << 20
	28	+	// It leaves room for the largest job result: outputs of 1 MiB in total (the
	29	+	// limit of ermite) plus the overhead of the encoding.
	30	+	const MaxFrame = 2 << 20
29	31
30	32		// ChunkSize is the largest output payload placed in one Output frame.
31	33		const ChunkSize = 32 * 1024

@@ -100,6 +102,24 @@ func (c Conn) Recv() (execpb.Frame, error) {
100	102		// a Cancel; the caller bounds that wait by stopping the sandbox, which closes
101	103		// the stream and makes Drive return an error.
102	104		func Drive(rw io.ReadWriter, run execpb.Run, cancel <-chan execpb.Cancel, onOutput func(execpb.Output)) (execpb.Hello, *execpb.Exit, error) {
	105	+	return DriveJob(rw, run, cancel, JobHandler{Output: onOutput})
	106	+	}
	107	+
	108	+	// JobHandler receives the frames of a session. Nil functions ignore their
	109	+	// frames. The functions are called from the goroutine of DriveJob, in the
	110	+	// order of the stream.
	111	+	type JobHandler struct {
	112	+	Hello func(*execpb.Hello)
	113	+	Output func(*execpb.Output)
	114	+	StepStarted func(*execpb.StepStarted)
	115	+	StepCompleted func(*execpb.StepCompleted)
	116	+	Log func(*execpb.Log)
	117	+	JobResult func(*execpb.JobResult)
	118	+	}
	119	+
	120	+	// DriveJob is Drive for a session that carries a job: besides Output frames it
	121	+	// delivers the step, log and result frames to h.
	122	+	func DriveJob(rw io.ReadWriter, run execpb.Run, cancel <-chan execpb.Cancel, h JobHandler) (execpb.Hello, execpb.Exit, error) {
103	123		c := NewConn(rw)
104	124		first, err := c.Recv()
105	125		if err != nil {

@@ -109,6 +129,9 @@ func Drive(rw io.ReadWriter, run execpb.Run, cancel <-chan execpb.Cancel, onOu
109	129		if hello == nil {
110	130		return nil, nil, fmt.Errorf("execstream: first frame is not hello")
111	131		}
	132	+	if h.Hello != nil {
	133	+	h.Hello(hello)
	134	+	}
112	135		if err := c.Send(&execpb.Frame{Body: &execpb.Frame_Run{Run: run}}); err != nil {
113	136		return hello, nil, fmt.Errorf("execstream: sending run: %w", err)
114	137		}

@@ -137,8 +160,24 @@ func Drive(rw io.ReadWriter, run execpb.Run, cancel <-chan execpb.Cancel, onOu
137	160		}
138	161		switch b := f.Body.(type) {
139	162		case *execpb.Frame_Output:
140		-	if onOutput != nil {
141		-	onOutput(b.Output)
	163	+	if h.Output != nil {
	164	+	h.Output(b.Output)
	165	+	}
	166	+	case *execpb.Frame_StepStarted:
	167	+	if h.StepStarted != nil {
	168	+	h.StepStarted(b.StepStarted)
	169	+	}
	170	+	case *execpb.Frame_StepCompleted:
	171	+	if h.StepCompleted != nil {
	172	+	h.StepCompleted(b.StepCompleted)
	173	+	}
	174	+	case *execpb.Frame_Log:
	175	+	if h.Log != nil {
	176	+	h.Log(b.Log)
	177	+	}
	178	+	case *execpb.Frame_JobResult:
	179	+	if h.JobResult != nil {
	180	+	h.JobResult(b.JobResult)
142	181		}
143	182		case *execpb.Frame_Exit:
144	183		return hello, b.Exit, nil

M internal/execstream/frame_test.go +36 -0


@@ -8,6 +8,7 @@ import (
8	8		"errors"
9	9		"io"
10	10		"net"
	11	+	"strings"
11	12		"testing"
12	13		"testing/iotest"
13	14		"time"

@@ -136,3 +137,38 @@ func TestDriveHelloMissing(t *testing.T) {
136	137		t.Fatal("want error when the first frame is not hello")
137	138		}
138	139		}
	140	+
	141	+	func TestDriveJobDeliversJobFrames(t *testing.T) {
	142	+	host, guest := net.Pipe()
	143	+	go func() {
	144	+	g := NewConn(guest)
	145	+	_ = g.Send(&execpb.Frame{Body: &execpb.Frame_Hello{Hello: &execpb.Hello{Version: "v"}}})
	146	+	r, _ := g.Recv()
	147	+	if r.GetRun().GetJob().GetJobKey() != "build" {
	148	+	t.Errorf("run: %v", r)
	149	+	}
	150	+	for _, b := range []*execpb.Frame{
	151	+	{Body: &execpb.Frame_StepStarted{StepStarted: &execpb.StepStarted{Number: 1, Name: "Set up job"}}},
	152	+	{Body: &execpb.Frame_Log{Log: &execpb.Log{Step: 1, Text: "hello", UnixNano: 5}}},
	153	+	{Body: &execpb.Frame_StepCompleted{StepCompleted: &execpb.StepCompleted{Number: 1, Conclusion: execpb.StepConclusion_STEP_CONCLUSION_SUCCESS}}},
	154	+	{Body: &execpb.Frame_JobResult{JobResult: &execpb.JobResult{Conclusion: execpb.StepConclusion_STEP_CONCLUSION_SUCCESS, Outputs: map[string]string{"a": "b"}}}},
	155	+	{Body: &execpb.Frame_Exit{Exit: &execpb.Exit{}}},
	156	+	} {
	157	+	_ = g.Send(b)
	158	+	}
	159	+	}()
	160	+	var got []string
	161	+	_, exit, err := DriveJob(host, &execpb.Run{Job: &execpb.JobSpec{JobKey: "build"}}, nil, JobHandler{
	162	+	StepStarted: func(s *execpb.StepStarted) { got = append(got, "start:"+s.Name) },
	163	+	Log: func(l *execpb.Log) { got = append(got, "log:"+l.Text) },
	164	+	StepCompleted: func(s *execpb.StepCompleted) { got = append(got, "done:"+s.Conclusion.String()) },
	165	+	JobResult: func(r *execpb.JobResult) { got = append(got, "result:"+r.Outputs["a"]) },
	166	+	})
	167	+	if err != nil \|\| exit == nil {
	168	+	t.Fatal(err)
	169	+	}
	170	+	want := "start:Set up job log:hello done:STEP_CONCLUSION_SUCCESS result:b"
	171	+	if strings.Join(got, " ") != want {
	172	+	t.Fatalf("%v", got)
	173	+	}
	174	+	}

A internal/execstream/split.go +38 -0

@@ -0,0 +1,38 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package execstream
	4	+
	5	+	import (
	6	+	"strings"
	7	+	"unicode/utf8"
	8	+	)
	9	+
	10	+	// MaxLogLine is the largest piece of a log line that fits an AppendLog line
	11	+	// (64 KiB); longer lines are cut by SplitLine.
	12	+	const MaxLogLine = 64 * 1024
	13	+
	14	+	// SplitLine cuts text into pieces of at most max bytes at UTF-8 character
	15	+	// boundaries. The result has one piece for a short line, and always at least
	16	+	// one (an empty line is one empty piece). Invalid UTF-8 in text is replaced
	17	+	// with U+FFFD first, because the protocol only carries valid text.
	18	+	func SplitLine(text string, max int) []string {
	19	+	if !utf8.ValidString(text) {
	20	+	text = strings.ToValidUTF8(text, "\ufffd")
	21	+	}
	22	+	if len(text) <= max {
	23	+	return []string{text}
	24	+	}
	25	+	var out []string
	26	+	for len(text) > max {
	27	+	cut := max
	28	+	for cut > 0 && !utf8.RuneStart(text[cut]) {
	29	+	cut--
	30	+	}
	31	+	if cut == 0 { // cannot happen with max >= utf8.UTFMax
	32	+	cut = max
	33	+	}
	34	+	out = append(out, text[:cut])
	35	+	text = text[cut:]
	36	+	}
	37	+	return append(out, text)
	38	+	}

A internal/execstream/split_test.go +41 -0

@@ -0,0 +1,41 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package execstream
	4	+
	5	+	import (
	6	+	"strings"
	7	+	"testing"
	8	+	"unicode/utf8"
	9	+	)
	10	+
	11	+	func TestSplitLine(t *testing.T) {
	12	+	if got := SplitLine("", 10); len(got) != 1 \|\| got[0] != "" {
	13	+	t.Fatalf("empty: %q", got)
	14	+	}
	15	+	if got := SplitLine("short", 10); len(got) != 1 \|\| got[0] != "short" {
	16	+	t.Fatalf("%q", got)
	17	+	}
	18	+	// Cuts at character boundaries and joins back exactly.
	19	+	in := strings.Repeat("é€😀a", 40) // 2, 3, 4, 1 bytes
	20	+	for _, max := range []int{4, 5, 7, 10, 33} {
	21	+	parts := SplitLine(in, max)
	22	+	var b strings.Builder
	23	+	for _, p := range parts {
	24	+	if len(p) > max \|\| !utf8.ValidString(p) \|\| p == "" {
	25	+	t.Fatalf("max %d: bad piece %q", max, p)
	26	+	}
	27	+	b.WriteString(p)
	28	+	}
	29	+	if b.String() != in {
	30	+	t.Fatalf("max %d: pieces do not rebuild the line", max)
	31	+	}
	32	+	}
	33	+	// Invalid UTF-8 is replaced before cutting.
	34	+	if got := SplitLine("a\xffb", 10); got[0] != "a\ufffdb" {
	35	+	t.Fatalf("%q", got)
	36	+	}
	37	+	big := SplitLine(strings.Repeat("x", 3*MaxLogLine+5), MaxLogLine)
	38	+	if len(big) != 4 \|\| len(big[0]) != MaxLogLine \|\| len(big[3]) != 5 {
	39	+	t.Fatalf("%d pieces", len(big))
	40	+	}
	41	+	}

A internal/executor/cancel.go +52 -0

@@ -0,0 +1,52 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package executor
	4	+
	5	+	import "sync"
	6	+
	7	+	// CancelReason says why a job is stopped.
	8	+	type CancelReason int
	9	+
	10	+	// Cancel reasons.
	11	+	const (
	12	+	// CancelUser is a cancellation by the owner or by fail-fast: the job ends
	13	+	// cancelled.
	14	+	CancelUser CancelReason = iota + 1
	15	+	// CancelTimeout is the job timeout: the job ends timed_out.
	16	+	CancelTimeout
	17	+	// CancelShutdown is the runner stopping: the job ends cancelled with
	18	+	// RunnerShutdown set, and the steps that did not start are skipped.
	19	+	CancelShutdown
	20	+	)
	21	+
	22	+	// Canceller stops a job. The first Cancel wins; later calls are ignored (the
	23	+	// server only sends the first reason too).
	24	+	type Canceller struct {
	25	+	once sync.Once
	26	+	ch chan struct{}
	27	+	mu sync.Mutex
	28	+	reason CancelReason
	29	+	}
	30	+
	31	+	// NewCanceller returns a Canceller that has not fired.
	32	+	func NewCanceller() *Canceller { return &Canceller{ch: make(chan struct{})} }
	33	+
	34	+	// Cancel stops the job for reason.
	35	+	func (c *Canceller) Cancel(reason CancelReason) {
	36	+	c.once.Do(func() {
	37	+	c.mu.Lock()
	38	+	c.reason = reason
	39	+	c.mu.Unlock()
	40	+	close(c.ch)
	41	+	})
	42	+	}
	43	+
	44	+	// Done is closed when the job is cancelled.
	45	+	func (c *Canceller) Done() <-chan struct{} { return c.ch }
	46	+
	47	+	// Reason returns why the job was cancelled; zero before Done is closed.
	48	+	func (c *Canceller) Reason() CancelReason {
	49	+	c.mu.Lock()
	50	+	defer c.mu.Unlock()
	51	+	return c.reason
	52	+	}

A internal/executor/commands.go +297 -0

@@ -0,0 +1,297 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package executor
	4	+
	5	+	import (
	6	+	"fmt"
	7	+	"strings"
	8	+	"unicode"
	9	+	)
	10	+
	11	+	// Workflow commands in the output of a step: a port of ActionCommand.cs
	12	+	// (Runner.Common) and of the parts of ActionCommandManager.cs that CI-1 needs.
	13	+	// Masking (add-mask) and problem matchers are CI-2; they are announced in the
	14	+	// log and otherwise ignored.
	15	+
	16	+	type escapeMapping struct{ token, replacement string }
	17	+
	18	+	// _escapeMappings, _escapeDataMappings, _escapePropertyMappings of
	19	+	// ActionCommand.cs:11-35, in order: the replacements run one after the other.
	20	+	var (
	21	+	escapeMappings = []escapeMapping{{";", "%3B"}, {"\r", "%0D"}, {"\n", "%0A"}, {"]", "%5D"}, {"%", "%25"}}
	22	+	escapeData = []escapeMapping{{"\r", "%0D"}, {"\n", "%0A"}, {"%", "%25"}}
	23	+	escapeProperty = []escapeMapping{{"\r", "%0D"}, {"\n", "%0A"}, {":", "%3A"}, {",", "%2C"}, {"%", "%25"}}
	24	+	)
	25	+
	26	+	func unescape(s string, maps []escapeMapping) string {
	27	+	for _, m := range maps {
	28	+	s = strings.ReplaceAll(s, m.replacement, m.token)
	29	+	}
	30	+	return s
	31	+	}
	32	+
	33	+	// actionCommand is a parsed workflow command.
	34	+	type actionCommand struct {
	35	+	Name string
	36	+	Props map[string]string // keys compared case-insensitively (lower-cased)
	37	+	Data string
	38	+	}
	39	+
	40	+	func (c *actionCommand) prop(name string) string { return c.Props[strings.ToLower(name)] }
	41	+
	42	+	const commandKey = "::"
	43	+
	44	+	// parseCommandV2 is ActionCommand.TryParseV2 (ActionCommand.cs:57): "::name
	45	+	// key=value,key=value::data".
	46	+	func parseCommandV2(message string, registered func(string) bool) (*actionCommand, bool) {
	47	+	// .NET TrimStart() trims Unicode white space.
	48	+	message = strings.TrimLeftFunc(message, unicode.IsSpace)
	49	+	if !strings.HasPrefix(message, commandKey) {
	50	+	return nil, false
	51	+	}
	52	+	end := strings.Index(message[len(commandKey):], commandKey)
	53	+	if end < 0 {
	54	+	return nil, false
	55	+	}
	56	+	end += len(commandKey)
	57	+	info := message[len(commandKey):end]
	58	+	name, rest, hasSpace := strings.Cut(info, " ")
	59	+	if !registered(name) {
	60	+	return nil, false
	61	+	}
	62	+	cmd := &actionCommand{Name: name, Props: map[string]string{}}
	63	+	if hasSpace && len(info) > len(name) { // spaceIndex > 0
	64	+	for _, p := range strings.Split(strings.TrimSpace(rest), ",") {
	65	+	if p == "" {
	66	+	continue
	67	+	}
	68	+	k, v, ok := strings.Cut(p, "=")
	69	+	if ok && k != "" && v != "" { // Split(RemoveEmptyEntries): both parts non-empty
	70	+	cmd.Props[strings.ToLower(k)] = unescape(v, escapeProperty)
	71	+	}
	72	+	}
	73	+	}
	74	+	cmd.Data = unescape(message[end+len(commandKey):], escapeData)
	75	+	return cmd, true
	76	+	}
	77	+
	78	+	// parseCommandV1 is ActionCommand.TryParse (ActionCommand.cs:127): the legacy
	79	+	// "##[name key=value;key=value]data".
	80	+	func parseCommandV1(message string, registered func(string) bool) (*actionCommand, bool) {
	81	+	const prefix = "##["
	82	+	pi := strings.Index(message, prefix)
	83	+	if pi < 0 {
	84	+	return nil, false
	85	+	}
	86	+	rb := strings.IndexByte(message[pi:], ']')
	87	+	if rb < 0 {
	88	+	return nil, false
	89	+	}
	90	+	rb += pi
	91	+	info := message[pi+len(prefix) : rb]
	92	+	name, rest, hasSpace := strings.Cut(info, " ")
	93	+	if !registered(name) {
	94	+	return nil, false
	95	+	}
	96	+	cmd := &actionCommand{Name: name, Props: map[string]string{}}
	97	+	if hasSpace && len(info) > len(name) {
	98	+	for _, p := range strings.Split(rest, ";") {
	99	+	if p == "" {
	100	+	continue
	101	+	}
	102	+	k, v, ok := strings.Cut(p, "=")
	103	+	if ok && k != "" && v != "" {
	104	+	cmd.Props[strings.ToLower(k)] = unescape(v, escapeMappings)
	105	+	}
	106	+	}
	107	+	}
	108	+	cmd.Data = unescape(message[rb+1:], escapeMappings)
	109	+	return cmd, true
	110	+	}
	111	+
	112	+	// Messages of Constants.cs:234-236.
	113	+	const (
	114	+	unsupportedCommandMessage = "The `%s` command is deprecated and will be disabled soon. Please upgrade to using Environment Files. For more information see: https://github.blog/changelog/2022-10-11-github-actions-deprecating-save-state-and-set-output-commands/"
	115	+	unsupportedCommandMessageDisabled = "The `%s` command is disabled. Please upgrade to using Environment Files or opt into unsecure command execution by setting the `ACTIONS_ALLOW_UNSECURE_COMMANDS` environment variable to `true`. For more information see: https://github.blog/changelog/2020-10-01-github-actions-deprecating-set-env-and-add-path-commands/"
	116	+	unsupportedStopTokenDisabled = "You cannot use a endToken that is an empty string, the string 'pause-logging', or another workflow command. For more information see: https://docs.github.com/actions/learn-github-actions/workflow-commands-for-github-actions#example-stopping-and-starting-workflow-commands or opt into insecure command execution by setting the `ACTIONS_ALLOW_UNSECURE_STOPCOMMAND_TOKENS` environment variable to `true`."
	117	+	)
	118	+
	119	+	// commandNames are the commands the executor registers, in ActionCommandManager
	120	+	// (the names of the extensions at ActionCommandManager.cs). add-mask,
	121	+	// add-matcher and remove-matcher are recognised and reported as not supported
	122	+	// yet.
	123	+	var commandNames = []string{
	124	+	"set-env", "set-output", "save-state", "add-mask", "add-path", "add-matcher", "remove-matcher",
	125	+	"debug", "warning", "error", "notice", "group", "endgroup", "echo", "stop-commands",
	126	+	}
	127	+
	128	+	// commandHost is what commands act on: the step that is running.
	129	+	type commandHost interface {
	130	+	// Out writes a log line (several when it contains line feeds).
	131	+	Out(text string)
	132	+	// Tagged writes a ##[tag]message line.
	133	+	Tagged(tag, message string)
	134	+	Debug(msg string)
	135	+	SetOutput(name, value string)
	136	+	SaveState(name, value string)
	137	+	SetEnv(name, value string)
	138	+	AddPath(path string)
	139	+	CommandFailed()
	140	+	// EnvFlag reads a variable of the env context, then of the process.
	141	+	EnvFlag(name string) bool
	142	+	}
	143	+
	144	+	// commandManager is ActionCommandManager: one per step.
	145	+	type commandManager struct {
	146	+	host commandHost
	147	+	registered map[string]bool
	148	+	stopped bool
	149	+	stopToken string
	150	+	echo bool
	151	+	}
	152	+
	153	+	func newCommandManager(h commandHost, echo bool) *commandManager {
	154	+	m := &commandManager{host: h, registered: map[string]bool{}, echo: echo}
	155	+	for _, n := range commandNames {
	156	+	m.registered[strings.ToLower(n)] = true
	157	+	}
	158	+	return m
	159	+	}
	160	+
	161	+	func (m *commandManager) isRegistered(name string) bool { return m.registered[strings.ToLower(name)] }
	162	+
	163	+	// TryProcess handles one line of output. It returns true when the line was a
	164	+	// command and must not be logged as plain text (ActionCommandManager.cs:61).
	165	+	func (m *commandManager) TryProcess(input string) bool {
	166	+	if input == "" {
	167	+	return false
	168	+	}
	169	+	cmd, ok := parseCommandV2(input, m.isRegistered)
	170	+	if !ok {
	171	+	cmd, ok = parseCommandV1(input, m.isRegistered)
	172	+	}
	173	+	if !ok {
	174	+	return false
	175	+	}
	176	+	name := strings.ToLower(cmd.Name)
	177	+	if m.stopped {
	178	+	if m.stopToken != "" && strings.EqualFold(cmd.Name, m.stopToken) {
	179	+	m.host.Out(input)
	180	+	m.host.Debug("Resume processing commands")
	181	+	delete(m.registered, strings.ToLower(m.stopToken))
	182	+	m.stopped, m.stopToken = false, ""
	183	+	return true
	184	+	}
	185	+	m.host.Debug(fmt.Sprintf("Process commands has been stopped and waiting for '##[%s]' to resume.", m.stopToken))
	186	+	return false
	187	+	}
	188	+	if name == "stop-commands" {
	189	+	if err := m.validateStopToken(cmd.Data); err != nil {
	190	+	m.fail(input, false, err)
	191	+	return true
	192	+	}
	193	+	m.stopToken, m.stopped = cmd.Data, true
	194	+	m.registered[strings.ToLower(cmd.Data)] = true
	195	+	m.host.Out(input)
	196	+	m.host.Debug("Paused processing commands until the token you called ::stopCommands:: with is received")
	197	+	return true
	198	+	}
	199	+	omitEcho := name == "debug" \|\| name == "warning" \|\| name == "error" \|\| name == "notice" \|\| name == "add-mask"
	200	+	if m.echo && !omitEcho {
	201	+	m.host.Out(input)
	202	+	}
	203	+	if err := m.run(name, cmd); err != nil {
	204	+	m.fail(input, omitEcho, err)
	205	+	}
	206	+	return true
	207	+	}
	208	+
	209	+	func (m *commandManager) fail(input string, omitEcho bool, err error) {
	210	+	info := input
	211	+	if omitEcho {
	212	+	info = ""
	213	+	}
	214	+	m.host.Tagged("##[error]", fmt.Sprintf("Unable to process command '%s' successfully.", info))
	215	+	m.host.Tagged("##[error]", err.Error())
	216	+	m.host.CommandFailed()
	217	+	}
	218	+
	219	+	func (m *commandManager) validateStopToken(token string) error {
	220	+	invalid := m.isRegistered(token) \|\| token == "" \|\| strings.EqualFold(token, "pause-logging")
	221	+	if invalid && !m.host.EnvFlag("ACTIONS_ALLOW_UNSECURE_STOPCOMMAND_TOKENS") {
	222	+	return msgError(unsupportedStopTokenDisabled)
	223	+	}
	224	+	return nil
	225	+	}
	226	+
	227	+	func (m commandManager) run(name string, c actionCommand) error {
	228	+	h := m.host
	229	+	switch name {
	230	+	case "group":
	231	+	h.Out("##[group]" + c.Data) // GroupingCommandExtension, ActionCommandManager.cs:802
	232	+	case "endgroup":
	233	+	h.Out("##[endgroup]")
	234	+	case "debug":
	235	+	h.Debug(c.Data)
	236	+	case "warning", "error", "notice":
	237	+	// IssueCommandExtension: the annotation is logged; ermite does not store
	238	+	// annotations yet (CI-2), so file, line and title are dropped.
	239	+	h.Tagged("##["+name+"]", c.Data)
	240	+	case "echo":
	241	+	if c.Data == "" {
	242	+	return ghErrorf("Value cannot be null or empty. (Parameter 'value')")
	243	+	}
	244	+	switch strings.ToUpper(strings.TrimSpace(c.Data)) {
	245	+	case "ON":
	246	+	m.echo = true
	247	+	h.Debug("Setting echo command value to 'on'")
	248	+	case "OFF":
	249	+	m.echo = false
	250	+	h.Debug("Setting echo command value to 'off'")
	251	+	default:
	252	+	return ghErrorf("Invalid echo command value. Possible values can be: 'on', 'off'. Current value is: '%s'.", c.Data)
	253	+	}
	254	+	case "set-output":
	255	+	h.Tagged("##[warning]", fmt.Sprintf(unsupportedCommandMessage, name))
	256	+	n := c.prop("name")
	257	+	if n == "" {
	258	+	return ghErrorf("Required field 'name' is missing in ##[set-output] command.")
	259	+	}
	260	+	h.SetOutput(n, c.Data)
	261	+	h.Debug(fmt.Sprintf("steps.<id>.outputs.%s='%s'", n, c.Data))
	262	+	case "save-state":
	263	+	h.Tagged("##[warning]", fmt.Sprintf(unsupportedCommandMessage, name))
	264	+	n := c.prop("name")
	265	+	if n == "" {
	266	+	return ghErrorf("Required field 'name' is missing in ##[save-state] command.")
	267	+	}
	268	+	h.SaveState(n, c.Data)
	269	+	h.Debug(fmt.Sprintf("Save intra-action state %s = %s", n, c.Data))
	270	+	case "set-env":
	271	+	if !h.EnvFlag("ACTIONS_ALLOW_UNSECURE_COMMANDS") {
	272	+	return ghErrorf(unsupportedCommandMessageDisabled, name)
	273	+	}
	274	+	n := c.prop("name")
	275	+	if n == "" {
	276	+	return ghErrorf("Required field 'name' is missing in ##[set-env] command.")
	277	+	}
	278	+	if b := blockedEnvName(n); b != "" {
	279	+	h.Tagged("##[error]", fmt.Sprintf("Can't update %s environment variable using ::set-env:: command.", b))
	280	+	return nil
	281	+	}
	282	+	h.SetEnv(n, c.Data)
	283	+	h.Debug(fmt.Sprintf("%s='%s'", n, c.Data))
	284	+	case "add-path":
	285	+	if !h.EnvFlag("ACTIONS_ALLOW_UNSECURE_COMMANDS") {
	286	+	return ghErrorf(unsupportedCommandMessageDisabled, name)
	287	+	}
	288	+	if c.Data == "" {
	289	+	return ghErrorf("Value cannot be null or empty. (Parameter 'path')")
	290	+	}
	291	+	h.AddPath(c.Data)
	292	+	case "add-mask", "add-matcher", "remove-matcher":
	293	+	// Not supported yet (CI-2). The value of add-mask is not echoed.
	294	+	h.Tagged("##[warning]", fmt.Sprintf("The `%s` command is not supported yet by this runner.", name))
	295	+	}
	296	+	return nil
	297	+	}

A internal/executor/contexts.go +465 -0

@@ -0,0 +1,465 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package executor
	4	+
	5	+	import (
	6	+	"fmt"
	7	+	"path/filepath"
	8	+	"runtime"
	9	+	"slices"
	10	+	"strings"
	11	+
	12	+	"bnema.dev/ermite-actions/expr"
	13	+	)
	14	+
	15	+	// ordered is a string map that keeps the position of a key when it is set
	16	+	// again, like the Dictionary<string,string> the official runner uses for
	17	+	// Global.EnvironmentVariables and the step environment (the order shows in the
	18	+	// "env:" block of the log). Keys are case-sensitive, as on Linux
	19	+	// (VarUtil.EnvironmentVariableKeyComparer).
	20	+	type ordered struct {
	21	+	keys []string
	22	+	vals map[string]string
	23	+	}
	24	+
	25	+	func newOrdered() *ordered { return &ordered{vals: map[string]string{}} }
	26	+
	27	+	func (o *ordered) Set(k, v string) {
	28	+	if _, ok := o.vals[k]; !ok {
	29	+	o.keys = append(o.keys, k)
	30	+	}
	31	+	o.vals[k] = v
	32	+	}
	33	+
	34	+	func (o *ordered) Get(k string) (string, bool) { v, ok := o.vals[k]; return v, ok }
	35	+	func (o *ordered) Len() int { return len(o.keys) }
	36	+	func (o *ordered) Keys() []string { return slices.Clone(o.keys) }
	37	+
	38	+	func (o ordered) Clone() ordered {
	39	+	c := newOrdered()
	40	+	for _, k := range o.keys {
	41	+	c.Set(k, o.vals[k])
	42	+	}
	43	+	return c
	44	+	}
	45	+
	46	+	// Dict is the value of the env context. The context of the official runner is
	47	+	// case-sensitive on Linux (CaseSensitiveDictionaryContextData); package expr
	48	+	// has only case-insensitive dictionaries, so two names that differ by case
	49	+	// collapse to the first in expressions (they stay distinct in the process
	50	+	// environment). A known deviation.
	51	+	func (o ordered) Dict() expr.Dictionary {
	52	+	d := expr.NewDictionary()
	53	+	for _, k := range o.keys {
	54	+	d.Add(k, expr.String(o.vals[k]))
	55	+	}
	56	+	return d
	57	+	}
	58	+
	59	+	// setKey returns d with key set to v: the value is replaced in place when the
	60	+	// key (compared case-insensitively) exists, else appended.
	61	+	func setKey(d expr.Dictionary, key string, v expr.Value) expr.Dictionary {
	62	+	out := expr.NewDictionary()
	63	+	done := false
	64	+	for _, p := range d.Pairs() {
	65	+	if strings.EqualFold(p.Key, key) {
	66	+	if !done {
	67	+	out.Add(p.Key, v)
	68	+	done = true
	69	+	}
	70	+	continue
	71	+	}
	72	+	out.Add(p.Key, p.Value)
	73	+	}
	74	+	if !done {
	75	+	out.Add(key, v)
	76	+	}
	77	+	return out
	78	+	}
	79	+
	80	+	func str(d *expr.Dictionary, key string) string {
	81	+	if v, ok := d.Get(key); ok {
	82	+	if s, ok := v.(expr.String); ok {
	83	+	return string(s)
	84	+	}
	85	+	}
	86	+	return ""
	87	+	}
	88	+
	89	+	// Layout is the directory layout of a job, like a hosted runner's
	90	+	// (/home/runner/work/<repo>/<repo>, RUNNER_TEMP, RUNNER_TOOL_CACHE).
	91	+	type Layout struct {
	92	+	Home string // HOME of the runner user
	93	+	// WorkRoot is the _work directory: <Home>/work.
	94	+	WorkRoot string
	95	+	// Temp is RUNNER_TEMP: <WorkRoot>/_temp.
	96	+	Temp string
	97	+	// ToolCache is RUNNER_TOOL_CACHE.
	98	+	ToolCache string
	99	+	// RunnerWorkspace is RUNNER_WORKSPACE: <WorkRoot>/<repo>.
	100	+	RunnerWorkspace string
	101	+	// Workspace is GITHUB_WORKSPACE: <WorkRoot>/<repo>/<repo>.
	102	+	Workspace string
	103	+	}
	104	+
	105	+	// DefaultToolCache is RUNNER_TOOL_CACHE on the ermite image.
	106	+	const DefaultToolCache = "/opt/hostedtoolcache"
	107	+
	108	+	// NewLayout lays out a job of the repository repo (its name, without owner)
	109	+	// under home. An empty toolCache means DefaultToolCache.
	110	+	func NewLayout(home, repo, toolCache string) Layout {
	111	+	if toolCache == "" {
	112	+	toolCache = DefaultToolCache
	113	+	}
	114	+	work := filepath.Join(home, "work")
	115	+	return Layout{
	116	+	Home: home, WorkRoot: work, Temp: filepath.Join(work, "_temp"), ToolCache: toolCache,
	117	+	RunnerWorkspace: filepath.Join(work, repo), Workspace: filepath.Join(work, repo, repo),
	118	+	}
	119	+	}
	120	+
	121	+	// RunnerInfo describes the runner for the runner context.
	122	+	type RunnerInfo struct {
	123	+	Name string
	124	+	// Environment is "self-hosted" (the default) or "github-hosted".
	125	+	Environment string
	126	+	// Arch is "X64" or "ARM64"; empty means the architecture of this process.
	127	+	Arch string
	128	+	}
	129	+
	130	+	func (r RunnerInfo) arch() string {
	131	+	if r.Arch != "" {
	132	+	return r.Arch
	133	+	}
	134	+	switch runtime.GOARCH {
	135	+	case "amd64":
	136	+	return "X64"
	137	+	case "arm64":
	138	+	return "ARM64"
	139	+	case "arm":
	140	+	return "ARM"
	141	+	}
	142	+	return "X86"
	143	+	}
	144	+
	145	+	// repoName returns the repository name of "owner/name".
	146	+	func repoName(full string) string {
	147	+	if i := strings.LastIndexByte(full, '/'); i >= 0 {
	148	+	return full[i+1:]
	149	+	}
	150	+	return full
	151	+	}
	152	+
	153	+	// stepCtx is one entry of the steps context.
	154	+	type stepCtx struct {
	155	+	outputs *ordered
	156	+	outcome string // "" until the step completes
	157	+	concl string
	158	+	}
	159	+
	160	+	// State holds the contexts of a job as the official runner does
	161	+	// (ExecutionContext.ExpressionValues): the contexts the server computed, and
	162	+	// github, runner, job, env and steps, which the runner owns.
	163	+	type State struct {
	164	+	layout Layout
	165	+	github *expr.Dictionary
	166	+	runner *expr.Dictionary
	167	+	job *expr.Dictionary
	168	+	// other holds needs, strategy, matrix, inputs, vars: copied from the server.
	169	+	other map[string]expr.Value
	170	+	// env is Global.EnvironmentVariables: the job-level environment and what
	171	+	// GITHUB_ENV adds. A step's own env is not in it.
	172	+	env *ordered
	173	+	// steps is the steps context, in order of first mention.
	174	+	stepOrder []string
	175	+	steps map[string]*stepCtx
	176	+	// prepend is Global.PrependPath in the order of the GITHUB_PATH entries.
	177	+	prepend []string
	178	+	// status is job.status: "success", "failure" or "cancelled".
	179	+	status string
	180	+	}
	181	+
	182	+	// serverContexts are the keys of the contexts JSON that the runner reads.
	183	+	var serverContexts = []string{"needs", "strategy", "matrix", "inputs", "vars"}
	184	+
	185	+	// NewState builds the contexts of a job from the JSON the server sent (the
	186	+	// "contexts" field of the job) and the matrix JSON, with the context
	187	+	// initialisation of JobRunner.cs:142-177 and ExecutionContext.InitializeJob
	188	+	// (ExecutionContext.cs:1001-1056).
	189	+	func NewState(contextsJSON, matrixJSON, jobKey string, r RunnerInfo, repoFallback string) (*State, Layout, error) {
	190	+	root := expr.NewDictionary()
	191	+	if strings.TrimSpace(contextsJSON) != "" {
	192	+	v, err := expr.FromJSON([]byte(contextsJSON))
	193	+	if err != nil {
	194	+	return nil, Layout{}, fmt.Errorf("job contexts: %w", err)
	195	+	}
	196	+	d, ok := v.(*expr.Dictionary)
	197	+	if !ok {
	198	+	return nil, Layout{}, fmt.Errorf("job contexts: want a JSON object")
	199	+	}
	200	+	root = d
	201	+	}
	202	+	s := &State{other: map[string]expr.Value{}, env: newOrdered(), steps: map[string]*stepCtx{}, status: "success"}
	203	+
	204	+	// github: token first, then what the server gave (ExecutionContext.cs:1023-1037).
	205	+	gh := expr.NewDictionary()
	206	+	gh.Add("token", expr.String(""))
	207	+	if v, ok := root.Get("github"); ok {
	208	+	d, ok := v.(*expr.Dictionary)
	209	+	if !ok {
	210	+	return nil, Layout{}, fmt.Errorf("job contexts: github must be an object")
	211	+	}
	212	+	for _, p := range d.Pairs() {
	213	+	if strings.EqualFold(p.Key, "token") {
	214	+	continue
	215	+	}
	216	+	gh.Add(p.Key, p.Value)
	217	+	}
	218	+	}
	219	+	if str(gh, "job") == "" && jobKey != "" {
	220	+	gh = setKey(gh, "job", expr.String(jobKey))
	221	+	}
	222	+	repo := repoName(str(gh, "repository"))
	223	+	if repo == "" {
	224	+	repo = repoFallback
	225	+	}
	226	+	if repo == "" {
	227	+	return nil, Layout{}, fmt.Errorf("job contexts: github.repository is missing")
	228	+	}
	229	+	home := "/home/runner"
	230	+	s.layout = NewLayout(home, repo, "")
	231	+	s.github = setKey(gh, "workspace", expr.String(s.layout.Workspace))
	232	+	return s, s.layout, s.initRest(root, matrixJSON, r)
	233	+	}
	234	+
	235	+	// WithLayout replaces the directory layout (tests and non-default homes). It
	236	+	// must be called before the first step.
	237	+	func (s *State) WithLayout(l Layout) {
	238	+	s.layout = l
	239	+	s.github = setKey(s.github, "workspace", expr.String(l.Workspace))
	240	+	s.runner = setKey(s.runner, "temp", expr.String(l.Temp))
	241	+	s.runner = setKey(s.runner, "tool_cache", expr.String(l.ToolCache))
	242	+	s.runner = setKey(s.runner, "workspace", expr.String(l.RunnerWorkspace))
	243	+	}
	244	+
	245	+	func (s State) initRest(root expr.Dictionary, matrixJSON string, r RunnerInfo) error {
	246	+	for _, k := range serverContexts {
	247	+	if v, ok := root.Get(k); ok {
	248	+	s.other[k] = v
	249	+	}
	250	+	}
	251	+	if matrixJSON != "" {
	252	+	if v, err := expr.FromJSON([]byte(matrixJSON)); err == nil {
	253	+	if d, ok := v.(*expr.Dictionary); ok && d.Len() > 0 {
	254	+	s.other["matrix"] = d
	255	+	}
	256	+	} else {
	257	+	return fmt.Errorf("job matrix: %w", err)
	258	+	}
	259	+	}
	260	+	// A context the job does not have is an empty object, like the server's
	261	+	// ("needs": {}, "inputs": {}, "vars": {}); matrix and strategy are null
	262	+	// when absent (recordings: "matrix": null).
	263	+	for _, k := range []string{"needs", "inputs", "vars"} {
	264	+	if _, ok := s.other[k]; !ok {
	265	+	s.other[k] = expr.NewDictionary()
	266	+	}
	267	+	}
	268	+
	269	+	// runner: os, arch, name, environment, tool_cache, temp (JobRunner.cs:165-176,
	270	+	// TempDirectoryManager.cs:31) and workspace (JobExtension.cs:215).
	271	+	env := r.Environment
	272	+	if env == "" {
	273	+	env = "self-hosted"
	274	+	}
	275	+	rc := expr.NewDictionary()
	276	+	rc.Add("os", expr.String("Linux"))
	277	+	rc.Add("arch", expr.String(r.arch()))
	278	+	rc.Add("name", expr.String(r.Name))
	279	+	rc.Add("environment", expr.String(env))
	280	+	rc.Add("tool_cache", expr.String(s.layout.ToolCache))
	281	+	rc.Add("temp", expr.String(s.layout.Temp))
	282	+	rc.Add("workspace", expr.String(s.layout.RunnerWorkspace))
	283	+	s.runner = rc
	284	+
	285	+	// job: status first (StepsRunner.cs:53), then what the server gave.
	286	+	jc := expr.NewDictionary()
	287	+	jc.Add("status", expr.String("success"))
	288	+	if v, ok := root.Get("job"); ok {
	289	+	if d, ok := v.(*expr.Dictionary); ok {
	290	+	for _, p := range d.Pairs() {
	291	+	if !strings.EqualFold(p.Key, "status") {
	292	+	jc.Add(p.Key, p.Value)
	293	+	}
	294	+	}
	295	+	}
	296	+	}
	297	+	s.job = jc
	298	+	return nil
	299	+	}
	300	+
	301	+	// Layout returns the directory layout.
	302	+	func (s *State) Layout() Layout { return s.layout }
	303	+
	304	+	// SetGitHub sets a key of the github context (SetGitHubContext).
	305	+	func (s *State) SetGitHub(k, v string) { s.github = setKey(s.github, k, expr.String(v)) }
	306	+
	307	+	// GitHub returns a string key of the github context ("" when absent).
	308	+	func (s *State) GitHub(k string) string { return str(s.github, k) }
	309	+
	310	+	// Event returns github.event, or nil.
	311	+	func (s *State) Event() expr.Value { v, _ := s.github.Get("event"); return v }
	312	+
	313	+	// SetEnv sets a variable of the job environment (Global.EnvironmentVariables
	314	+	// and the env context).
	315	+	func (s *State) SetEnv(k, v string) { s.env.Set(k, v) }
	316	+
	317	+	// JobEnv returns a copy of the job environment.
	318	+	func (s State) JobEnv() ordered { return s.env.Clone() }
	319	+
	320	+	// AddPath records a GITHUB_PATH entry: a repeated entry moves to the end
	321	+	// (AddPathFileCommand, FileCommandManager.cs:152).
	322	+	func (s *State) AddPath(p string) {
	323	+	s.prepend = slices.DeleteFunc(s.prepend, func(x string) bool { return x == p })
	324	+	s.prepend = append(s.prepend, p)
	325	+	}
	326	+
	327	+	// Prepend returns the GITHUB_PATH entries in the order added.
	328	+	func (s *State) Prepend() []string { return slices.Clone(s.prepend) }
	329	+
	330	+	// SetStatus sets job.status ("success", "failure" or "cancelled").
	331	+	func (s *State) SetStatus(status string) {
	332	+	s.status = status
	333	+	s.job = setKey(s.job, "status", expr.String(status))
	334	+	}
	335	+
	336	+	// Status returns job.status.
	337	+	func (s *State) Status() string { return s.status }
	338	+
	339	+	// stepIDVisible reports whether a step with this id appears in the steps
	340	+	// context: generated ids start with "__" and do not (ExecutionContext.cs:644).
	341	+	func stepIDVisible(id string) bool { return id != "" && !strings.HasPrefix(id, "__") }
	342	+
	343	+	func (s State) step(id string) stepCtx {
	344	+	c, ok := s.steps[id]
	345	+	if !ok {
	346	+	c = &stepCtx{outputs: newOrdered()}
	347	+	s.steps[id] = c
	348	+	s.stepOrder = append(s.stepOrder, id)
	349	+	}
	350	+	return c
	351	+	}
	352	+
	353	+	// SetOutput sets an output of step id (StepsContext.SetOutput). Steps with a
	354	+	// generated id keep no outputs.
	355	+	func (s *State) SetOutput(id, name, value string) {
	356	+	if !stepIDVisible(id) {
	357	+	return
	358	+	}
	359	+	s.step(id).outputs.Set(name, value)
	360	+	}
	361	+
	362	+	// SetStepResult records outcome and conclusion of a completed step
	363	+	// (ExecutionContext.UpdateGlobalStepsContext).
	364	+	func (s *State) SetStepResult(id, outcome, conclusion string) {
	365	+	if !stepIDVisible(id) {
	366	+	return
	367	+	}
	368	+	c := s.step(id)
	369	+	c.outcome, c.concl = outcome, conclusion
	370	+	}
	371	+
	372	+	// Output returns an output of a step ("" when absent).
	373	+	func (s *State) Output(id, name string) string {
	374	+	if c, ok := s.steps[id]; ok {
	375	+	v, _ := c.outputs.Get(name)
	376	+	return v
	377	+	}
	378	+	return ""
	379	+	}
	380	+
	381	+	func (s State) stepsValue() expr.Dictionary {
	382	+	d := expr.NewDictionary()
	383	+	for _, id := range s.stepOrder {
	384	+	c := s.steps[id]
	385	+	sd := expr.NewDictionary()
	386	+	sd.Add("outputs", c.outputs.Dict())
	387	+	if c.outcome != "" {
	388	+	sd.Add("outcome", expr.String(c.outcome))
	389	+	sd.Add("conclusion", expr.String(c.concl))
	390	+	}
	391	+	d.Add(id, sd)
	392	+	}
	393	+	return d
	394	+	}
	395	+
	396	+	// Values returns the contexts for expression evaluation. stepEnv, when not
	397	+	// nil, is the env context of a step (job environment with the step's own
	398	+	// variables); otherwise the env context is the job environment.
	399	+	func (s State) Values(env ordered) map[string]expr.Value {
	400	+	if env == nil {
	401	+	env = s.env
	402	+	}
	403	+	m := map[string]expr.Value{
	404	+	"github": s.github,
	405	+	"runner": s.runner,
	406	+	"job": s.job,
	407	+	"env": env.Dict(),
	408	+	"steps": s.stepsValue(),
	409	+	"secrets": expr.NewDictionary(),
	410	+	}
	411	+	for k, v := range s.other {
	412	+	m[k] = v
	413	+	}
	414	+	return m
	415	+	}
	416	+
	417	+	// githubEnvAllowlist is GitHubContext._contextEnvAllowlist (GitHubContext.cs:9).
	418	+	var githubEnvAllowlist = map[string]bool{
	419	+	"action_path": true, "action_ref": true, "action_repository": true, "action": true, "actor": true,
	420	+	"actor_id": true, "api_url": true, "artifacts": true, "artifacts_list": true, "base_ref": true,
	421	+	"env": true, "event_name": true, "event_path": true, "graphql_url": true, "head_ref": true,
	422	+	"job": true, "output": true, "path": true, "ref_name": true, "ref_protected": true, "ref_type": true,
	423	+	"ref": true, "repository": true, "repository_id": true, "repository_owner": true,
	424	+	"repository_owner_id": true, "retention_days": true, "run_attempt": true, "run_id": true,
	425	+	"run_number": true, "server_url": true, "sha": true, "state": true, "step_summary": true,
	426	+	"triggering_actor": true, "workflow": true, "workflow_ref": true, "workflow_sha": true, "workspace": true,
	427	+	}
	428	+
	429	+	// RuntimeEnv is the environment that the contexts add to every step: the
	430	+	// allow-listed keys of github as GITHUB_<KEY> (strings and booleans only, as in
	431	+	// GitHubContext.GetRuntimeEnvironmentVariables), and every key of runner as
	432	+	// RUNNER_<KEY> (RunnerContext.cs:11). Numbers and objects are skipped: the
	433	+	// server must send GitHub's string forms ("run_id": "123").
	434	+	func (s State) RuntimeEnv() ordered {
	435	+	out := newOrdered()
	436	+	for _, p := range s.github.Pairs() {
	437	+	if !githubEnvAllowlist[strings.ToLower(p.Key)] {
	438	+	continue
	439	+	}
	440	+	switch v := p.Value.(type) {
	441	+	case expr.String:
	442	+	out.Set("GITHUB_"+strings.ToUpper(p.Key), string(v))
	443	+	case expr.Boolean:
	444	+	out.Set("GITHUB_"+strings.ToUpper(p.Key), map[bool]string{true: "true", false: "false"}[bool(v)])
	445	+	}
	446	+	}
	447	+	for _, p := range s.runner.Pairs() {
	448	+	if v, ok := p.Value.(expr.String); ok {
	449	+	out.Set("RUNNER_"+strings.ToUpper(p.Key), string(v))
	450	+	}
	451	+	}
	452	+	return out
	453	+	}
	454	+
	455	+	// EventJSON returns github.event as JSON for event.json, or "" without event.
	456	+	func (s *State) EventJSON() string {
	457	+	v := s.Event()
	458	+	if v == nil {
	459	+	return ""
	460	+	}
	461	+	if _, isNull := v.(expr.Null); isNull {
	462	+	return ""
	463	+	}
	464	+	return expr.ToJSON(v)
	465	+	}

A internal/executor/contexts_test.go +245 -0

@@ -0,0 +1,245 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package executor
	4	+
	5	+	import (
	6	+	"reflect"
	7	+	"strings"
	8	+	"testing"
	9	+
	10	+	"bnema.dev/ermite-actions/expr"
	11	+	)
	12	+
	13	+	const testContexts = `{
	14	+	"github": {"repository":"octo/hello","repository_owner":"octo","run_id":"42","run_number":"7","run_attempt":"1",
	15	+	"ref":"refs/heads/main","ref_name":"main","ref_type":"branch","sha":"abc","event_name":"push","job":"build",
	16	+	"ref_protected":false,"workflow":"CI","actor":"octo","server_url":"https://ermite.example",
	17	+	"event":{"ref":"refs/heads/main","n":1},"retention_days":"90","unknown_key":"x"},
	18	+	"needs": {"a":{"result":"success","outputs":{"o":"1"}}},
	19	+	"strategy": {"fail-fast":true,"job-index":0,"job-total":1,"max-parallel":1},
	20	+	"matrix": {"os":"linux"},
	21	+	"inputs": {},
	22	+	"vars": {"V":"1"}
	23	+	}`
	24	+
	25	+	func newTestState(t testing.T) State {
	26	+	t.Helper()
	27	+	s, _, err := NewState(testContexts, `{"os":"linux"}`, "build", RunnerInfo{Name: "r1"}, "")
	28	+	if err != nil {
	29	+	t.Fatal(err)
	30	+	}
	31	+	return s
	32	+	}
	33	+
	34	+	func TestNewStateLayoutAndGitHub(t *testing.T) {
	35	+	s := newTestState(t)
	36	+	l := s.Layout()
	37	+	if l.Workspace != "/home/runner/work/hello/hello" \|\| l.RunnerWorkspace != "/home/runner/work/hello" \|\|
	38	+	l.Temp != "/home/runner/work/_temp" \|\| l.ToolCache != "/opt/hostedtoolcache" \|\| l.Home != "/home/runner" {
	39	+	t.Fatalf("%+v", l)
	40	+	}
	41	+	if s.GitHub("workspace") != l.Workspace \|\| s.GitHub("job") != "build" {
	42	+	t.Fatalf("workspace %q job %q", s.GitHub("workspace"), s.GitHub("job"))
	43	+	}
	44	+	// token comes first (ExecutionContext.cs:1023): an empty token, no GITHUB_TOKEN yet.
	45	+	gh := s.Values(nil)["github"].(*expr.Dictionary)
	46	+	if gh.Keys()[0] != "token" {
	47	+	t.Fatalf("keys %v", gh.Keys())
	48	+	}
	49	+	}
	50	+
	51	+	func TestRuntimeEnv(t *testing.T) {
	52	+	s := newTestState(t)
	53	+	env := s.RuntimeEnv()
	54	+	want := map[string]string{
	55	+	"GITHUB_REPOSITORY": "octo/hello", "GITHUB_REPOSITORY_OWNER": "octo", "GITHUB_RUN_ID": "42",
	56	+	"GITHUB_RUN_NUMBER": "7", "GITHUB_RUN_ATTEMPT": "1", "GITHUB_REF": "refs/heads/main",
	57	+	"GITHUB_REF_NAME": "main", "GITHUB_REF_TYPE": "branch", "GITHUB_SHA": "abc", "GITHUB_EVENT_NAME": "push",
	58	+	"GITHUB_JOB": "build", "GITHUB_REF_PROTECTED": "false", "GITHUB_WORKFLOW": "CI", "GITHUB_ACTOR": "octo",
	59	+	"GITHUB_SERVER_URL": "https://ermite.example", "GITHUB_RETENTION_DAYS": "90",
	60	+	"GITHUB_WORKSPACE": "/home/runner/work/hello/hello",
	61	+	"RUNNER_OS": "Linux", "RUNNER_NAME": "r1", "RUNNER_ENVIRONMENT": "self-hosted",
	62	+	"RUNNER_TEMP": "/home/runner/work/_temp", "RUNNER_TOOL_CACHE": "/opt/hostedtoolcache",
	63	+	"RUNNER_WORKSPACE": "/home/runner/work/hello",
	64	+	}
	65	+	for k, v := range want {
	66	+	if got, ok := env.Get(k); !ok \|\| got != v {
	67	+	t.Errorf("%s = %q (%v), want %q", k, got, ok, v)
	68	+	}
	69	+	}
	70	+	// Only allow-listed string/boolean keys; objects are not exported.
	71	+	for _, k := range []string{"GITHUB_EVENT", "GITHUB_UNKNOWN_KEY", "GITHUB_TOKEN"} {
	72	+	if _, ok := env.Get(k); ok {
	73	+	t.Errorf("%s must not be set", k)
	74	+	}
	75	+	}
	76	+	if a, _ := env.Get("RUNNER_ARCH"); a != "X64" && a != "ARM64" {
	77	+	t.Errorf("RUNNER_ARCH = %q", a)
	78	+	}
	79	+	}
	80	+
	81	+	func TestRunnerContextOrderAndValues(t *testing.T) {
	82	+	s := newTestState(t)
	83	+	r := s.Values(nil)["runner"].(*expr.Dictionary)
	84	+	if got := strings.Join(r.Keys(), ","); got != "os,arch,name,environment,tool_cache,temp,workspace" {
	85	+	t.Fatal(got)
	86	+	}
	87	+	s2, _, _ := NewState(testContexts, "", "build", RunnerInfo{Name: "h", Environment: "github-hosted", Arch: "ARM64"}, "")
	88	+	r2 := s2.Values(nil)["runner"].(*expr.Dictionary)
	89	+	if str(r2, "environment") != "github-hosted" \|\| str(r2, "arch") != "ARM64" {
	90	+	t.Fatal(expr.ToJSON(r2))
	91	+	}
	92	+	}
	93	+
	94	+	func TestServerContextsAndDefaults(t *testing.T) {
	95	+	s := newTestState(t)
	96	+	v := s.Values(nil)
	97	+	if expr.ToJSON(v["matrix"]) != "{\n \"os\": \"linux\"\n}" {
	98	+	t.Fatal(expr.ToJSON(v["matrix"]))
	99	+	}
	100	+	for _, k := range []string{"needs", "strategy", "inputs", "vars", "steps", "env", "job", "secrets"} {
	101	+	if v[k] == nil {
	102	+	t.Errorf("context %s missing", k)
	103	+	}
	104	+	}
	105	+	// Absent contexts: needs/inputs/vars are empty objects, matrix is not set
	106	+	// (null in expressions), as the recordings show.
	107	+	s2, _, err := NewState(`{"github":{"repository":"o/r"}}`, "{}", "j", RunnerInfo{}, "")
	108	+	if err != nil {
	109	+	t.Fatal(err)
	110	+	}
	111	+	v2 := s2.Values(nil)
	112	+	if _, ok := v2["matrix"]; ok {
	113	+	t.Error("matrix must be absent without a matrix")
	114	+	}
	115	+	if d, ok := v2["needs"].(*expr.Dictionary); !ok \|\| d.Len() != 0 {
	116	+	t.Errorf("needs %v", v2["needs"])
	117	+	}
	118	+	}
	119	+
	120	+	func TestNewStateErrors(t *testing.T) {
	121	+	for _, tc := range []struct{ ctx, matrix, repo, want string }{
	122	+	{`[]`, "", "", "want a JSON object"},
	123	+	{`{`, "", "", "job contexts"},
	124	+	{`{"github":1}`, "", "", "github must be an object"},
	125	+	{`{"github":{}}`, "", "", "github.repository is missing"},
	126	+	{`{"github":{"repository":"o/r"}}`, "{", "", "job matrix"},
	127	+	} {
	128	+	_, _, err := NewState(tc.ctx, tc.matrix, "j", RunnerInfo{}, tc.repo)
	129	+	if err == nil \|\| !strings.Contains(err.Error(), tc.want) {
	130	+	t.Errorf("%q: %v, want %q", tc.ctx, err, tc.want)
	131	+	}
	132	+	}
	133	+	// The repository name can come from the job when the context lacks it.
	134	+	if _, _, err := NewState(`{"github":{}}`, "", "j", RunnerInfo{}, "hello"); err != nil {
	135	+	t.Fatal(err)
	136	+	}
	137	+	}
	138	+
	139	+	func TestJobStatus(t *testing.T) {
	140	+	s := newTestState(t)
	141	+	job := func() expr.Dictionary { return s.Values(nil)["job"].(expr.Dictionary) }
	142	+	if str(job(), "status") != "success" \|\| job().Keys()[0] != "status" {
	143	+	t.Fatal(expr.ToJSON(job()))
	144	+	}
	145	+	s.SetStatus("failure")
	146	+	if str(job(), "status") != "failure" \|\| s.Status() != "failure" {
	147	+	t.Fatal(expr.ToJSON(job()))
	148	+	}
	149	+	}
	150	+
	151	+	func TestEnvOrderAndOverride(t *testing.T) {
	152	+	o := newOrdered()
	153	+	o.Set("A", "1")
	154	+	o.Set("B", "2")
	155	+	o.Set("A", "3") // keeps its position
	156	+	if !reflect.DeepEqual(o.Keys(), []string{"A", "B"}) \|\| o.vals["A"] != "3" {
	157	+	t.Fatalf("%v", o)
	158	+	}
	159	+	c := o.Clone()
	160	+	c.Set("C", "4")
	161	+	if o.Len() != 2 \|\| c.Len() != 3 {
	162	+	t.Fatal("Clone shares state")
	163	+	}
	164	+	// Case-sensitive keys, like Linux.
	165	+	o.Set("a", "x")
	166	+	if o.Len() != 3 {
	167	+	t.Fatal("keys must be case-sensitive")
	168	+	}
	169	+	}
	170	+
	171	+	func TestStepsContext(t *testing.T) {
	172	+	s := newTestState(t)
	173	+	s.SetOutput("one", "out", "v")
	174	+	s.SetOutput("__run", "ignored", "x") // generated ids are not in the context
	175	+	s.SetStepResult("one", "failure", "success")
	176	+	s.SetStepResult("__run_2", "success", "success")
	177	+	s.SetStepResult("skipped", "skipped", "skipped")
	178	+	steps := s.Values(nil)["steps"].(*expr.Dictionary)
	179	+	if got := strings.Join(steps.Keys(), ","); got != "one,skipped" {
	180	+	t.Fatalf("keys %s", got)
	181	+	}
	182	+	one, _ := steps.Get("one")
	183	+	od := one.(*expr.Dictionary)
	184	+	if str(od, "outcome") != "failure" \|\| str(od, "conclusion") != "success" {
	185	+	t.Fatal(expr.ToJSON(one))
	186	+	}
	187	+	outs, _ := od.Get("outputs")
	188	+	if str(outs.(*expr.Dictionary), "out") != "v" \|\| s.Output("one", "out") != "v" \|\| s.Output("none", "x") != "" {
	189	+	t.Fatal(expr.ToJSON(outs))
	190	+	}
	191	+	// A step that has output but has not completed has no outcome yet.
	192	+	s.SetOutput("running", "k", "v")
	193	+	r, _ := s.Values(nil)["steps"].(*expr.Dictionary).Get("running")
	194	+	if _, ok := r.(*expr.Dictionary).Get("outcome"); ok {
	195	+	t.Fatal("outcome before completion")
	196	+	}
	197	+	}
	198	+
	199	+	func TestAddPathMovesRepeatedEntryToTheEnd(t *testing.T) {
	200	+	s := newTestState(t)
	201	+	for _, p := range []string{"/a", "/b", "/a"} {
	202	+	s.AddPath(p)
	203	+	}
	204	+	if !reflect.DeepEqual(s.Prepend(), []string{"/b", "/a"}) {
	205	+	t.Fatal(s.Prepend())
	206	+	}
	207	+	}
	208	+
	209	+	func TestEnvContextForSteps(t *testing.T) {
	210	+	s := newTestState(t)
	211	+	s.SetEnv("JOB", "j")
	212	+	step := s.JobEnv()
	213	+	step.Set("STEP", "s")
	214	+	env := s.Values(step)["env"].(*expr.Dictionary)
	215	+	if str(env, "JOB") != "j" \|\| str(env, "STEP") != "s" {
	216	+	t.Fatal(expr.ToJSON(env))
	217	+	}
	218	+	// The job environment is untouched by the step's copy.
	219	+	if _, ok := s.JobEnv().Get("STEP"); ok {
	220	+	t.Fatal("step env leaked into the job env")
	221	+	}
	222	+	}
	223	+
	224	+	func TestEventJSON(t *testing.T) {
	225	+	s := newTestState(t)
	226	+	if got := s.EventJSON(); !strings.Contains(got, `"ref": "refs/heads/main"`) {
	227	+	t.Fatal(got)
	228	+	}
	229	+	s2, _, _ := NewState(`{"github":{"repository":"o/r"}}`, "", "j", RunnerInfo{}, "")
	230	+	if s2.EventJSON() != "" {
	231	+	t.Fatal("no event, no event.json")
	232	+	}
	233	+	}
	234	+
	235	+	func TestWithLayout(t *testing.T) {
	236	+	s := newTestState(t)
	237	+	l := NewLayout("/tmp/h", "hello", "/tmp/tc")
	238	+	s.WithLayout(l)
	239	+	env := s.RuntimeEnv()
	240	+	for k, v := range map[string]string{"GITHUB_WORKSPACE": "/tmp/h/work/hello/hello", "RUNNER_TEMP": "/tmp/h/work/_temp", "RUNNER_TOOL_CACHE": "/tmp/tc"} {
	241	+	if got, _ := env.Get(k); got != v {
	242	+	t.Errorf("%s = %q", k, got)
	243	+	}
	244	+	}
	245	+	}

A internal/executor/corpus_test.go +327 -0

@@ -0,0 +1,327 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package executor
	4	+
	5	+	import (
	6	+	"encoding/json"
	7	+	"fmt"
	8	+	"os"
	9	+	"os/exec"
	10	+	"path/filepath"
	11	+	"regexp"
	12	+	"strings"
	13	+	"sync"
	14	+	"testing"
	15	+	"time"
	16	+	)
	17	+
	18	+	// These tests run jobs of the ermite-conformance corpus (the CI-1 scenarios)
	19	+	// through the executor on this machine and compare the steps, their
	20	+	// conclusions and their logs with the recordings made on github.com. They need
	21	+	// a checkout of ermite-conformance (the corpus and recordings directories),
	22	+	// found through ERMITE_CONFORMANCE_DIR or as a sibling of this repository;
	23	+	// they skip without it.
	24	+
	25	+	func conformanceDir(t *testing.T) string {
	26	+	t.Helper()
	27	+	cands := []string{os.Getenv("ERMITE_CONFORMANCE_DIR")}
	28	+	wd, _ := os.Getwd()
	29	+	for d := wd; d != "/" && d != "."; d = filepath.Dir(d) {
	30	+	cands = append(cands,
	31	+	filepath.Join(d, "ermite-conformance", ".worktrees", "corpus"),
	32	+	filepath.Join(d, "ermite-conformance"))
	33	+	}
	34	+	for _, c := range cands {
	35	+	if c == "" {
	36	+	continue
	37	+	}
	38	+	if st, err := os.Stat(filepath.Join(c, "corpus")); err == nil && st.IsDir() {
	39	+	return c
	40	+	}
	41	+	}
	42	+	t.Skip("ermite-conformance checkout not found (set ERMITE_CONFORMANCE_DIR)")
	43	+	return ""
	44	+	}
	45	+
	46	+	func needTools(t *testing.T, names ...string) {
	47	+	t.Helper()
	48	+	for _, n := range names {
	49	+	if _, err := exec.LookPath(n); err != nil {
	50	+	t.Skipf("%s not installed", n)
	51	+	}
	52	+	}
	53	+	}
	54	+
	55	+	// recorder is a Reporter that keeps everything.
	56	+	type recorder struct {
	57	+	mu sync.Mutex
	58	+	steps []*recStep
	59	+	byNum map[int]*recStep
	60	+	result Result
	61	+	done bool
	62	+	lastLogTime time.Time
	63	+	}
	64	+
	65	+	type recStep struct {
	66	+	Num int
	67	+	Name string
	68	+	Conclusion Conclusion
	69	+	Log []string
	70	+	Started time.Time
	71	+	Completed time.Time
	72	+	}
	73	+
	74	+	func newRecorder() recorder { return &recorder{byNum: map[int]recStep{}} }
	75	+
	76	+	func (r *recorder) StepStarted(n int, name string, at time.Time) {
	77	+	r.mu.Lock()
	78	+	defer r.mu.Unlock()
	79	+	s := &recStep{Num: n, Name: name, Started: at}
	80	+	r.steps = append(r.steps, s)
	81	+	r.byNum[n] = s
	82	+	}
	83	+	func (r *recorder) Log(n int, at time.Time, line string) {
	84	+	r.mu.Lock()
	85	+	defer r.mu.Unlock()
	86	+	if s := r.byNum[n]; s != nil {
	87	+	s.Log = append(s.Log, line)
	88	+	}
	89	+	r.lastLogTime = at
	90	+	}
	91	+	func (r *recorder) StepCompleted(n int, name string, c Conclusion, at time.Time) {
	92	+	r.mu.Lock()
	93	+	defer r.mu.Unlock()
	94	+	s := r.byNum[n]
	95	+	s.Conclusion, s.Completed = c, at
	96	+	if name != "" {
	97	+	s.Name = name
	98	+	}
	99	+	}
	100	+	func (r *recorder) JobCompleted(res Result) {
	101	+	r.mu.Lock()
	102	+	defer r.mu.Unlock()
	103	+	r.result, r.done = res, true
	104	+	}
	105	+
	106	+	func (r recorder) step(t testing.T, name string) *recStep {
	107	+	t.Helper()
	108	+	for _, s := range r.steps {
	109	+	if s.Name == name {
	110	+	return s
	111	+	}
	112	+	}
	113	+	t.Fatalf("no step %q in %v", name, r.names())
	114	+	return nil
	115	+	}
	116	+
	117	+	func (r *recorder) names() []string {
	118	+	var out []string
	119	+	for _, s := range r.steps {
	120	+	out = append(out, s.Name)
	121	+	}
	122	+	return out
	123	+	}
	124	+
	125	+	// testGitHubContext is a push on a branch, with the keys the corpus reads.
	126	+	func testGitHubContext(job string) string {
	127	+	return fmt.Sprintf(`{"github":{"repository":"octo/hello","repository_owner":"octo","run_id":"1","run_number":"1","run_attempt":"1",
	128	+	"ref":"refs/heads/scn/x/r1","ref_name":"scn/x/r1","ref_type":"branch","ref_protected":false,"sha":"0123456789abcdef0123456789abcdef01234567",
	129	+	"event_name":"push","event":{"ref":"refs/heads/scn/x/r1"},"job":%q,"workflow":"w","actor":"octo","triggering_actor":"octo",
	130	+	"server_url":"https://ermite.example","retention_days":"90"}}`, job)
	131	+	}
	132	+
	133	+	// runLocal runs one job of a workflow file on this machine under a temporary
	134	+	// home. The environment is minimal and fixed.
	135	+	func runLocal(t testing.T, yaml, jobKey string, tweak func(Config)) (*recorder, Result) {
	136	+	t.Helper()
	137	+	home := t.TempDir()
	138	+	layout := NewLayout(home, "hello", filepath.Join(home, "toolcache"))
	139	+	rec := newRecorder()
	140	+	cfg := Config{
	141	+	WorkflowPath: ".github/workflows/ci.yml", WorkflowYAML: []byte(yaml), JobKey: jobKey,
	142	+	ContextsJSON: testGitHubContext(jobKey), Runner: RunnerInfo{Name: "test-runner", Environment: "github-hosted"},
	143	+	RunnerVersion: "test", Layout: &layout, Reporter: rec,
	144	+	BaseEnv: []string{"PATH=" + os.Getenv("PATH"), "HOME=" + home, "LANG=C.UTF-8"},
	145	+	TermAfter: 200 * time.Millisecond, KillAfter: 200 * time.Millisecond, WaitDelay: 500 * time.Millisecond,
	146	+	}
	147	+	if tweak != nil {
	148	+	tweak(&cfg)
	149	+	}
	150	+	res := Run(nil, cfg)
	151	+	return rec, res
	152	+	}
	153	+
	154	+	func readFile(t *testing.T, p string) string {
	155	+	t.Helper()
	156	+	b, err := os.ReadFile(p)
	157	+	if err != nil {
	158	+	t.Fatal(err)
	159	+	}
	160	+	return string(b)
	161	+	}
	162	+
	163	+	type recording struct {
	164	+	Pushes []struct {
	165	+	Runs []struct {
	166	+	Conclusion string `json:"conclusion"`
	167	+	Jobs []struct {
	168	+	Name string `json:"name"`
	169	+	Conclusion string `json:"conclusion"`
	170	+	Steps []struct {
	171	+	Name string `json:"name"`
	172	+	Conclusion string `json:"conclusion"`
	173	+	Log []string `json:"log"`
	174	+	} `json:"steps"`
	175	+	} `json:"jobs"`
	176	+	} `json:"runs"`
	177	+	} `json:"pushes"`
	178	+	}
	179	+
	180	+	var (
	181	+	shellPathRe = regexp.MustCompile(`^shell: \S*/(\w+) `)
	182	+	startPathRe = regexp.MustCompile(`process '\S*/(\w+)'`)
	183	+	uuidRe = regexp.MustCompile(`[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}`)
	184	+	)
	185	+
	186	+	// normalizeLine maps the machine-dependent parts of a line to the form of the
	187	+	// recordings, like the normalisation rules of ermite-conformance: the
	188	+	// directory of the shell binary, the home directory, the repository name and
	189	+	// the random names of script files.
	190	+	func normalizeLine(l, home string) string {
	191	+	l = shellPathRe.ReplaceAllString(l, "shell: /usr/bin/$1 ")
	192	+	l = startPathRe.ReplaceAllString(l, "process '/usr/bin/$1'")
	193	+	l = strings.ReplaceAll(l, home, "/home/runner")
	194	+	l = strings.ReplaceAll(l, "/work/hello/hello", "/work/<repo>/<repo>")
	195	+	return uuidRe.ReplaceAllString(l, "<uuid>")
	196	+	}
	197	+
	198	+	// shIsBash reports whether sh on this machine is bash (Arch, Fedora): the
	199	+	// image of ermite has dash, so a step that checks BASH_VERSION under sh would
	200	+	// differ here for a reason that has nothing to do with the executor.
	201	+	func shIsBash() bool {
	202	+	out, err := exec.Command("sh", "-c", `echo "${BASH_VERSION-unset}"`).Output()
	203	+	return err == nil && strings.TrimSpace(string(out)) != "unset"
	204	+	}
	205	+
	206	+	// compareWithRecording runs every job without needs of a scenario and checks
	207	+	// that the steps, conclusions and logs equal the recording of github.com. Set
	208	+	// up job (runner and image details) and the dump step the harness appends are
	209	+	// not compared.
	210	+	func compareWithRecording(t *testing.T, scenario string, skipJobs ...string) {
	211	+	t.Helper()
	212	+	needTools(t, "bash", "sh")
	213	+	dir := conformanceDir(t)
	214	+	files, _ := filepath.Glob(filepath.Join(dir, "corpus", scenario, "tree", ".github", "workflows", "*.yml"))
	215	+	if len(files) != 1 {
	216	+	t.Fatalf("scenario %s: workflow files %v", scenario, files)
	217	+	}
	218	+	yaml := readFile(t, files[0])
	219	+	var rec recording
	220	+	if err := json.Unmarshal([]byte(readFile(t, filepath.Join(dir, "recordings", scenario+".json"))), &rec); err != nil {
	221	+	t.Fatal(err)
	222	+	}
	223	+	if len(rec.Pushes) == 0 \|\| len(rec.Pushes[0].Runs) == 0 {
	224	+	t.Skip("scenario not recorded")
	225	+	}
	226	+	skip := map[string]bool{}
	227	+	for _, s := range skipJobs {
	228	+	skip[s] = true
	229	+	}
	230	+	n := 0
	231	+	for _, jr := range rec.Pushes[0].Runs[0].Jobs {
	232	+	if skip[jr.Name] {
	233	+	continue
	234	+	}
	235	+	jr := jr
	236	+	t.Run(jr.Name, func(t *testing.T) {
	237	+	var home string
	238	+	got, res := runLocal(t, yaml, jr.Name, func(c *Config) { home = c.Layout.Home })
	239	+	// Expected steps: the recording without Set up job and the dump step.
	240	+	type es struct {
	241	+	Name, Conclusion string
	242	+	Log []string
	243	+	}
	244	+	var want []es
	245	+	for _, s := range jr.Steps {
	246	+	if s.Name == SetUpJobName \|\| s.Name == "ermite-dump" {
	247	+	continue
	248	+	}
	249	+	want = append(want, es{s.Name, s.Conclusion, s.Log})
	250	+	}
	251	+	var have []es
	252	+	for _, s := range got.steps {
	253	+	if s.Name == SetUpJobName {
	254	+	continue
	255	+	}
	256	+	var log []string
	257	+	for _, l := range s.Log {
	258	+	log = append(log, normalizeLine(l, home))
	259	+	}
	260	+	have = append(have, es{s.Name, string(s.Conclusion), log})
	261	+	}
	262	+	if len(have) != len(want) {
	263	+	t.Fatalf("steps: got %v, want %d steps", got.names(), len(want))
	264	+	}
	265	+	for i := range want {
	266	+	w, h := want[i], have[i]
	267	+	if h.Name != w.Name \|\| h.Conclusion != w.Conclusion {
	268	+	t.Errorf("step %d: got %q (%s), want %q (%s)", i+1, h.Name, h.Conclusion, w.Name, w.Conclusion)
	269	+	continue
	270	+	}
	271	+	if shIsBash() && strings.Contains(strings.Join(w.Log, "\n"), "shell: /usr/bin/sh ") {
	272	+	continue // sh is bash here, dash on the image: the output is not comparable
	273	+	}
	274	+	if strings.Join(h.Log, "\n") != strings.Join(w.Log, "\n") {
	275	+	t.Errorf("step %q log differs:\n--- got\n%s\n--- want\n%s", w.Name, strings.Join(h.Log, "\n"), strings.Join(w.Log, "\n"))
	276	+	}
	277	+	}
	278	+	if string(res.Conclusion) != jr.Conclusion && jr.Conclusion != "" {
	279	+	// The dump step fails on GitHub when an earlier step failed in
	280	+	// some scenarios; the job conclusion is still the one of the steps.
	281	+	t.Errorf("job conclusion %s, recording %s", res.Conclusion, jr.Conclusion)
	282	+	}
	283	+	})
	284	+	n++
	285	+	}
	286	+	if n == 0 {
	287	+	t.Fatal("no job compared")
	288	+	}
	289	+	}
	290	+
	291	+	// Scenarios whose jobs have no dependencies and whose logs do not depend on
	292	+	// the machine, in the order of the plan's list (shells, env, outputs, file
	293	+	// commands, conditions).
	294	+	func TestCorpusEnvPrecedence(t *testing.T) { compareWithRecording(t, "env-precedence") }
	295	+	func TestCorpusEnvExpressions(t *testing.T) { compareWithRecording(t, "env-expressions") }
	296	+	func TestCorpusEnvSpecialValues(t *testing.T) { compareWithRecording(t, "env-special-values") }
	297	+	func TestCorpusOutGithubEnv(t *testing.T) { compareWithRecording(t, "out-github-env") }
	298	+	func TestCorpusOutGithubEnvReserved(t *testing.T) { compareWithRecording(t, "out-github-env-reserved") }
	299	+	func TestCorpusOutGithubPath(t *testing.T) { compareWithRecording(t, "out-github-path") }
	300	+	func TestCorpusOutStepOutputs(t *testing.T) { compareWithRecording(t, "out-step-outputs") }
	301	+	func TestCorpusCtxSteps(t *testing.T) { compareWithRecording(t, "ctx-steps") }
	302	+	func TestCorpusIfStepConditions(t *testing.T) { compareWithRecording(t, "if-step-conditions") }
	303	+	func TestCorpusShellWorkingDirectory(t *testing.T) {
	304	+	compareWithRecording(t, "shell-working-directory")
	305	+	}
	306	+	func TestCorpusShellExitCodes(t *testing.T) { compareWithRecording(t, "shell-exit-codes") }
	307	+	func TestCorpusIfJobLevel(t *testing.T) {
	308	+	compareWithRecording(t, "if-job-level-fail-effects", "after-job-continue")
	309	+	}
	310	+
	311	+	func envPath() string { return os.Getenv("PATH") }
	312	+
	313	+	func TestCorpusShellBuiltin(t *testing.T) { compareWithRecording(t, "shell-builtin") }
	314	+	func TestCorpusShellCustom(t *testing.T) { compareWithRecording(t, "shell-custom") }
	315	+	func TestCorpusStatusFunctions(t *testing.T) { compareWithRecording(t, "expr-status-functions") }
	316	+
	317	+	// GitHub rejects the workflow of expr-status-in-env (status functions are not
	318	+	// allowed in a step's env): the recorded run has no jobs. The executor must
	319	+	// refuse it at "Set up job" too.
	320	+	func TestCorpusStatusInEnvIsRejected(t *testing.T) {
	321	+	dir := conformanceDir(t)
	322	+	yaml := readFile(t, filepath.Join(dir, "corpus", "expr-status-in-env", "tree", ".github", "workflows", "ci.yml"))
	323	+	rec, res := runLocal(t, yaml, "main", nil)
	324	+	if res.Conclusion != ConclusionFailure \|\| !strings.Contains(res.Error, "The workflow is not valid") \|\| len(rec.steps) != 1 {
	325	+	t.Fatalf("%+v %v", res, rec.names())
	326	+	}
	327	+	}

A internal/executor/exec_step.go +248 -0

@@ -0,0 +1,248 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package executor
	4	+
	5	+	import (
	6	+	"fmt"
	7	+	"path/filepath"
	8	+	"strings"
	9	+
	10	+	"bnema.dev/ermite-actions/workflow"
	11	+	)
	12	+
	13	+	// execute is ActionRunner.RunAsync plus ScriptHandler for one run step.
	14	+	func (j job) execute(s stepRun, stepCancel <-chan struct{}) stepResult {
	15	+	num, st := s.num, s.st
	16	+	if msg := notSupported(st); msg != "" {
	17	+	if st.Uses != nil {
	18	+	j.out(num, "##[group]Run "+st.Uses.Value)
	19	+	j.out(num, "##[endgroup]")
	20	+	}
	21	+	j.errorf(num, "%s", msg)
	22	+	return resFailed
	23	+	}
	24	+	fail := func(err error) stepResult { j.errorf(num, "%s", err.Error()); return resFailed }
	25	+
	26	+	if err := j.prepareFileCommands(st); err != nil {
	27	+	return fail(err)
	28	+	}
	29	+	// ProcessFiles runs after the handler, even when it failed (a finally block).
	30	+	defer j.fc.Process(s)
	31	+
	32	+	in, err := j.scriptInputs(s)
	33	+	if err != nil {
	34	+	return fail(err)
	35	+	}
	36	+	plan, err := j.printHeader(s, in)
	37	+	if err != nil {
	38	+	return fail(err) // the group stays open, like the original
	39	+	}
	40	+	dir, argv, err := j.scriptCommand(plan, in)
	41	+	if err != nil {
	42	+	return fail(err)
	43	+	}
	44	+	return j.startProcess(s, plan, dir, argv, stepCancel)
	45	+	}
	46	+
	47	+	// scriptIn is what a run step evaluates before it prints its header.
	48	+	type scriptIn struct {
	49	+	script, shell, workingDirectory string
	50	+	hasWorkingDirectory bool
	51	+	}
	52	+
	53	+	// scriptInputs evaluates the run, shell and working-directory keys of the step
	54	+	// (ActionRunner.cs:174-185). The shell and the directory fall back to
	55	+	// defaults.run (ScriptHandler.cs:155-180).
	56	+	func (j job) scriptInputs(s stepRun) (scriptIn, error) {
	57	+	st := s.st
	58	+	var in scriptIn
	59	+	var err error
	60	+	if in.script, err = j.evalString(st.Run, s.env); err != nil {
	61	+	return in, err
	62	+	}
	63	+	if in.shell, err = j.evalString(st.Shell, s.env); err != nil {
	64	+	return in, err
	65	+	}
	66	+	if in.shell == "" {
	67	+	in.shell = j.runDefaults.shell
	68	+	}
	69	+	if st.WorkingDirectory != nil {
	70	+	in.hasWorkingDirectory = true
	71	+	if in.workingDirectory, err = j.evalString(st.WorkingDirectory, s.env); err != nil {
	72	+	return in, err
	73	+	}
	74	+	}
	75	+	if !in.hasWorkingDirectory {
	76	+	in.workingDirectory = j.runDefaults.workingDirectory
	77	+	}
	78	+	return in, nil
	79	+	}
	80	+
	81	+	// prepareFileCommands creates the files of the step, puts their paths in the
	82	+	// github context (FileCommandManager.InitializeFiles) and writes event.json
	83	+	// (ExecutionContext.WriteWebhookPayload).
	84	+	func (j job) prepareFileCommands(st workflow.Step) error {
	85	+	j.mu.Lock()
	86	+	defer j.mu.Unlock()
	87	+	files, err := j.fc.Init()
	88	+	if err != nil {
	89	+	return err
	90	+	}
	91	+	for _, c := range fileCommands {
	92	+	j.state.SetGitHub(c.ContextName, files[envVarName(c.ContextName)])
	93	+	}
	94	+	j.state.SetGitHub("action", st.ID)
	95	+	j.state.SetGitHub("action_repository", "")
	96	+	j.state.SetGitHub("action_ref", "")
	97	+	if ev := j.state.EventJSON(); ev != "" {
	98	+	if p, err := j.writeEvent(ev); err == nil {
	99	+	j.state.SetGitHub("event_path", p)
	100	+	}
	101	+	}
	102	+	return nil
	103	+	}
	104	+
	105	+	// printHeader prints the group of the step as ScriptHandler.PrintActionDetails
	106	+	// does (ScriptHandler.cs:27-137): the first line of the script, the script in
	107	+	// bright cyan, the shell, the env block of the step, and the end of the group.
	108	+	func (j job) printHeader(s stepRun, in scriptIn) (ShellPlan, error) {
	109	+	num := s.num
	110	+	j.out(num, "##[group]Run "+firstLine(in.script))
	111	+	for _, l := range strings.Split(strings.TrimRight(strings.ReplaceAll(in.script, "\r\n", "\n"), "\n"), "\n") {
	112	+	j.write(num, "\x1b[36;1m"+l+"\x1b[0m")
	113	+	}
	114	+	j.mu.Lock()
	115	+	lookup := Lookup{Path: j.lookupPath, Prepend: j.state.Prepend()}
	116	+	j.mu.Unlock()
	117	+	plan, err := PlanShell(in.shell, lookup)
	118	+	if err != nil {
	119	+	return plan, err
	120	+	}
	121	+	j.out(num, "shell: "+plan.ShellLine())
	122	+	if s.env.Len() > 0 {
	123	+	j.write(num, "env:")
	124	+	for _, k := range s.env.Keys() {
	125	+	v, _ := s.env.Get(k)
	126	+	j.out(num, fmt.Sprintf(" %s: %s", k, v))
	127	+	}
	128	+	}
	129	+	j.write(num, "##[endgroup]")
	130	+	return plan, nil
	131	+	}
	132	+
	133	+	// scriptCommand writes the script file and returns the working directory and
	134	+	// the command line (ScriptHandler.RunAsync, :140-285).
	135	+	func (j *job) scriptCommand(plan ShellPlan, in scriptIn) (dir string, argv []string, err error) {
	136	+	j.mu.Lock()
	137	+	workspace := j.state.GitHub("workspace")
	138	+	j.mu.Unlock()
	139	+	sc, err := WriteScript(j.tempRoot, plan, in.script)
	140	+	if err != nil {
	141	+	return "", nil, err
	142	+	}
	143	+	return ResolveWorkingDirectory(workspace, in.workingDirectory), sc.Argv, nil
	144	+	}
	145	+
	146	+	// processEnv builds the environment of the process: the inherited one, the
	147	+	// step's environment, PATH with GITHUB_PATH in front, then the runtime variables
	148	+	// of the contexts, GITHUB_ACTIONS, CI and the tracking id.
	149	+	func (j job) processEnv(stepEnv ordered) []string {
	150	+	j.mu.Lock()
	151	+	runtime := j.state.RuntimeEnv()
	152	+	prepend := j.state.Prepend()
	153	+	j.mu.Unlock()
	154	+	penv := newOrdered()
	155	+	for _, kv := range j.baseEnv {
	156	+	if k, v, ok := strings.Cut(kv, "="); ok {
	157	+	penv.Set(k, v)
	158	+	}
	159	+	}
	160	+	for _, k := range stepEnv.Keys() {
	161	+	v, _ := stepEnv.Get(k)
	162	+	penv.Set(k, v)
	163	+	}
	164	+	if len(prepend) > 0 {
	165	+	orig, ok := stepEnv.Get("PATH")
	166	+	if !ok {
	167	+	orig = j.lookupPath
	168	+	}
	169	+	penv.Set("PATH", PrependedPath(prepend, orig))
	170	+	}
	171	+	for _, k := range runtime.Keys() {
	172	+	v, _ := runtime.Get(k)
	173	+	penv.Set(k, v)
	174	+	}
	175	+	penv.Set("GITHUB_ACTIONS", "true") // ProcessInvoker.cs:283
	176	+	if _, ok := penv.Get("CI"); !ok {
	177	+	penv.Set("CI", "true")
	178	+	}
	179	+	penv.Set(ProcessTrackingID, j.trackID)
	180	+	var list []string
	181	+	for _, k := range penv.Keys() {
	182	+	v, _ := penv.Get(k)
	183	+	list = append(list, k+"="+v)
	184	+	}
	185	+	return list
	186	+	}
	187	+
	188	+	// startProcess runs the script and turns how it ended into the result of the
	189	+	// step.
	190	+	func (j job) startProcess(s stepRun, plan ShellPlan, dir string, argv []string, stepCancel <-chan struct{}) stepResult {
	191	+	num := s.num
	192	+	res, err := runProcess(procSpec{
	193	+	Argv: argv, Dir: dir, Env: j.processEnv(s.env), Credential: j.cfg.Credential,
	194	+	OnLine: func(line string) {
	195	+	if strings.Contains(line, "##[start-action") \|\| strings.Contains(line, "##[end-action") {
	196	+	line = strings.ReplaceAll(strings.ReplaceAll(line, "##[start-action", `##[\start-action`), "##[end-action", `##[\end-action`)
	197	+	}
	198	+	if (strings.Contains(line, "##[") \|\| strings.Contains(line, "::")) && s.cmds.TryProcess(line) {
	199	+	return
	200	+	}
	201	+	j.write(num, line)
	202	+	},
	203	+	Cancel: stepCancel, TermAfter: j.cfg.TermAfter, KillAfter: j.cfg.KillAfter,
	204	+	WaitDelay: j.cfg.WaitDelay, Signal: j.cfg.Signal,
	205	+	})
	206	+	if err != nil {
	207	+	j.errorf(num, "%s", startError(plan.CommandPath, dir, err))
	208	+	return resFailed
	209	+	}
	210	+	if res.Cancelled {
	211	+	// The process was stopped on cancellation: ExecuteAsync throws
	212	+	// (ProcessInvoker.cs:369) and StepsRunner logs the exception
	213	+	// (StepsRunner.cs:333-337).
	214	+	j.errorf(num, "The operation was canceled.")
	215	+	return resCanceled
	216	+	}
	217	+	if res.ExitCode != 0 {
	218	+	j.errorf(num, "Process completed with exit code %d.", res.ExitCode)
	219	+	return resFailed
	220	+	}
	221	+	return resSucceeded
	222	+	}
	223	+
	224	+	// writeEvent writes the webhook payload to RUNNER_TEMP/_github_workflow/event.json
	225	+	// (ExecutionContext.WriteWebhookPayload) and returns its path. The directory
	226	+	// is the user's: the file is replaced, never followed (see FileCommands).
	227	+	func (j *job) writeEvent(payload string) (string, error) {
	228	+	const dir, name = "_github_workflow", "event.json"
	229	+	if err := j.tempRoot.MkdirAll(dir, 0o755); err != nil {
	230	+	return "", err
	231	+	}
	232	+	if c := j.cfg.Credential; c != nil {
	233	+	_ = j.tempRoot.Lchown(dir, int(c.Uid), int(c.Gid))
	234	+	}
	235	+	sub, err := j.tempRoot.OpenRoot(dir)
	236	+	if err != nil {
	237	+	return "", err
	238	+	}
	239	+	defer func() { _ = sub.Close() }()
	240	+	var owner *[2]int
	241	+	if c := j.cfg.Credential; c != nil {
	242	+	owner = &[2]int{int(c.Uid), int(c.Gid)}
	243	+	}
	244	+	if err := createExcl(sub, name, []byte(payload), 0o644, owner); err != nil {
	245	+	return "", err
	246	+	}
	247	+	return filepath.Join(j.state.Layout().Temp, dir, name), nil
	248	+	}

A internal/executor/filecmd.go +453 -0

@@ -0,0 +1,453 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	// Package executor runs the steps of a GitHub Actions job inside a sandbox.
	4	+	// It is a port of the parts of the official runner (actions/runner, C#, MIT)
	5	+	// that CI-1 needs: file commands, shells, contexts and the steps runner. The
	6	+	// package does no sandboxing and no networking: it runs local processes and
	7	+	// reports to a Reporter.
	8	+	package executor
	9	+
	10	+	import (
	11	+	"bytes"
	12	+	"errors"
	13	+	"fmt"
	14	+	"io"
	15	+	"os"
	16	+	"path/filepath"
	17	+	"slices"
	18	+	"strings"
	19	+	"syscall"
	20	+	"unicode/utf8"
	21	+
	22	+	"github.com/google/uuid"
	23	+	)
	24	+
	25	+	// Names of the file commands (the GitHub context names) and their
	26	+	// environment variables.
	27	+	const (
	28	+	cmdPath = "path"
	29	+	cmdEnv = "env"
	30	+	cmdStepSummary = "step_summary"
	31	+	cmdState = "state"
	32	+	cmdOutput = "output"
	33	+	)
	34	+
	35	+	// fileCommand describes one file command like the IFileCommandExtension of
	36	+	// FileCommandManager.cs.
	37	+	type fileCommand struct {
	38	+	ContextName string // github context name; the variable is GITHUB_<UPPER>
	39	+	FilePrefix string
	40	+	}
	41	+
	42	+	// fileCommands is in the order of the official runner's extension list, which
	43	+	// is the order ProcessFiles runs them.
	44	+	var fileCommands = []fileCommand{
	45	+	{cmdPath, "add_path_"}, // AddPathFileCommand (FileCommandManager.cs:127)
	46	+	{cmdEnv, "set_env_"}, // SetEnvFileCommand (:160)
	47	+	{cmdStepSummary, "step_summary_"}, // CreateStepSummaryCommand (:221)
	48	+	{cmdState, "save_state_"}, // SaveStateFileCommand (:295)
	49	+	{cmdOutput, "set_output_"}, // SetOutputFileCommand (:328)
	50	+	}
	51	+
	52	+	// EnvVarName returns the environment variable that holds the file of a command:
	53	+	// GITHUB_ENV, GITHUB_OUTPUT, GITHUB_PATH, GITHUB_STATE, GITHUB_STEP_SUMMARY
	54	+	// (GitHubContext.GetRuntimeEnvironmentVariables, GitHubContext.cs:52).
	55	+	func envVarName(contextName string) string { return "GITHUB_" + strings.ToUpper(contextName) }
	56	+
	57	+	// stepSummaryLimit is CreateStepSummaryCommand.AttachmentSizeLimit
	58	+	// (FileCommandManager.cs:223).
	59	+	const stepSummaryLimit = 1024 * 1024
	60	+
	61	+	// setEnvBlockList is SetEnvFileCommand._setEnvBlockList (FileCommandManager.cs:215).
	62	+	var setEnvBlockList = []string{"NODE_OPTIONS"}
	63	+
	64	+	// msgError is an error whose text is GitHub's own message, capitalised and
	65	+	// punctuated like the official runner's.
	66	+	type msgError string
	67	+
	68	+	func (e msgError) Error() string { return string(e) }
	69	+
	70	+	func ghErrorf(format string, a ...any) error { return msgError(fmt.Sprintf(format, a...)) }
	71	+
	72	+	// Pair is one name and value read from an environment-style file.
	73	+	type Pair struct{ Key, Value string }
	74	+
	75	+	// Sink receives the effects of the file commands of a step.
	76	+	type Sink interface {
	77	+	// SetEnv sets an environment variable of the job (GITHUB_ENV).
	78	+	SetEnv(name, value string)
	79	+	// SetOutput sets an output of the step (GITHUB_OUTPUT).
	80	+	SetOutput(name, value string)
	81	+	// AddPath prepends a directory to PATH for the next steps (GITHUB_PATH). The
	82	+	// caller keeps the list like Global.PrependPath: the last added is first
	83	+	// in PATH, and a repeated entry moves to the end of the list.
	84	+	AddPath(path string)
	85	+	// SaveState keeps a value for the step's own post phase (GITHUB_STATE).
	86	+	SaveState(name, value string)
	87	+	// Error writes an error issue to the log (##[error]).
	88	+	Error(msg string)
	89	+	// Debug writes a ##[debug] line when step debugging is on.
	90	+	Debug(msg string)
	91	+	// CommandFailed marks the step as failed (context.CommandResult = Failed).
	92	+	CommandFailed()
	93	+	}
	94	+
	95	+	// FileCommands manages the per-step files, like FileCommandManager.
	96	+	//
	97	+	// The directory belongs to the user of the steps, and the executor reads the
	98	+	// files as root after each step, so every access goes through an os.Root opened
	99	+	// before the first step ran: replacing the directory or a file by a symbolic
	100	+	// link cannot make the executor read or write anything outside it. Files are
	101	+	// opened with O_NOFOLLOW and must be regular files.
	102	+	type FileCommands struct {
	103	+	dir string
	104	+	root *os.Root
	105	+	suffix string
	106	+	// owner, when set, is the user the files are given to.
	107	+	owner *[2]int
	108	+	}
	109	+
	110	+	// NewFileCommands uses dir (RUNNER_TEMP/_runner_file_commands), creating it.
	111	+	// Call Close when the job is over.
	112	+	func NewFileCommands(dir string) (*FileCommands, error) {
	113	+	if err := os.MkdirAll(dir, 0o755); err != nil {
	114	+	return nil, err
	115	+	}
	116	+	root, err := os.OpenRoot(dir)
	117	+	if err != nil {
	118	+	return nil, err
	119	+	}
	120	+	return &FileCommands{dir: dir, root: root}, nil
	121	+	}
	122	+
	123	+	// Close releases the directory.
	124	+	func (f *FileCommands) Close() error { return f.root.Close() }
	125	+
	126	+	// SetOwner gives the files of the next steps to uid:gid.
	127	+	func (f *FileCommands) SetOwner(uid, gid int) { f.owner = &[2]int{uid, gid} }
	128	+
	129	+	func (f *FileCommands) name(c fileCommand, suffix string) string { return c.FilePrefix + suffix }
	130	+
	131	+	func (f *FileCommands) path(c fileCommand, suffix string) string {
	132	+	return filepath.Join(f.dir, f.name(c, suffix))
	133	+	}
	134	+
	135	+	// Init creates fresh, empty files for the next step (after deleting the ones
	136	+	// of the previous step) and returns the environment variables that name them
	137	+	// (InitializeFiles, FileCommandManager.cs:42).
	138	+	func (f *FileCommands) Init() (map[string]string, error) {
	139	+	old := f.suffix
	140	+	f.suffix = uuid.NewString()
	141	+	env := make(map[string]string, len(fileCommands))
	142	+	for _, c := range fileCommands {
	143	+	if old != "" {
	144	+	_ = f.root.Remove(f.name(c, old))
	145	+	}
	146	+	if err := createExcl(f.root, f.name(c, f.suffix), nil, 0o644, f.owner); err != nil {
	147	+	return nil, err
	148	+	}
	149	+	env[envVarName(c.ContextName)] = f.path(c, f.suffix)
	150	+	}
	151	+	return env, nil
	152	+	}
	153	+
	154	+	// Process reads the files of the step that just ran and applies them to sink.
	155	+	// A command that fails does not stop the others (ProcessFiles, :75).
	156	+	func (f *FileCommands) Process(sink Sink) {
	157	+	for _, c := range fileCommands {
	158	+	p := f.name(c, f.suffix)
	159	+	var err error
	160	+	switch c.ContextName {
	161	+	case cmdPath:
	162	+	err = processAddPath(sink, f.root, p)
	163	+	case cmdEnv:
	164	+	err = processSetEnv(sink, f.root, p)
	165	+	case cmdStepSummary:
	166	+	err = processStepSummary(sink, f.root, p)
	167	+	case cmdState:
	168	+	err = processPairs(sink, f.root, p, sink.SaveState, func(k, v string) { sink.Debug(fmt.Sprintf("Save intra-action state %s = %s", k, v)) })
	169	+	case cmdOutput:
	170	+	err = processPairs(sink, f.root, p, sink.SetOutput, func(k, v string) { sink.Debug(fmt.Sprintf("Set output %s = %s", k, v)) })
	171	+	}
	172	+	if err != nil {
	173	+	sink.Error(fmt.Sprintf("Unable to process file command '%s' successfully.", c.ContextName))
	174	+	sink.Error(err.Error())
	175	+	sink.CommandFailed()
	176	+	}
	177	+	}
	178	+	}
	179	+
	180	+	// errNotRegular is the error of a file command file that was replaced by
	181	+	// something else (a symbolic link, a FIFO, a directory).
	182	+	var errNotRegular = ghErrorf("The file is not a regular file")
	183	+
	184	+	// openRegular opens name in root for reading without following a symbolic link
	185	+	// and without blocking on a FIFO, and checks that it is a regular file.
	186	+	func openRegular(root os.Root, name string) (os.File, error) {
	187	+	fh, err := root.OpenFile(name, os.O_RDONLY\|syscall.O_NOFOLLOW\|syscall.O_NONBLOCK, 0)
	188	+	if err != nil {
	189	+	if errors.Is(err, syscall.ELOOP) \|\| errors.Is(err, syscall.EMLINK) {
	190	+	return nil, errNotRegular
	191	+	}
	192	+	return nil, err
	193	+	}
	194	+	st, err := fh.Stat()
	195	+	if err != nil {
	196	+	_ = fh.Close()
	197	+	return nil, err
	198	+	}
	199	+	if !st.Mode().IsRegular() {
	200	+	_ = fh.Close()
	201	+	return nil, errNotRegular
	202	+	}
	203	+	return fh, nil
	204	+	}
	205	+
	206	+	// readText is File.ReadAllText: a UTF-8 BOM is dropped and invalid bytes
	207	+	// become U+FFFD. A missing file gives ok=false and no error (the callers
	208	+	// debug-log it; AddPath and StepSummary skip silently); a file that is not a
	209	+	// regular file is an error.
	210	+	func readText(root *os.Root, name string) (text string, ok bool, err error) {
	211	+	fh, err := openRegular(root, name)
	212	+	if errors.Is(err, os.ErrNotExist) {
	213	+	return "", false, nil
	214	+	}
	215	+	if err != nil {
	216	+	return "", false, err
	217	+	}
	218	+	defer func() { _ = fh.Close() }()
	219	+	b, err := io.ReadAll(fh)
	220	+	if err != nil {
	221	+	return "", false, err
	222	+	}
	223	+	b = bytes.TrimPrefix(b, []byte("\xef\xbb\xbf"))
	224	+	if !utf8.Valid(b) {
	225	+	b = bytes.ToValidUTF8(b, []byte("\ufffd"))
	226	+	}
	227	+	return string(b), true, nil
	228	+	}
	229	+
	230	+	// createExcl creates name in root with the content, failing if it exists (an
	231	+	// existing entry, a symbolic link included, is removed first, never followed),
	232	+	// and gives it to owner when set. The creation uses O_EXCL\|O_NOFOLLOW, so
	233	+	// something planted between the removal and the creation makes it fail. The
	234	+	// owner is set on the open file, not by name: the user of the steps owns the
	235	+	// directory and could swap the name for something else between the close and a
	236	+	// chown by name.
	237	+	func createExcl(root os.Root, name string, content []byte, perm os.FileMode, owner [2]int) error {
	238	+	if err := root.Remove(name); err != nil && !errors.Is(err, os.ErrNotExist) {
	239	+	return err
	240	+	}
	241	+	fh, err := root.OpenFile(name, os.O_WRONLY\|os.O_CREATE\|os.O_EXCL\|syscall.O_NOFOLLOW, perm)
	242	+	if err != nil {
	243	+	return err
	244	+	}
	245	+	if _, err := fh.Write(content); err != nil {
	246	+	_ = fh.Close()
	247	+	return err
	248	+	}
	249	+	if owner != nil {
	250	+	if err := fh.Chown(owner[0], owner[1]); err != nil {
	251	+	_ = fh.Close()
	252	+	return err
	253	+	}
	254	+	}
	255	+	return fh.Close()
	256	+	}
	257	+
	258	+	// processAddPath is AddPathFileCommand.ProcessCommand (FileCommandManager.cs:134).
	259	+	func processAddPath(sink Sink, root *os.Root, name string) error {
	260	+	text, ok, err := readText(root, name)
	261	+	if err != nil \|\| !ok {
	262	+	return err
	263	+	}
	264	+	// File.ReadAllLines splits on \r\n, \n and \r.
	265	+	for _, line := range splitLines(text) {
	266	+	if line == "" {
	267	+	continue
	268	+	}
	269	+	sink.AddPath(line)
	270	+	}
	271	+	return nil
	272	+	}
	273	+
	274	+	// splitLines is File.ReadAllLines / StreamReader.ReadLine: \r\n, \r and \n
	275	+	// end a line; no empty last line is produced.
	276	+	func splitLines(s string) []string {
	277	+	var out []string
	278	+	for len(s) > 0 {
	279	+	i := strings.IndexAny(s, "\r\n")
	280	+	if i < 0 {
	281	+	out = append(out, s)
	282	+	break
	283	+	}
	284	+	out = append(out, s[:i])
	285	+	if s[i] == '\r' && i+1 < len(s) && s[i+1] == '\n' {
	286	+	i++
	287	+	}
	288	+	s = s[i+1:]
	289	+	}
	290	+	return out
	291	+	}
	292	+
	293	+	// processSetEnv is SetEnvFileCommand.ProcessCommand (:167). Pairs read before
	294	+	// a format error are applied, like the lazy enumeration of the original.
	295	+	func processSetEnv(sink Sink, root *os.Root, name string) error {
	296	+	pairs, err := readEnvFile(sink, root, name)
	297	+	for _, p := range pairs {
	298	+	if blocked := blockedEnvName(p.Key); blocked != "" {
	299	+	sink.Error(fmt.Sprintf("Can't store %s output parameter using '$GITHUB_ENV' command.", blocked))
	300	+	continue
	301	+	}
	302	+	sink.SetEnv(p.Key, p.Value)
	303	+	sink.Debug(fmt.Sprintf("%s='%s'", p.Key, p.Value))
	304	+	}
	305	+	return err
	306	+	}
	307	+
	308	+	func blockedEnvName(name string) string {
	309	+	for _, b := range setEnvBlockList {
	310	+	if strings.EqualFold(b, name) {
	311	+	return b
	312	+	}
	313	+	}
	314	+	return ""
	315	+	}
	316	+
	317	+	// processPairs serves GITHUB_STATE and GITHUB_OUTPUT.
	318	+	func processPairs(sink Sink, root *os.Root, name string, apply func(k, v string), debug func(k, v string)) error {
	319	+	pairs, err := readEnvFile(sink, root, name)
	320	+	for _, p := range pairs {
	321	+	apply(p.Key, p.Value)
	322	+	debug(p.Key, p.Value)
	323	+	}
	324	+	return err
	325	+	}
	326	+
	327	+	func readEnvFile(sink Sink, root *os.Root, name string) ([]Pair, error) {
	328	+	text, ok, err := readText(root, name)
	329	+	if err != nil {
	330	+	return nil, err
	331	+	}
	332	+	if !ok {
	333	+	sink.Debug(fmt.Sprintf("File does not exist '%s'", filepath.Join(root.Name(), name)))
	334	+	return nil, nil
	335	+	}
	336	+	return ParseEnvFile(text)
	337	+	}
	338	+
	339	+	// processStepSummary is CreateStepSummaryCommand.ProcessCommand (:230) for
	340	+	// CI-1: the file is only checked (an empty file is ignored, a file above 1 MiB
	341	+	// is an error); ermite does not store summaries yet.
	342	+	func processStepSummary(sink Sink, root *os.Root, name string) error {
	343	+	fh, err := openRegular(root, name)
	344	+	if errors.Is(err, os.ErrNotExist) {
	345	+	return nil
	346	+	}
	347	+	if err != nil {
	348	+	return err
	349	+	}
	350	+	st, err := fh.Stat()
	351	+	_ = fh.Close()
	352	+	if err != nil \|\| st.Size() == 0 {
	353	+	return nil
	354	+	}
	355	+	if st.Size() > stepSummaryLimit {
	356	+	sink.Error(fmt.Sprintf("$GITHUB_STEP_SUMMARY upload aborted, supports content up to a size of %dk, got %dk. For more information see: https://docs.github.com/actions/using-workflows/workflow-commands-for-github-actions#adding-a-markdown-summary", stepSummaryLimit/1024, st.Size()/1024))
	357	+	}
	358	+	return nil
	359	+	}
	360	+
	361	+	// ParseEnvFile parses the contents of GITHUB_ENV, GITHUB_OUTPUT or
	362	+	// GITHUB_STATE: EnvFileKeyValuePairs.GetEnumerator (FileCommandManager.cs:364).
	363	+	//
	364	+	// Lines are NAME=VALUE (split at the first "=") or heredocs NAME<<DELIMITER up
	365	+	// to a line equal to DELIMITER; the form whose marker comes first wins. Empty
	366	+	// lines between entries are skipped. Only "\n" ends a line (on Linux). The
	367	+	// pairs read before an error are returned with it, because the original
	368	+	// applies them as it enumerates.
	369	+	func ParseEnvFile(text string) ([]Pair, error) {
	370	+	var out []Pair
	371	+	index := 0
	372	+	line, _, ok := readLine(text, &index)
	373	+	for ok {
	374	+	if line != "" {
	375	+	var key, output string
	376	+	equals := strings.Index(line, "=")
	377	+	heredoc := strings.Index(line, "<<")
	378	+	switch {
	379	+	case equals >= 0 && (heredoc < 0 \|\| equals < heredoc):
	380	+	// The original tests string.IsNullOrEmpty(line) here, which cannot
	381	+	// be true; the intent (the message says so) is an empty name, and
	382	+	// an empty variable name is useless: we reject it.
	383	+	k, v, _ := strings.Cut(line, "=")
	384	+	if k == "" {
	385	+	return out, ghErrorf("Invalid format '%s'. Name must not be empty", line)
	386	+	}
	387	+	key, output = k, v
	388	+	case heredoc >= 0 && (equals < 0 \|\| heredoc < equals):
	389	+	k, delim, _ := strings.Cut(line, "<<")
	390	+	if k == "" \|\| delim == "" {
	391	+	return out, ghErrorf("Invalid format '%s'. Name must not be empty and delimiter must not be empty", line)
	392	+	}
	393	+	key = k
	394	+	start, end := index, index
	395	+	temp, newline, tok := readLine(text, &index)
	396	+	for temp != delim \|\| !tok {
	397	+	if !tok {
	398	+	return out, ghErrorf("Invalid value. Matching delimiter not found '%s'", delim)
	399	+	}
	400	+	if newline == "" {
	401	+	return out, ghErrorf("Invalid value. EOF marker missing new line.")
	402	+	}
	403	+	end = index - len(newline)
	404	+	temp, newline, tok = readLine(text, &index)
	405	+	}
	406	+	if end > start {
	407	+	output = text[start:end]
	408	+	}
	409	+	default:
	410	+	return out, ghErrorf("Invalid format '%s'", line)
	411	+	}
	412	+	out = append(out, Pair{key, output})
	413	+	}
	414	+	line, _, ok = readLine(text, &index)
	415	+	}
	416	+	return out, nil
	417	+	}
	418	+
	419	+	// readLine is the private ReadLine of EnvFileKeyValuePairs (:462): it returns
	420	+	// the line without its "\n", the newline ("\n" or "" at the end of the text)
	421	+	// and false once the text is exhausted.
	422	+	func readLine(text string, index *int) (line, newline string, ok bool) {
	423	+	if *index >= len(text) {
	424	+	return "", "", false
	425	+	}
	426	+	orig := *index
	427	+	lf := strings.IndexByte(text[orig:], '\n')
	428	+	if lf < 0 {
	429	+	*index = len(text)
	430	+	return text[orig:], "", true
	431	+	}
	432	+	*index = orig + lf + 1
	433	+	return text[orig : orig+lf], "\n", true
	434	+	}
	435	+
	436	+	// PrependedPath returns the PATH value that results from the AddPath calls
	437	+	// recorded in list (in call order) in front of orig, like
	438	+	// Handler.AddPrependPathToEnvironment (Handler.cs:205) with PathUtil.PrependPath.
	439	+	func PrependedPath(list []string, orig string) string {
	440	+	if len(list) == 0 {
	441	+	return orig
	442	+	}
	443	+	rev := slices.Clone(list)
	444	+	slices.Reverse(rev)
	445	+	prepend := strings.Join(rev, ":")
	446	+	if orig == "" {
	447	+	return prepend
	448	+	}
	449	+	if strings.HasPrefix(orig, prepend+":") {
	450	+	return orig
	451	+	}
	452	+	return prepend + ":" + orig
	453	+	}

A internal/executor/filecmd_test.go +378 -0

@@ -0,0 +1,378 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package executor
	4	+
	5	+	import (
	6	+	"os"
	7	+	"path/filepath"
	8	+	"reflect"
	9	+	"strings"
	10	+	"syscall"
	11	+	"testing"
	12	+	"time"
	13	+	)
	14	+
	15	+	// Cases ported from SetEnvFileCommandL0, SetOutputFileCommandL0 and
	16	+	// SaveStateFileCommandL0 (actions/runner, src/Test/L0/Worker). The C# tests
	17	+	// write their files with a UTF-8 BOM and join lines with the platform newline;
	18	+	// "\n" is used here and the BOM is added by writeEnvFile.
	19	+
	20	+	func TestParseEnvFile(t *testing.T) {
	21	+	type kv = Pair
	22	+	tests := []struct {
	23	+	name string
	24	+	in string
	25	+	want []kv
	26	+	wantErr string
	27	+	}{
	28	+	{"empty file", "", nil, ""},
	29	+	{"simple", "MY_ENV=MY VALUE", []kv{{"MY_ENV", "MY VALUE"}}, ""},
	30	+	{"skip empty lines", "\nMY_ENV=my value\n\nMY_ENV_2=my second value\n", []kv{{"MY_ENV", "my value"}, {"MY_ENV_2", "my second value"}}, ""},
	31	+	{"empty value", "MY_ENV=", []kv{{"MY_ENV", ""}}, ""},
	32	+	{"multiple values", "MY_ENV=my value\nMY_ENV_2=\nMY_ENV_3=my third value", []kv{{"MY_ENV", "my value"}, {"MY_ENV_2", ""}, {"MY_ENV_3", "my third value"}}, ""},
	33	+	{"special characters", "MY_ENV==abc\nMY_ENV_2=def=ghi\nMY_ENV_3=jkl=", []kv{{"MY_ENV", "=abc"}, {"MY_ENV_2", "def=ghi"}, {"MY_ENV_3", "jkl="}}, ""},
	34	+	{"heredoc", "MY_ENV<<EOF\nline one\nline two\nline three\nEOF", []kv{{"MY_ENV", "line one\nline two\nline three"}}, ""},
	35	+	{"heredoc trailing newline", "MY_ENV<<EOF\nhello\nworld\nEOF\n", []kv{{"MY_ENV", "hello\nworld"}}, ""},
	36	+	{"heredoc empty value", "MY_ENV<<EOF\nEOF", []kv{{"MY_ENV", ""}}, ""},
	37	+	{"heredoc skip empty lines", "\nMY_ENV<<EOF\nhello\nworld\nEOF\n\nMY_ENV_2<<EOF\nHELLO\nAGAIN\nEOF\n", []kv{{"MY_ENV", "hello\nworld"}, {"MY_ENV_2", "HELLO\nAGAIN"}}, ""},
	38	+	{"heredoc special characters",
	39	+	"MY_ENV<<=EOF\nhello\none\n=EOF\nMY_ENV_2<<<EOF\nhello\ntwo\n<EOF\nMY_ENV_3<<EOF\nhello\n\nthree\n\nEOF\nMY_ENV_4<<EOF\nhello=four\nEOF\nMY_ENV_5<<EOF\n EOF\nEOF",
	40	+	[]kv{{"MY_ENV", "hello\none"}, {"MY_ENV_2", "hello\ntwo"}, {"MY_ENV_3", "hello\n\nthree\n"}, {"MY_ENV_4", "hello=four"}, {"MY_ENV_5", " EOF"}}, ""},
	41	+	{"heredoc delimiter missing", "MY_ENV<<EOF line one line two line three EOF", nil, "Matching delimiter not found"},
	42	+	{"heredoc missing delimiter at end of text", "MY_ENV<<EOF\nline one\nline two", nil, "EOF marker missing new line"},
	43	+	{"heredoc multi-line missing newline", "MY_ENV<<EOF line one\n line two\n line three EOF", nil, "EOF marker missing new line"},
	44	+	{"equals first wins over heredoc", "A=b<<C", []kv{{"A", "b<<C"}}, ""},
	45	+	{"heredoc first wins over equals", "A<<B=C\nx\nB=C", []kv{{"A", "x"}}, ""},
	46	+	{"no separator", "just text", nil, "Invalid format 'just text'"},
	47	+	{"empty name", "=value", nil, "Name must not be empty"},
	48	+	{"empty heredoc name", "<<EOF\nx\nEOF", nil, "Name must not be empty and delimiter must not be empty"},
	49	+	{"empty delimiter", "A<<\nx", nil, "delimiter must not be empty"},
	50	+	{"pairs before the error are kept", "A=1\nbroken", []kv{{"A", "1"}}, "Invalid format 'broken'"},
	51	+	{"CR is data", "A=1\r\n", []kv{{"A", "1\r"}}, ""},
	52	+	}
	53	+	for _, tc := range tests {
	54	+	t.Run(tc.name, func(t *testing.T) {
	55	+	got, err := ParseEnvFile(tc.in)
	56	+	if tc.wantErr != "" {
	57	+	if err == nil \|\| !strings.Contains(err.Error(), tc.wantErr) {
	58	+	t.Fatalf("error %v, want %q", err, tc.wantErr)
	59	+	}
	60	+	} else if err != nil {
	61	+	t.Fatal(err)
	62	+	}
	63	+	if !reflect.DeepEqual(got, tc.want) {
	64	+	t.Fatalf("got %q, want %q", got, tc.want)
	65	+	}
	66	+	})
	67	+	}
	68	+	}
	69	+
	70	+	// fakeSink records the effects of file commands.
	71	+	type fakeSink struct {
	72	+	env, outputs, state map[string]string
	73	+	path []string
	74	+	errors, debugs []string
	75	+	failed bool
	76	+	}
	77	+
	78	+	func newSink() *fakeSink {
	79	+	return &fakeSink{env: map[string]string{}, outputs: map[string]string{}, state: map[string]string{}}
	80	+	}
	81	+	func (s *fakeSink) SetEnv(k, v string) { s.env[k] = v }
	82	+	func (s *fakeSink) SetOutput(k, v string) { s.outputs[k] = v }
	83	+	func (s *fakeSink) SaveState(k, v string) { s.state[k] = v }
	84	+	func (s *fakeSink) AddPath(p string) { s.path = append(s.path, p) }
	85	+	func (s *fakeSink) Error(m string) { s.errors = append(s.errors, m) }
	86	+	func (s *fakeSink) Debug(m string) { s.debugs = append(s.debugs, m) }
	87	+	func (s *fakeSink) CommandFailed() { s.failed = true }
	88	+
	89	+	func writeEnvFile(t *testing.T, path, content string) {
	90	+	t.Helper()
	91	+	// The C# tests write a BOM; ReadAllText drops it.
	92	+	if err := os.WriteFile(path, []byte("\xef\xbb\xbf"+content), 0o644); err != nil {
	93	+	t.Fatal(err)
	94	+	}
	95	+	}
	96	+
	97	+	func TestFileCommandsInit(t *testing.T) {
	98	+	dir := filepath.Join(t.TempDir(), "_runner_file_commands")
	99	+	fc, err := NewFileCommands(dir)
	100	+	if err != nil {
	101	+	t.Fatal(err)
	102	+	}
	103	+	env, err := fc.Init()
	104	+	if err != nil {
	105	+	t.Fatal(err)
	106	+	}
	107	+	for _, name := range []string{"GITHUB_PATH", "GITHUB_ENV", "GITHUB_STEP_SUMMARY", "GITHUB_STATE", "GITHUB_OUTPUT"} {
	108	+	p := env[name]
	109	+	if p == "" \|\| filepath.Dir(p) != dir {
	110	+	t.Fatalf("%s = %q", name, p)
	111	+	}
	112	+	if st, err := os.Stat(p); err != nil \|\| st.Size() != 0 {
	113	+	t.Fatalf("%s: %v %v", name, st, err)
	114	+	}
	115	+	}
	116	+	if !strings.HasPrefix(filepath.Base(env["GITHUB_ENV"]), "set_env_") \|\| !strings.HasPrefix(filepath.Base(env["GITHUB_PATH"]), "add_path_") {
	117	+	t.Fatalf("prefixes: %v", env)
	118	+	}
	119	+	// A second Init deletes the previous files and uses new names.
	120	+	env2, _ := fc.Init()
	121	+	if env2["GITHUB_ENV"] == env["GITHUB_ENV"] {
	122	+	t.Fatal("same file for two steps")
	123	+	}
	124	+	if _, err := os.Stat(env["GITHUB_ENV"]); !os.IsNotExist(err) {
	125	+	t.Fatalf("old file kept: %v", err)
	126	+	}
	127	+	}
	128	+
	129	+	func TestProcessAllCommands(t *testing.T) {
	130	+	fc, _ := NewFileCommands(t.TempDir())
	131	+	env, _ := fc.Init()
	132	+	writeEnvFile(t, env["GITHUB_ENV"], "A=1\nM<<X\nl1\nl2\nX\n")
	133	+	writeEnvFile(t, env["GITHUB_OUTPUT"], "o=2\n")
	134	+	writeEnvFile(t, env["GITHUB_STATE"], "s=3\n")
	135	+	writeEnvFile(t, env["GITHUB_PATH"], "/a\n\n/b\n/a\n")
	136	+	writeEnvFile(t, env["GITHUB_STEP_SUMMARY"], "# hi")
	137	+	s := newSink()
	138	+	fc.Process(s)
	139	+	if !reflect.DeepEqual(s.env, map[string]string{"A": "1", "M": "l1\nl2"}) \|\| s.outputs["o"] != "2" \|\| s.state["s"] != "3" {
	140	+	t.Fatalf("%+v", s)
	141	+	}
	142	+	if !reflect.DeepEqual(s.path, []string{"/a", "/b", "/a"}) {
	143	+	t.Fatalf("path %v", s.path)
	144	+	}
	145	+	if s.failed \|\| len(s.errors) != 0 {
	146	+	t.Fatalf("errors %v", s.errors)
	147	+	}
	148	+	}
	149	+
	150	+	func TestProcessBlockedEnvNames(t *testing.T) {
	151	+	// SetEnvFileCommand_BlockListItemsFiltered and _Heredoc.
	152	+	for _, content := range []string{"NODE_OPTIONS=asdf", "NODE_OPTIONS<<EOF\nasdf\nEOF", "node_options=asdf"} {
	153	+	fc, _ := NewFileCommands(t.TempDir())
	154	+	env, _ := fc.Init()
	155	+	writeEnvFile(t, env["GITHUB_ENV"], content)
	156	+	s := newSink()
	157	+	fc.Process(s)
	158	+	if len(s.env) != 0 \|\| len(s.errors) != 1 \|\| s.errors[0] != "Can't store NODE_OPTIONS output parameter using '$GITHUB_ENV' command." \|\| s.failed {
	159	+	t.Fatalf("%q: env=%v errors=%v failed=%v", content, s.env, s.errors, s.failed)
	160	+	}
	161	+	}
	162	+	}
	163	+
	164	+	func TestProcessFormatErrorFailsTheStep(t *testing.T) {
	165	+	fc, _ := NewFileCommands(t.TempDir())
	166	+	env, _ := fc.Init()
	167	+	writeEnvFile(t, env["GITHUB_OUTPUT"], "good=1\nbad line\n")
	168	+	writeEnvFile(t, env["GITHUB_ENV"], "E=1\n")
	169	+	s := newSink()
	170	+	fc.Process(s)
	171	+	// Pairs before the error are applied; other commands still run.
	172	+	if s.outputs["good"] != "1" \|\| s.env["E"] != "1" \|\| !s.failed {
	173	+	t.Fatalf("%+v", s)
	174	+	}
	175	+	want := []string{"Unable to process file command 'output' successfully.", "Invalid format 'bad line'"}
	176	+	if !reflect.DeepEqual(s.errors, want) {
	177	+	t.Fatalf("errors %q", s.errors)
	178	+	}
	179	+	}
	180	+
	181	+	func TestProcessMissingFilesAreSilent(t *testing.T) {
	182	+	// _DirectoryNotFound and _NotFound: nothing is set, no issue.
	183	+	fc, _ := NewFileCommands(t.TempDir())
	184	+	env, _ := fc.Init()
	185	+	for _, p := range env {
	186	+	_ = os.Remove(p)
	187	+	}
	188	+	s := newSink()
	189	+	fc.Process(s)
	190	+	if len(s.env)+len(s.outputs)+len(s.state)+len(s.path) != 0 \|\| len(s.errors) != 0 \|\| s.failed {
	191	+	t.Fatalf("%+v", s)
	192	+	}
	193	+	}
	194	+
	195	+	func TestStepSummarySize(t *testing.T) {
	196	+	fc, _ := NewFileCommands(t.TempDir())
	197	+	env, _ := fc.Init()
	198	+	if err := os.WriteFile(env["GITHUB_STEP_SUMMARY"], make([]byte, 1024*1024+1), 0o644); err != nil {
	199	+	t.Fatal(err)
	200	+	}
	201	+	s := newSink()
	202	+	fc.Process(s)
	203	+	if len(s.errors) != 1 \|\| !strings.HasPrefix(s.errors[0], "$GITHUB_STEP_SUMMARY upload aborted, supports content up to a size of 1024k, got 1024k.") {
	204	+	t.Fatalf("%q", s.errors)
	205	+	}
	206	+	if s.failed {
	207	+	t.Fatal("an oversized summary only logs an error")
	208	+	}
	209	+	}
	210	+
	211	+	func TestSplitLines(t *testing.T) {
	212	+	got := splitLines("a\r\nb\rc\n\nd")
	213	+	if !reflect.DeepEqual(got, []string{"a", "b", "c", "", "d"}) {
	214	+	t.Fatalf("%q", got)
	215	+	}
	216	+	}
	217	+
	218	+	func TestPrependedPath(t *testing.T) {
	219	+	// PathUtil.PrependPath: the last added entry comes first; no trailing
	220	+	// separator on an empty PATH; nothing changes when already first.
	221	+	if got := PrependedPath([]string{"/a", "/b"}, "/usr/bin"); got != "/b:/a:/usr/bin" {
	222	+	t.Fatal(got)
	223	+	}
	224	+	if got := PrependedPath([]string{"/a"}, ""); got != "/a" {
	225	+	t.Fatal(got)
	226	+	}
	227	+	if got := PrependedPath([]string{"/a"}, "/a:/usr/bin"); got != "/a:/usr/bin" {
	228	+	t.Fatal(got)
	229	+	}
	230	+	if got := PrependedPath(nil, "/usr/bin"); got != "/usr/bin" {
	231	+	t.Fatal(got)
	232	+	}
	233	+	}
	234	+
	235	+	// ---- the directory belongs to the user of the steps: nothing may be followed
	236	+
	237	+	func TestFileCommandSymlinkIsNotFollowed(t *testing.T) {
	238	+	for _, name := range []string{"GITHUB_ENV", "GITHUB_OUTPUT", "GITHUB_PATH", "GITHUB_STATE", "GITHUB_STEP_SUMMARY"} {
	239	+	t.Run(name, func(t *testing.T) {
	240	+	secret := filepath.Join(t.TempDir(), "secret")
	241	+	if err := os.WriteFile(secret, []byte("STOLEN=1\n/stolen/bin\n"), 0o600); err != nil {
	242	+	t.Fatal(err)
	243	+	}
	244	+	fc, err := NewFileCommands(t.TempDir())
	245	+	if err != nil {
	246	+	t.Fatal(err)
	247	+	}
	248	+	defer func() { _ = fc.Close() }()
	249	+	env, _ := fc.Init()
	250	+	// The step replaces its file by a link to a file only root may read.
	251	+	p := env[name]
	252	+	if err := os.Remove(p); err != nil {
	253	+	t.Fatal(err)
	254	+	}
	255	+	if err := os.Symlink(secret, p); err != nil {
	256	+	t.Fatal(err)
	257	+	}
	258	+	s := newSink()
	259	+	fc.Process(s)
	260	+	if len(s.env)+len(s.outputs)+len(s.state)+len(s.path) != 0 {
	261	+	t.Fatalf("the content of the target was read: %+v", s)
	262	+	}
	263	+	if !s.failed \|\| len(s.errors) == 0 {
	264	+	t.Fatalf("a replaced file must fail the step: %+v", s)
	265	+	}
	266	+	})
	267	+	}
	268	+	}
	269	+
	270	+	func TestFileCommandFifoDoesNotBlock(t *testing.T) {
	271	+	fc, _ := NewFileCommands(t.TempDir())
	272	+	defer func() { _ = fc.Close() }()
	273	+	env, _ := fc.Init()
	274	+	p := env["GITHUB_OUTPUT"]
	275	+	_ = os.Remove(p)
	276	+	if err := syscall.Mkfifo(p, 0o644); err != nil {
	277	+	t.Skip(err)
	278	+	}
	279	+	done := make(chan *fakeSink, 1)
	280	+	go func() { s := newSink(); fc.Process(s); done <- s }()
	281	+	select {
	282	+	case s := <-done:
	283	+	if !s.failed {
	284	+	t.Fatalf("%+v", s)
	285	+	}
	286	+	case <-time.After(5 * time.Second):
	287	+	t.Fatal("reading a FIFO blocked the executor")
	288	+	}
	289	+	}
	290	+
	291	+	func TestFileCommandDirectoryReplacedByASymlink(t *testing.T) {
	292	+	outside := t.TempDir()
	293	+	if err := os.WriteFile(filepath.Join(outside, "set_env_x"), []byte("LEAK=1\n"), 0o600); err != nil {
	294	+	t.Fatal(err)
	295	+	}
	296	+	parent := t.TempDir()
	297	+	dir := filepath.Join(parent, "cmds")
	298	+	fc, err := NewFileCommands(dir)
	299	+	if err != nil {
	300	+	t.Fatal(err)
	301	+	}
	302	+	defer func() { _ = fc.Close() }()
	303	+	if _, err := fc.Init(); err != nil {
	304	+	t.Fatal(err)
	305	+	}
	306	+	// The step swaps the directory for a link to another one.
	307	+	if err := os.Rename(dir, dir+".old"); err != nil {
	308	+	t.Fatal(err)
	309	+	}
	310	+	if err := os.Symlink(outside, dir); err != nil {
	311	+	t.Fatal(err)
	312	+	}
	313	+	fc.suffix = "x"
	314	+	s := newSink()
	315	+	fc.Process(s)
	316	+	if len(s.env) != 0 {
	317	+	t.Fatalf("a file of the swapped directory was read: %v", s.env)
	318	+	}
	319	+	// Init keeps working on the original directory, not on the link's target.
	320	+	env, err := fc.Init()
	321	+	if err != nil {
	322	+	t.Fatal(err)
	323	+	}
	324	+	if _, err := os.Stat(filepath.Join(outside, filepath.Base(env["GITHUB_ENV"]))); err == nil {
	325	+	t.Fatal("a file was created through the swapped directory")
	326	+	}
	327	+	}
	328	+
	329	+	func TestCreateExclReplacesAndNeverFollows(t *testing.T) {
	330	+	dir := t.TempDir()
	331	+	root, err := os.OpenRoot(dir)
	332	+	if err != nil {
	333	+	t.Fatal(err)
	334	+	}
	335	+	defer func() { _ = root.Close() }()
	336	+	victim := filepath.Join(t.TempDir(), "victim")
	337	+	if err := os.WriteFile(victim, []byte("keep"), 0o600); err != nil {
	338	+	t.Fatal(err)
	339	+	}
	340	+	if err := os.Symlink(victim, filepath.Join(dir, "f")); err != nil {
	341	+	t.Fatal(err)
	342	+	}
	343	+	if err := createExcl(root, "f", []byte("new"), 0o644, nil); err != nil {
	344	+	t.Fatal(err)
	345	+	}
	346	+	if b, _ := os.ReadFile(victim); string(b) != "keep" {
	347	+	t.Fatalf("the target of the link was written: %q", b)
	348	+	}
	349	+	if b, _ := os.ReadFile(filepath.Join(dir, "f")); string(b) != "new" {
	350	+	t.Fatalf("%q", b)
	351	+	}
	352	+	if fi, _ := os.Lstat(filepath.Join(dir, "f")); fi.Mode()&os.ModeSymlink != 0 {
	353	+	t.Fatal("still a link")
	354	+	}
	355	+	}
	356	+
	357	+	// The owner is applied to the open file (GO-003): chown by name after the close
	358	+	// would follow a name that the user of the steps could have swapped. Own uid and
	359	+	// gid are accepted by an unprivileged process, so the path runs here.
	360	+	func TestCreateExclChownsTheOpenFile(t *testing.T) {
	361	+	dir := t.TempDir()
	362	+	root, err := os.OpenRoot(dir)
	363	+	if err != nil {
	364	+	t.Fatal(err)
	365	+	}
	366	+	defer func() { _ = root.Close() }()
	367	+	owner := &[2]int{os.Getuid(), os.Getgid()}
	368	+	if err := createExcl(root, "f", []byte("x"), 0o644, owner); err != nil {
	369	+	t.Fatal(err)
	370	+	}
	371	+	// A uid that this process may not give files to makes the creation fail, and
	372	+	// the half-made file is not reported as success.
	373	+	if os.Getuid() != 0 {
	374	+	if err := createExcl(root, "g", []byte("x"), 0o644, &[2]int{0, 0}); err == nil {
	375	+	t.Fatal("chown to root by an unprivileged process succeeded")
	376	+	}
	377	+	}
	378	+	}

A internal/executor/job.go +554 -0

@@ -0,0 +1,554 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package executor
	4	+
	5	+	import (
	6	+	"context"
	7	+	"errors"
	8	+	"fmt"
	9	+	"os"
	10	+	"path/filepath"
	11	+	"strings"
	12	+	"sync"
	13	+	"syscall"
	14	+	"time"
	15	+
	16	+	"bnema.dev/ermite-actions/expr"
	17	+	"bnema.dev/ermite-actions/template"
	18	+	"bnema.dev/ermite-actions/workflow"
	19	+	"github.com/google/uuid"
	20	+	)
	21	+
	22	+	// Names of the implicit steps (JobExtension.cs:70 and :15).
	23	+	const (
	24	+	SetUpJobName = "Set up job"
	25	+	CompleteJobName = "Complete job"
	26	+	)
	27	+
	28	+	// DefaultTimeoutMinutes is the timeout of a job without timeout-minutes.
	29	+	const DefaultTimeoutMinutes = 360
	30	+
	31	+	// MaxOutputBytes is the limit of ermite on the job outputs, names and values
	32	+	// together (1 MiB).
	33	+	const MaxOutputBytes = 1 << 20
	34	+
	35	+	// maxIssueMessageLength is ExecutionContext._maxIssueMessageLength.
	36	+	const maxIssueMessageLength = 4096
	37	+
	38	+	// Config describes the job to run.
	39	+	type Config struct {
	40	+	// WorkflowPath and WorkflowYAML are the workflow file, which the executor
	41	+	// parses itself; JobKey selects the job.
	42	+	WorkflowPath string
	43	+	WorkflowYAML []byte
	44	+	JobKey string
	45	+	// JobName is the display name of the job, for "Complete job name: ...".
	46	+	// Empty means the job key.
	47	+	JobName string
	48	+	// ContextsJSON and MatrixJSON are the fields of the job message.
	49	+	ContextsJSON string
	50	+	MatrixJSON string
	51	+	// Repo is the repository name used when the github context lacks one.
	52	+	Repo string
	53	+	// TimeoutMinutes is the job timeout; 0 means DefaultTimeoutMinutes. It is
	54	+	// used for the message and, without Deadline, for the timer.
	55	+	TimeoutMinutes int
	56	+	// Deadline is the time left of the job timeout when Run is called, when the
	57	+	// timeout started counting earlier (at the receipt of the job by the host);
	58	+	// 0 means TimeoutMinutes from now.
	59	+	Deadline time.Duration
	60	+
	61	+	Runner RunnerInfo
	62	+	RunnerVersion string
	63	+	// Layout overrides the directory layout (tests); nil means the hosted layout
	64	+	// under Home.
	65	+	Layout *Layout
	66	+	// Home is the home directory of the user of the steps; empty means
	67	+	// /home/runner.
	68	+	Home string
	69	+	// ToolCache is RUNNER_TOOL_CACHE; empty means DefaultToolCache. The official
	70	+	// runner reads it from its own environment (HostContext.cs:411).
	71	+	ToolCache string
	72	+	// BaseEnv is the environment steps inherit; nil means os.Environ().
	73	+	BaseEnv []string
	74	+	// Debug turns ##[debug] lines on (ACTIONS_STEP_DEBUG).
	75	+	Debug bool
	76	+
	77	+	Reporter Reporter
	78	+
	79	+	// Credential, when set, is the user the steps run as.
	80	+	Credential *syscall.Credential
	81	+	// Cancellation timings and test hooks, see procSpec.
	82	+	TermAfter, KillAfter, WaitDelay time.Duration
	83	+	Signal func(pid int, sig syscall.Signal)
	84	+	// Now returns the current time; nil means time.Now.
	85	+	Now func() time.Time
	86	+	}
	87	+
	88	+	type stepResult int
	89	+
	90	+	// Values are ordered like TaskResult so that merge picks the worst
	91	+	// (TaskResultUtil.MergeTaskResults).
	92	+	const (
	93	+	resSucceeded stepResult = iota
	94	+	resFailed
	95	+	resCanceled
	96	+	resSkipped
	97	+	)
	98	+
	99	+	func (r stepResult) conclusion() Conclusion {
	100	+	switch r {
	101	+	case resFailed:
	102	+	return ConclusionFailure
	103	+	case resCanceled:
	104	+	return ConclusionCancelled
	105	+	case resSkipped:
	106	+	return ConclusionSkipped
	107	+	}
	108	+	return ConclusionSuccess
	109	+	}
	110	+
	111	+	// merge is TaskResultUtil.MergeTaskResults (Runner.Common/Util/TaskResultUtil.cs:40).
	112	+	func merge(cur *stepResult, coming stepResult) stepResult {
	113	+	if cur == nil {
	114	+	return coming
	115	+	}
	116	+	if *cur > resFailed {
	117	+	return *cur
	118	+	}
	119	+	if coming >= *cur {
	120	+	return coming
	121	+	}
	122	+	return *cur
	123	+	}
	124	+
	125	+	type job struct {
	126	+	cfg Config
	127	+	cancel *Canceller
	128	+	wf *workflow.Workflow
	129	+	jb *workflow.Job
	130	+	fc *FileCommands
	131	+	// tempRoot is RUNNER_TEMP, opened before the first step (see FileCommands).
	132	+	tempRoot *os.Root
	133	+	trackID string
	134	+
	135	+	mu sync.Mutex // serialises the Reporter and guards state and result
	136	+	state *State
	137	+	result *stepResult // nil until a step fails or the job is cancelled (Global result)
	138	+	cancelled bool
	139	+	runDefaults struct{ shell, workingDirectory string }
	140	+	baseEnv []string
	141	+	lookupPath string
	142	+	before map[string]bool
	143	+	}
	144	+
	145	+	func (j *job) now() time.Time {
	146	+	if j.cfg.Now != nil {
	147	+	return j.cfg.Now()
	148	+	}
	149	+	return time.Now()
	150	+	}
	151	+
	152	+	// Run executes one job and reports to cfg.Reporter. It always ends with a
	153	+	// JobCompleted call and returns the same Result. cancel may be nil.
	154	+	func Run(cancel *Canceller, cfg Config) Result {
	155	+	if cancel == nil {
	156	+	cancel = NewCanceller()
	157	+	}
	158	+	j := &job{cfg: cfg, cancel: cancel, trackID: "github_" + uuid.NewString()}
	159	+	defer func() {
	160	+	if j.fc != nil {
	161	+	_ = j.fc.Close()
	162	+	}
	163	+	if j.tempRoot != nil {
	164	+	_ = j.tempRoot.Close()
	165	+	}
	166	+	}()
	167	+	res := j.run()
	168	+	cfg.Reporter.JobCompleted(res)
	169	+	return res
	170	+	}
	171	+
	172	+	func (j *job) run() Result {
	173	+	limit := j.cfg.Deadline
	174	+	if limit <= 0 {
	175	+	limit = time.Duration(j.timeoutMinutes()) * time.Minute
	176	+	}
	177	+	timer := time.AfterFunc(limit, func() { j.cancel.Cancel(CancelTimeout) })
	178	+	defer timer.Stop()
	179	+
	180	+	// ---- Set up job (JobExtension.InitializeJob) ----
	181	+	rep := j.cfg.Reporter
	182	+	rep.StepStarted(1, SetUpJobName, j.now())
	183	+	steps, err := j.setUp()
	184	+	if err != nil {
	185	+	j.errorf(1, "%s", err.Error())
	186	+	rep.StepCompleted(1, SetUpJobName, ConclusionFailure, j.now())
	187	+	// The job ends here, without "Complete job" (JobRunner.cs:107).
	188	+	return Result{Conclusion: ConclusionFailure, Error: err.Error()}
	189	+	}
	190	+	rep.StepCompleted(1, SetUpJobName, ConclusionSuccess, j.now())
	191	+
	192	+	// ---- the steps (StepsRunner) ----
	193	+	for i, st := range steps {
	194	+	j.runStep(i+2, st)
	195	+	}
	196	+
	197	+	// ---- Complete job (JobExtension.FinalizeJob) ----
	198	+	last := len(steps) + 2
	199	+	outputs := j.completeJob(last)
	200	+
	201	+	return j.finish(outputs)
	202	+	}
	203	+
	204	+	func (j *job) finish(outputs map[string]string) Result {
	205	+	res := Result{Outputs: outputs}
	206	+	j.mu.Lock()
	207	+	defer j.mu.Unlock()
	208	+	r := resSucceeded
	209	+	if j.result != nil {
	210	+	r = *j.result
	211	+	}
	212	+	switch {
	213	+	case j.cancel.Reason() == CancelTimeout && j.cancelled:
	214	+	res.Conclusion = ConclusionTimedOut
	215	+	res.Error = fmt.Sprintf("The job has exceeded the maximum execution time of %d minutes.", j.timeoutMinutes())
	216	+	case r == resCanceled:
	217	+	res.Conclusion = ConclusionCancelled
	218	+	res.RunnerShutdown = j.cancel.Reason() == CancelShutdown
	219	+	case r == resFailed:
	220	+	res.Conclusion = ConclusionFailure
	221	+	default:
	222	+	res.Conclusion = ConclusionSuccess
	223	+	}
	224	+	return res
	225	+	}
	226	+
	227	+	func (j *job) timeoutMinutes() int {
	228	+	if j.cfg.TimeoutMinutes > 0 {
	229	+	return j.cfg.TimeoutMinutes
	230	+	}
	231	+	return DefaultTimeoutMinutes
	232	+	}
	233	+
	234	+	// ---------------------------------------------------------------- logging
	235	+
	236	+	// write sends one log line to the Reporter, in order.
	237	+	func (j *job) write(num int, line string) {
	238	+	j.mu.Lock()
	239	+	defer j.mu.Unlock()
	240	+	j.cfg.Reporter.Log(num, j.now(), line)
	241	+	}
	242	+
	243	+	// out writes a message that may hold several lines (ExecutionContext.Output).
	244	+	func (j *job) out(num int, text string) {
	245	+	for _, l := range strings.Split(strings.ReplaceAll(text, "\r\n", "\n"), "\n") {
	246	+	j.write(num, l)
	247	+	}
	248	+	}
	249	+
	250	+	// tagged writes tag+message; only the first line carries the tag.
	251	+	func (j *job) tagged(num int, tag, msg string) {
	252	+	if len(msg) > maxIssueMessageLength && tag != "" {
	253	+	msg = msg[:maxIssueMessageLength]
	254	+	}
	255	+	lines := strings.Split(strings.ReplaceAll(msg, "\r\n", "\n"), "\n")
	256	+	j.write(num, tag+lines[0])
	257	+	for _, l := range lines[1:] {
	258	+	j.write(num, l)
	259	+	}
	260	+	}
	261	+
	262	+	func (j *job) errorf(num int, format string, a ...any) {
	263	+	j.tagged(num, "##[error]", fmt.Sprintf(format, a...))
	264	+	}
	265	+
	266	+	func (j *job) debug(num int, msg string) {
	267	+	if j.cfg.Debug {
	268	+	for _, l := range strings.Split(strings.ReplaceAll(msg, "\r\n", "\n"), "\n") {
	269	+	j.write(num, "##[debug]"+l)
	270	+	}
	271	+	}
	272	+	}
	273	+
	274	+	// ---------------------------------------------------------------- set up
	275	+
	276	+	// evalEnv is the environment for expressions of the job and step tokens.
	277	+	func (j job) evalEnv(env ordered) template.Env {
	278	+	return template.Env{
	279	+	Contexts: j.state.Values(env),
	280	+	Funcs: map[string]expr.Func{
	281	+	// hashFiles is a CI-2 feature: the schema allows it in steps, so it
	282	+	// parses, and fails when evaluated.
	283	+	"hashFiles": func(context.Context, []expr.Value) (expr.Value, error) {
	284	+	return nil, errors.New("not supported yet: hashFiles")
	285	+	},
	286	+	},
	287	+	}
	288	+	}
	289	+
	290	+	func (j job) evalString(tok template.Token, env ordered) (string, error) {
	291	+	if tok == nil {
	292	+	return "", nil
	293	+	}
	294	+	return template.EvaluateString(context.Background(), tok, j.evalEnv(env))
	295	+	}
	296	+
	297	+	// evalMap evaluates a mapping of strings (EvaluateStepEnvironment): values
	298	+	// are converted like GitHub converts them (null is the empty string).
	299	+	func (j job) evalMap(m template.Mapping, env *ordered) ([]Pair, error) {
	300	+	if m == nil {
	301	+	return nil, nil
	302	+	}
	303	+	v, err := template.Evaluate(context.Background(), m, j.evalEnv(env))
	304	+	if err != nil {
	305	+	return nil, err
	306	+	}
	307	+	d, ok := v.(*expr.Dictionary)
	308	+	if !ok {
	309	+	return nil, nil
	310	+	}
	311	+	var out []Pair
	312	+	for _, p := range d.Pairs() {
	313	+	if k := p.Value.Kind(); k == expr.KindArray \|\| k == expr.KindDictionary {
	314	+	return nil, ghErrorf("Unexpected type '%s' encountered while reading 'environment value'. The type 'StringToken' was expected.", k)
	315	+	}
	316	+	out = append(out, Pair{p.Key, expr.CoerceString(p.Value)})
	317	+	}
	318	+	return out, nil
	319	+	}
	320	+
	321	+	// setUp parses the workflow and prepares the job: JobExtension.InitializeJob
	322	+	// (JobExtension.cs:63-182) and the job-start code of JobRunner.cs:142-177. Its
	323	+	// log lines belong to step 1.
	324	+	func (j job) setUp() ([]workflow.Step, error) {
	325	+	cfg := j.cfg
	326	+	j.out(1, fmt.Sprintf("Current runner version: '%s'", cfg.RunnerVersion))
	327	+	if cfg.Runner.Name != "" {
	328	+	j.out(1, fmt.Sprintf("Runner name: '%s'", cfg.Runner.Name))
	329	+	}
	330	+	if host, err := os.Hostname(); err == nil {
	331	+	j.out(1, fmt.Sprintf("Machine name: '%s'", host))
	332	+	}
	333	+
	334	+	wf, errs := workflow.Parse(cfg.WorkflowPath, cfg.WorkflowYAML)
	335	+	if len(errs) > 0 {
	336	+	var msgs []string
	337	+	for _, e := range errs {
	338	+	msgs = append(msgs, e.Error())
	339	+	}
	340	+	return nil, ghErrorf("The workflow is not valid. %s", strings.Join(msgs, "\n"))
	341	+	}
	342	+	jb := wf.Job(cfg.JobKey)
	343	+	if jb == nil {
	344	+	return nil, ghErrorf("The workflow has no job '%s'.", cfg.JobKey)
	345	+	}
	346	+	j.wf, j.jb = wf, jb
	347	+
	348	+	state, layout, err := NewState(cfg.ContextsJSON, cfg.MatrixJSON, cfg.JobKey, cfg.Runner, cfg.Repo)
	349	+	if err != nil {
	350	+	return nil, err
	351	+	}
	352	+	switch {
	353	+	case cfg.Layout != nil:
	354	+	state.WithLayout(*cfg.Layout)
	355	+	layout = *cfg.Layout
	356	+	case (cfg.Home != "" && cfg.Home != layout.Home) \|\| cfg.ToolCache != "":
	357	+	home := cfg.Home
	358	+	if home == "" {
	359	+	home = layout.Home
	360	+	}
	361	+	layout = NewLayout(home, filepath.Base(layout.Workspace), cfg.ToolCache)
	362	+	state.WithLayout(layout)
	363	+	}
	364	+	j.state = state
	365	+
	366	+	// Features that arrive in later phases fail the job with a clear message.
	367	+	switch {
	368	+	case jb.IsReusable():
	369	+	return nil, errors.New("not supported yet: reusable workflows (jobs.<id>.uses)")
	370	+	case jb.Container != nil:
	371	+	return nil, errors.New("not supported yet: jobs.<id>.container")
	372	+	case jb.Services != nil:
	373	+	return nil, errors.New("not supported yet: jobs.<id>.services")
	374	+	}
	375	+
	376	+	// Directories (TempDirectoryManager, PipelineDirectoryManager).
	377	+	j.out(1, "Prepare workflow directory")
	378	+	for _, d := range []string{layout.Temp, layout.Workspace, layout.ToolCache} {
	379	+	if err := os.MkdirAll(d, 0o755); err != nil {
	380	+	return nil, fmt.Errorf("create %s: %w", d, err)
	381	+	}
	382	+	}
	383	+	clearDir(layout.Temp)
	384	+	if err := os.MkdirAll(layout.Temp, 0o755); err != nil {
	385	+	return nil, err
	386	+	}
	387	+	fc, err := NewFileCommands(filepath.Join(layout.Temp, "_runner_file_commands"))
	388	+	if err != nil {
	389	+	return nil, err
	390	+	}
	391	+	if c := cfg.Credential; c != nil {
	392	+	fc.SetOwner(int(c.Uid), int(c.Gid))
	393	+	}
	394	+	j.fc = fc
	395	+	// Everything the executor writes below the temp directory later goes through
	396	+	// this root (see FileCommands): the user of the steps owns the directory.
	397	+	if j.tempRoot, err = os.OpenRoot(layout.Temp); err != nil {
	398	+	return nil, err
	399	+	}
	400	+	for _, d := range []string{layout.WorkRoot, layout.RunnerWorkspace, layout.Workspace, layout.Temp, fc.dir} {
	401	+	j.own(d)
	402	+	}
	403	+
	404	+	j.baseEnv = cfg.BaseEnv
	405	+	if j.baseEnv == nil {
	406	+	j.baseEnv = os.Environ()
	407	+	}
	408	+	if !hasEnv(j.baseEnv, "HOME") {
	409	+	j.baseEnv = append(append([]string(nil), j.baseEnv...), "HOME="+layout.Home)
	410	+	}
	411	+	j.lookupPath = envValue(j.baseEnv, "PATH")
	412	+
	413	+	// Job-level environment: workflow env, then job env, each evaluated with an
	414	+	// empty env context (JobExtension.cs:38-49, ExecutionContext.cs:1048).
	415	+	for _, m := range []*template.Mapping{wf.Env, jb.Env} {
	416	+	pairs, err := j.evalMap(m, newOrdered())
	417	+	if err != nil {
	418	+	return nil, err
	419	+	}
	420	+	for _, p := range pairs {
	421	+	j.state.SetEnv(p.Key, p.Value)
	422	+	}
	423	+	}
	424	+
	425	+	// defaults.run: a job-level "run" replaces the workflow's whole (JobExtension.cs:78-96).
	426	+	for _, d := range []*workflow.Defaults{wf.Defaults, jb.Defaults} {
	427	+	if d == nil \|\| d.Run == nil {
	428	+	continue
	429	+	}
	430	+	shell, err := j.evalString(d.Run.Shell, newOrdered())
	431	+	if err != nil {
	432	+	return nil, err
	433	+	}
	434	+	wd, err := j.evalString(d.Run.WorkingDirectory, newOrdered())
	435	+	if err != nil {
	436	+	return nil, err
	437	+	}
	438	+	j.runDefaults.shell, j.runDefaults.workingDirectory = shell, wd
	439	+	}
	440	+
	441	+	j.out(1, "Prepare all required actions")
	442	+	name := cfg.JobName
	443	+	if name == "" {
	444	+	name = cfg.JobKey
	445	+	}
	446	+	if strings.TrimSpace(name) != "" {
	447	+	j.out(1, "Complete job name: "+name)
	448	+	}
	449	+
	450	+	// Process tracking for the orphan cleanup of "Complete job".
	451	+	j.before = map[string]bool{}
	452	+	for _, p := range snapshotProcesses() {
	453	+	j.before[p.key()] = true
	454	+	}
	455	+	return jb.Steps, nil
	456	+	}
	457	+
	458	+	// own gives a file or directory created by the executor to the user the steps
	459	+	// run as, so that scripts can write to RUNNER_TEMP, the workspace and the file
	460	+	// command files. Best effort: without Credential nothing is done.
	461	+	func (j *job) own(path string) {
	462	+	if c := j.cfg.Credential; c != nil {
	463	+	_ = os.Lchown(path, int(c.Uid), int(c.Gid))
	464	+	}
	465	+	}
	466	+
	467	+	func clearDir(dir string) {
	468	+	ents, err := os.ReadDir(dir)
	469	+	if err != nil {
	470	+	return
	471	+	}
	472	+	for _, e := range ents {
	473	+	_ = os.RemoveAll(filepath.Join(dir, e.Name()))
	474	+	}
	475	+	}
	476	+
	477	+	func hasEnv(env []string, key string) bool {
	478	+	for _, kv := range env {
	479	+	if strings.HasPrefix(kv, key+"=") {
	480	+	return true
	481	+	}
	482	+	}
	483	+	return false
	484	+	}
	485	+
	486	+	func envValue(env []string, key string) string {
	487	+	v := ""
	488	+	for _, kv := range env {
	489	+	if k, val, ok := strings.Cut(kv, "="); ok && k == key {
	490	+	v = val
	491	+	}
	492	+	}
	493	+	return v
	494	+	}
	495	+
	496	+	// ---------------------------------------------------------------- complete
	497	+
	498	+	// completeJob is JobExtension.FinalizeJob (JobExtension.cs:9-190): job outputs
	499	+	// and the cleanup of orphan processes.
	500	+	func (j *job) completeJob(num int) map[string]string {
	501	+	rep := j.cfg.Reporter
	502	+	rep.StepStarted(num, CompleteJobName, j.now())
	503	+	result := resSucceeded
	504	+	outputs := map[string]string{}
	505	+
	506	+	if j.jb.Outputs != nil {
	507	+	// The runner of the official service prints nothing here: the recordings
	508	+	// of github.com show only "Cleaning up orphan processes" in "Complete
	509	+	// job", although the jobs have outputs (the log lines of
	510	+	// JobExtension.cs:48,68 belong to the legacy message that carries
	511	+	// JobOutputs).
	512	+	j.mu.Lock()
	513	+	env := j.state.JobEnv()
	514	+	j.mu.Unlock()
	515	+	pairs, err := j.evalMap(j.jb.Outputs, env)
	516	+	if err != nil {
	517	+	result = resFailed
	518	+	j.errorf(num, "Fail to evaluate job outputs")
	519	+	j.errorf(num, "%s", err.Error())
	520	+	j.mu.Lock()
	521	+	m := merge(j.result, resFailed)
	522	+	j.result = &m
	523	+	j.mu.Unlock()
	524	+	}
	525	+	total := 0
	526	+	for _, p := range pairs {
	527	+	if p.Value == "" {
	528	+	j.debug(num, fmt.Sprintf("Skip output '%s' since it's empty", p.Key))
	529	+	continue
	530	+	}
	531	+	// ermite refuses outputs above 1 MiB in total (runner.proto,
	532	+	// CompleteJobRequest.outputs), and they would not fit a frame of the
	533	+	// stream either: the job fails without outputs.
	534	+	if total += len(p.Key) + len(p.Value); total > MaxOutputBytes {
	535	+	result = resFailed
	536	+	j.errorf(num, "Fail to evaluate job outputs")
	537	+	j.errorf(num, "The job outputs are larger than 1 MiB in total.")
	538	+	j.mu.Lock()
	539	+	m := merge(j.result, resFailed)
	540	+	j.result = &m
	541	+	j.mu.Unlock()
	542	+	outputs = map[string]string{}
	543	+	break
	544	+	}
	545	+	outputs[p.Key] = p.Value
	546	+	}
	547	+	}
	548	+
	549	+	j.out(num, "Cleaning up orphan processes")
	550	+	killOrphans(j.trackID, j.before, func(l string) { j.out(num, l) })
	551	+
	552	+	rep.StepCompleted(num, CompleteJobName, result.conclusion(), j.now())
	553	+	return outputs
	554	+	}

A internal/executor/job_test.go +621 -0

@@ -0,0 +1,621 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package executor
	4	+
	5	+	import (
	6	+	"os"
	7	+	"path/filepath"
	8	+	"strings"
	9	+	"syscall"
	10	+	"testing"
	11	+	"time"
	12	+	)
	13	+
	14	+	const wfHead = "on: push\njobs:\n j:\n runs-on: x\n"
	15	+
	16	+	func logOf(s *recStep) string { return strings.Join(s.Log, "\n") }
	17	+
	18	+	func TestImplicitStepsAndNumbering(t *testing.T) {
	19	+	needTools(t, "bash")
	20	+	rec, res := runLocal(t, wfHead+" steps:\n - run: echo hi\n - name: second\n run: echo two\n", "j", nil)
	21	+	if res.Conclusion != ConclusionSuccess \|\| !rec.done {
	22	+	t.Fatalf("%+v", res)
	23	+	}
	24	+	want := []string{"Set up job", "Run echo hi", "second", "Complete job"}
	25	+	if got := rec.names(); strings.Join(got, "\|") != strings.Join(want, "\|") {
	26	+	t.Fatalf("steps %v", got)
	27	+	}
	28	+	for i, s := range rec.steps {
	29	+	if s.Num != i+1 \|\| s.Conclusion != ConclusionSuccess {
	30	+	t.Errorf("%+v", s)
	31	+	}
	32	+	}
	33	+	setup := logOf(rec.steps[0])
	34	+	for _, w := range []string{"Current runner version: 'test'", "Runner name: 'test-runner'", "Prepare workflow directory", "Prepare all required actions", "Complete job name: j"} {
	35	+	if !strings.Contains(setup, w) {
	36	+	t.Errorf("Set up job lacks %q:\n%s", w, setup)
	37	+	}
	38	+	}
	39	+	if got := logOf(rec.steps[3]); got != "Cleaning up orphan processes" {
	40	+	t.Errorf("Complete job: %q", got)
	41	+	}
	42	+	if got := logOf(rec.steps[1]); !strings.HasSuffix(got, "##[endgroup]\nhi") {
	43	+	t.Errorf("log %q", got)
	44	+	}
	45	+	}
	46	+
	47	+	func TestFailureSkipsAndStatusFunctions(t *testing.T) {
	48	+	needTools(t, "bash")
	49	+	rec, res := runLocal(t, wfHead+` steps:
	50	+	- id: boom
	51	+	run: exit 3
	52	+	- name: skipped by default success()
	53	+	run: echo NO
	54	+	- name: always
	55	+	if: always()
	56	+	run: echo ran-always
	57	+	- name: failure
	58	+	if: failure()
	59	+	run: echo ran-failure
	60	+	- name: cancelled
	61	+	if: cancelled()
	62	+	run: echo NO
	63	+	- name: outcome
	64	+	if: always() && steps.boom.outcome == 'failure' && steps.boom.conclusion == 'failure'
	65	+	run: echo ran-outcome
	66	+	`, "j", nil)
	67	+	if res.Conclusion != ConclusionFailure {
	68	+	t.Fatalf("%+v", res)
	69	+	}
	70	+	for name, c := range map[string]Conclusion{
	71	+	"Run exit 3": ConclusionFailure, "skipped by default success()": ConclusionSkipped, "always": ConclusionSuccess,
	72	+	"failure": ConclusionSuccess, "cancelled": ConclusionSkipped, "outcome": ConclusionSuccess,
	73	+	} {
	74	+	if got := rec.step(t, name).Conclusion; got != c {
	75	+	t.Errorf("%s: %s, want %s", name, got, c)
	76	+	}
	77	+	}
	78	+	if got := logOf(rec.step(t, "Run exit 3")); !strings.HasSuffix(got, "##[error]Process completed with exit code 3.") {
	79	+	t.Errorf("%q", got)
	80	+	}
	81	+	if len(rec.step(t, "skipped by default success()").Log) != 0 {
	82	+	t.Error("a skipped step logs nothing")
	83	+	}
	84	+	}
	85	+
	86	+	func TestEnvPrecedenceAndExpressions(t *testing.T) {
	87	+	needTools(t, "bash")
	88	+	rec, res := runLocal(t, `on: push
	89	+	env:
	90	+	A: wf
	91	+	B: ${{ github.event_name }}
	92	+	jobs:
	93	+	j:
	94	+	runs-on: x
	95	+	env:
	96	+	A: job
	97	+	C: ${{ github.ref_type }}-c
	98	+	steps:
	99	+	- env:
	100	+	D: ${{ env.A }}-${{ env.C }}
	101	+	A: step
	102	+	run: echo "A=$A B=$B C=$C D=$D"
	103	+	- run: echo "A=$A D=${D-unset}"
	104	+	`, "j", nil)
	105	+	if res.Conclusion != ConclusionSuccess {
	106	+	t.Fatalf("%+v", res)
	107	+	}
	108	+	// The step env sees the job env (env.A is "job", not "step"), and the
	109	+	// workflow and job env are evaluated before the steps.
	110	+	if got := logOf(rec.steps[1]); !strings.HasSuffix(got, "A=step B=push C=branch-c D=job-branch-c") {
	111	+	t.Errorf("first step:\n%s", got)
	112	+	}
	113	+	if got := logOf(rec.steps[2]); !strings.HasSuffix(got, "A=job D=unset") {
	114	+	t.Errorf("second step:\n%s", got)
	115	+	}
	116	+	}
	117	+
	118	+	func TestWorkflowCommands(t *testing.T) {
	119	+	needTools(t, "bash")
	120	+	rec, res := runLocal(t, wfHead+` steps:
	121	+	- id: s
	122	+	run: \|
	123	+	echo "::group::My group"
	124	+	echo inside
	125	+	echo "::endgroup::"
	126	+	echo "::warning file=a.txt,line=1::careful"
	127	+	echo "::error::broken"
	128	+	echo "::debug::hidden"
	129	+	echo "::set-output name=old::v"
	130	+	echo "::add-mask::secret"
	131	+	echo "::not-a-command::x"
	132	+	- run: echo "old=${{ steps.s.outputs.old }}"
	133	+	`, "j", nil)
	134	+	if res.Conclusion != ConclusionSuccess {
	135	+	t.Fatalf("%+v", res)
	136	+	}
	137	+	got := logOf(rec.steps[1])
	138	+	for _, w := range []string{"##[group]My group\ninside\n##[endgroup]", "##[warning]careful", "##[error]broken", "##[warning]The `set-output` command is deprecated", "::not-a-command::x"} {
	139	+	if !strings.Contains(got, w) {
	140	+	t.Errorf("log lacks %q:\n%s", w, got)
	141	+	}
	142	+	}
	143	+	_, body, _ := strings.Cut(got, "##[endgroup]\n") // after the header, which echoes the script
	144	+	if strings.Contains(body, "hidden") \|\| strings.Contains(body, "::add-mask") \|\| strings.Contains(body, "secret") {
	145	+	t.Errorf("debug and add-mask must not show:\n%s", body)
	146	+	}
	147	+	if !strings.HasSuffix(logOf(rec.steps[2]), "old=v") {
	148	+	t.Errorf("legacy output not visible: %s", logOf(rec.steps[2]))
	149	+	}
	150	+	}
	151	+
	152	+	func TestCommandFailureFailsTheStep(t *testing.T) {
	153	+	needTools(t, "bash")
	154	+	rec, res := runLocal(t, wfHead+" steps:\n - run: \|\n echo '::set-env name=X::1'\n", "j", nil)
	155	+	if res.Conclusion != ConclusionFailure {
	156	+	t.Fatalf("%+v", res)
	157	+	}
	158	+	got := logOf(rec.steps[1])
	159	+	if !strings.Contains(got, "##[error]Unable to process command '::set-env name=X::1' successfully.") \|\| !strings.Contains(got, "The `set-env` command is disabled.") {
	160	+	t.Errorf("%s", got)
	161	+	}
	162	+	}
	163	+
	164	+	func TestJobOutputs(t *testing.T) {
	165	+	needTools(t, "bash")
	166	+	rec, res := runLocal(t, `on: push
	167	+	jobs:
	168	+	j:
	169	+	runs-on: x
	170	+	outputs:
	171	+	a: ${{ steps.one.outputs.v }}
	172	+	empty: ${{ steps.one.outputs.none }}
	173	+	lit: fixed
	174	+	env: ${{ env.E }}
	175	+	env:
	176	+	E: from-env
	177	+	steps:
	178	+	- id: one
	179	+	run: echo "v=1" >> "$GITHUB_OUTPUT"
	180	+	- run: exit 1
	181	+	`, "j", nil)
	182	+	if res.Conclusion != ConclusionFailure {
	183	+	t.Fatalf("%+v", res)
	184	+	}
	185	+	// Outputs are collected even after a failure; empty ones are dropped.
	186	+	if len(res.Outputs) != 3 \|\| res.Outputs["a"] != "1" \|\| res.Outputs["lit"] != "fixed" \|\| res.Outputs["env"] != "from-env" {
	187	+	t.Fatalf("%v", res.Outputs)
	188	+	}
	189	+	_ = rec
	190	+	}
	191	+
	192	+	func TestNotSupportedYet(t *testing.T) {
	193	+	needTools(t, "bash")
	194	+	rec, res := runLocal(t, wfHead+` steps:
	195	+	- uses: actions/checkout@v4
	196	+	- name: after
	197	+	if: always()
	198	+	run: echo still-runs
	199	+	- run: echo skipped
	200	+	`, "j", nil)
	201	+	if res.Conclusion != ConclusionFailure {
	202	+	t.Fatalf("%+v", res)
	203	+	}
	204	+	u := rec.step(t, "Run actions/checkout@v4")
	205	+	if u.Conclusion != ConclusionFailure \|\| !strings.Contains(logOf(u), "##[error]not supported yet: uses (actions/checkout@v4)") {
	206	+	t.Fatalf("%+v", u)
	207	+	}
	208	+	if rec.step(t, "after").Conclusion != ConclusionSuccess {
	209	+	t.Error("always() steps still run after a failure")
	210	+	}
	211	+	for _, f := range []string{"container: alpine", "services:\n s:\n image: x"} {
	212	+	rec2, res2 := runLocal(t, wfHead+" "+f+"\n steps:\n - run: echo hi\n", "j", nil)
	213	+	if res2.Conclusion != ConclusionFailure \|\| res2.Error == "" \|\| !strings.Contains(logOf(rec2.steps[0]), "not supported yet") {
	214	+	t.Errorf("%q: %+v %v", f, res2, rec2.names())
	215	+	}
	216	+	}
	217	+	}
	218	+
	219	+	func TestInvalidWorkflowAndUnknownJob(t *testing.T) {
	220	+	rec, res := runLocal(t, "on: push\njobs: [", "j", nil)
	221	+	if res.Conclusion != ConclusionFailure \|\| !strings.Contains(res.Error, "The workflow is not valid") \|\| len(rec.steps) != 1 \|\| !rec.done {
	222	+	t.Fatalf("%+v %v", res, rec.names())
	223	+	}
	224	+	_, res = runLocal(t, wfHead+" steps: [{run: a}]\n", "nope", nil)
	225	+	if res.Conclusion != ConclusionFailure \|\| !strings.Contains(res.Error, "no job 'nope'") {
	226	+	t.Fatalf("%+v", res)
	227	+	}
	228	+	}
	229	+
	230	+	func TestShellVariants(t *testing.T) {
	231	+	needTools(t, "bash", "sh", "python3")
	232	+	rec, res := runLocal(t, wfHead+` steps:
	233	+	- shell: python3 {0}
	234	+	run: print("py-ok")
	235	+	- shell: bash {0}
	236	+	run: echo "no-e $-"
	237	+	- shell: sh
	238	+	run: echo sh-ok
	239	+	- shell: nonexistent-shell {0}
	240	+	run: echo NO
	241	+	`, "j", nil)
	242	+	if res.Conclusion != ConclusionFailure {
	243	+	t.Fatalf("%+v", res)
	244	+	}
	245	+	if !strings.HasSuffix(logOf(rec.steps[1]), "py-ok") \|\| !strings.Contains(logOf(rec.steps[2]), "shell: ") {
	246	+	t.Errorf("%s \| %s", logOf(rec.steps[1]), logOf(rec.steps[2]))
	247	+	}
	248	+	last := rec.steps[4]
	249	+	if last.Conclusion != ConclusionFailure \|\| !strings.Contains(logOf(last), "##[error]nonexistent-shell: command not found") {
	250	+	t.Errorf("%+v", last)
	251	+	}
	252	+	}
	253	+
	254	+	func TestGithubPathAndEnvReachNextSteps(t *testing.T) {
	255	+	needTools(t, "bash")
	256	+	rec, _ := runLocal(t, wfHead+` steps:
	257	+	- run: \|
	258	+	mkdir -p "$RUNNER_TEMP/bin1" "$RUNNER_TEMP/bin2"
	259	+	printf '#!/bin/sh\necho tool-1\n' > "$RUNNER_TEMP/bin1/tool-ermite"; chmod +x "$RUNNER_TEMP/bin1/tool-ermite"
	260	+	printf '#!/bin/sh\necho tool-2\n' > "$RUNNER_TEMP/bin2/tool-ermite"; chmod +x "$RUNNER_TEMP/bin2/tool-ermite"
	261	+	echo "$RUNNER_TEMP/bin1" >> "$GITHUB_PATH"
	262	+	echo "$RUNNER_TEMP/bin2" >> "$GITHUB_PATH"
	263	+	echo "MULTI<<X" >> "$GITHUB_ENV"; echo "a" >> "$GITHUB_ENV"; echo "b" >> "$GITHUB_ENV"; echo "X" >> "$GITHUB_ENV"
	264	+	- run: \|
	265	+	tool-ermite
	266	+	printf 'MULTI=<%s>\n' "$MULTI"
	267	+	echo "ci=$CI actions=$GITHUB_ACTIONS ws=${GITHUB_WORKSPACE##/} cwd=${PWD##/}"
	268	+	`, "j", nil)
	269	+	got := logOf(rec.steps[2])
	270	+	if !strings.HasSuffix(got, "tool-2\nMULTI=<a\nb>\nci=true actions=true ws=hello cwd=hello") {
	271	+	t.Errorf("%s", got)
	272	+	}
	273	+	}
	274	+
	275	+	// ---- cancellation and timeouts
	276	+
	277	+	func TestCancelSendsSIGINTThenStopsTheJob(t *testing.T) {
	278	+	needTools(t, "bash")
	279	+	c := NewCanceller()
	280	+	started := make(chan struct{})
	281	+	// The script prints "ready" after it installed its trap: the cancel can come
	282	+	// at once.
	283	+	go func() {
	284	+	<-started
	285	+	c.Cancel(CancelUser)
	286	+	}()
	287	+	rec := newRecorder()
	288	+	rep := &signalReporter{recorder: rec, started: started, match: "ready"}
	289	+	home := t.TempDir()
	290	+	layout := NewLayout(home, "hello", home+"/tc")
	291	+	res := Run(c, Config{
	292	+	WorkflowYAML: []byte(wfHead + ` steps:
	293	+	- run: \|
	294	+	trap 'echo got-int; exit 130' INT
	295	+	echo ready
	296	+	while :; do sleep 0.05; done
	297	+	- name: after
	298	+	run: echo NO
	299	+	- name: cleanup
	300	+	if: always()
	301	+	run: echo cleanup-ran
	302	+	`), JobKey: "j", ContextsJSON: testGitHubContext("j"), Layout: &layout, Reporter: rep,
	303	+	BaseEnv: []string{"PATH=" + envPath(), "HOME=" + home},
	304	+	TermAfter: 300 * time.Millisecond, KillAfter: 300 * time.Millisecond,
	305	+	})
	306	+	if res.Conclusion != ConclusionCancelled \|\| res.RunnerShutdown {
	307	+	t.Fatalf("%+v", res)
	308	+	}
	309	+	run := rec.steps[1]
	310	+	if run.Conclusion != ConclusionCancelled \|\| !strings.Contains(logOf(run), "got-int") \|\| !strings.Contains(logOf(run), "##[error]The operation was canceled.") {
	311	+	t.Fatalf("%s %s", run.Conclusion, logOf(run))
	312	+	}
	313	+	if rec.step(t, "after").Conclusion != ConclusionSkipped {
	314	+	t.Error("after must be skipped")
	315	+	}
	316	+	if got := rec.step(t, "cleanup"); got.Conclusion != ConclusionSuccess \|\| !strings.HasSuffix(logOf(got), "cleanup-ran") {
	317	+	t.Errorf("always() step must run after a cancel: %+v", got)
	318	+	}
	319	+	}
	320	+
	321	+	// signalReporter closes started when a log line equals match.
	322	+	type signalReporter struct {
	323	+	*recorder
	324	+	started chan struct{}
	325	+	match string
	326	+	once bool
	327	+	}
	328	+
	329	+	func (r *signalReporter) Log(n int, at time.Time, line string) {
	330	+	r.recorder.Log(n, at, line)
	331	+	if !r.once && line == r.match {
	332	+	r.once = true
	333	+	close(r.started)
	334	+	}
	335	+	}
	336	+
	337	+	func TestCancelEscalatesToTermAndKill(t *testing.T) {
	338	+	needTools(t, "bash")
	339	+	c := NewCanceller()
	340	+	var mu = make(chan struct{}, 8)
	341	+	var got []syscall.Signal
	342	+	started := make(chan struct{})
	343	+	rec := newRecorder()
	344	+	rep := &signalReporter{recorder: rec, started: started, match: "ready"}
	345	+	go func() { <-started; c.Cancel(CancelUser) }()
	346	+	home := t.TempDir()
	347	+	layout := NewLayout(home, "hello", home+"/tc")
	348	+	t0 := time.Now()
	349	+	var times []time.Duration
	350	+	res := Run(c, Config{
	351	+	WorkflowYAML: []byte(wfHead + " steps:\n - run: \|\n trap '' INT TERM\n echo ready\n while :; do sleep 0.05; done\n"),
	352	+	JobKey: "j", ContextsJSON: testGitHubContext("j"), Layout: &layout, Reporter: rep,
	353	+	BaseEnv: []string{"PATH=" + envPath(), "HOME=" + home},
	354	+	TermAfter: 250 * time.Millisecond, KillAfter: 250 * time.Millisecond,
	355	+	Signal: func(pid int, sig syscall.Signal) {
	356	+	got = append(got, sig)
	357	+	times = append(times, time.Since(t0))
	358	+	_ = syscall.Kill(pid, sig)
	359	+	mu <- struct{}{}
	360	+	},
	361	+	})
	362	+	if res.Conclusion != ConclusionCancelled {
	363	+	t.Fatalf("%+v", res)
	364	+	}
	365	+	if len(got) != 3 \|\| got[0] != syscall.SIGINT \|\| got[1] != syscall.SIGTERM \|\| got[2] != syscall.SIGKILL {
	366	+	t.Fatalf("signals %v", got)
	367	+	}
	368	+	if d := times[2] - times[0]; d < 450time.Millisecond \|\| d > 2time.Second {
	369	+	t.Errorf("kill came %v after the interrupt, want about 500ms", d)
	370	+	}
	371	+	}
	372	+
	373	+	func TestShutdownCancelSkipsRemainingSteps(t *testing.T) {
	374	+	needTools(t, "bash")
	375	+	c := NewCanceller()
	376	+	started := make(chan struct{})
	377	+	rec := newRecorder()
	378	+	rep := &signalReporter{recorder: rec, started: started, match: "ready"}
	379	+	go func() { <-started; c.Cancel(CancelShutdown) }()
	380	+	home := t.TempDir()
	381	+	layout := NewLayout(home, "hello", home+"/tc")
	382	+	res := Run(c, Config{
	383	+	WorkflowYAML: []byte(wfHead + " steps:\n - run: \|\n echo ready\n sleep 30\n - if: always()\n run: echo NO\n"),
	384	+	JobKey: "j", ContextsJSON: testGitHubContext("j"), Layout: &layout, Reporter: rep,
	385	+	BaseEnv: []string{"PATH=" + envPath(), "HOME=" + home}, TermAfter: 200 * time.Millisecond, KillAfter: 200 * time.Millisecond,
	386	+	})
	387	+	if res.Conclusion != ConclusionCancelled \|\| !res.RunnerShutdown {
	388	+	t.Fatalf("%+v", res)
	389	+	}
	390	+	// On runner shutdown conditions are not evaluated (StepsRunner.cs:205-208):
	391	+	// even always() steps are skipped.
	392	+	if got := rec.steps[2]; got.Conclusion != ConclusionSkipped {
	393	+	t.Errorf("%+v", got)
	394	+	}
	395	+	}
	396	+
	397	+	func TestJobTimeout(t *testing.T) {
	398	+	needTools(t, "bash")
	399	+	home := t.TempDir()
	400	+	layout := NewLayout(home, "hello", home+"/tc")
	401	+	rec := newRecorder()
	402	+	cfg := Config{
	403	+	WorkflowYAML: []byte(wfHead + " steps:\n - run: sleep 30\n"),
	404	+	JobKey: "j", ContextsJSON: testGitHubContext("j"), Layout: &layout, Reporter: rec,
	405	+	BaseEnv: []string{"PATH=" + envPath(), "HOME=" + home}, TermAfter: 100 * time.Millisecond, KillAfter: 100 * time.Millisecond,
	406	+	}
	407	+	c := NewCanceller()
	408	+	// A one-minute job timeout cannot be waited for: fire it as the timer does.
	409	+	time.AfterFunc(300*time.Millisecond, func() { c.Cancel(CancelTimeout) })
	410	+	res := Run(c, cfg)
	411	+	if res.Conclusion != ConclusionTimedOut \|\| !strings.Contains(res.Error, "maximum execution time of 360 minutes") {
	412	+	t.Fatalf("%+v", res)
	413	+	}
	414	+	if rec.steps[1].Conclusion != ConclusionCancelled {
	415	+	t.Errorf("%+v", rec.steps[1])
	416	+	}
	417	+	}
	418	+
	419	+	func TestStartErrorsAndOrphans(t *testing.T) {
	420	+	needTools(t, "bash")
	421	+	rec, res := runLocal(t, wfHead+` steps:
	422	+	- run: (sleep 60 &) ; echo started-background
	423	+	`, "j", nil)
	424	+	if res.Conclusion != ConclusionSuccess {
	425	+	t.Fatalf("%+v", res)
	426	+	}
	427	+	last := logOf(rec.steps[len(rec.steps)-1])
	428	+	if !strings.Contains(last, "Cleaning up orphan processes") \|\| !strings.Contains(last, "Terminate orphan process: pid (") \|\| !strings.Contains(last, ") (sleep)") {
	429	+	t.Errorf("orphan cleanup: %q", last)
	430	+	}
	431	+	}
	432	+
	433	+	func TestLineWriter(t *testing.T) {
	434	+	var got []string
	435	+	w := &lineWriter{fn: func(l string) { got = append(got, l) }}
	436	+	for _, chunk := range []string{"a\nb\r", "\nc\rd", "\ne\xff", "f\n", "tail"} {
	437	+	_, _ = w.Write([]byte(chunk))
	438	+	}
	439	+	w.Flush()
	440	+	want := []string{"a", "b", "c", "d", "e\ufffdf", "tail"}
	441	+	if strings.Join(got, "\|") != strings.Join(want, "\|") {
	442	+	t.Fatalf("%q", got)
	443	+	}
	444	+	}
	445	+
	446	+	func TestParseCommands(t *testing.T) {
	447	+	reg := func(s string) bool { return s == "set-output" \|\| s == "group" \|\| s == "warning" }
	448	+	c, ok := parseCommandV2("::set-output name=a%2Cb,x=y::v%0Aw%25", reg)
	449	+	if !ok \|\| c.Name != "set-output" \|\| c.prop("name") != "a,b" \|\| c.Data != "v\nw%" {
	450	+	t.Fatalf("%+v %v", c, ok)
	451	+	}
	452	+	if c, ok = parseCommandV2(" ::group::title", reg); !ok \|\| c.Data != "title" {
	453	+	t.Fatalf("leading space: %+v", c)
	454	+	}
	455	+	if _, ok = parseCommandV2("::unknown::x", reg); ok {
	456	+	t.Fatal("unregistered command")
	457	+	}
	458	+	if _, ok = parseCommandV2("x ::group::t", reg); ok {
	459	+	t.Fatal("V2 must start the line")
	460	+	}
	461	+	c, ok = parseCommandV1("text ##[set-output name=n;k=v%3Bz]data%5D", reg)
	462	+	if !ok \|\| c.prop("name") != "n" \|\| c.prop("k") != "v;z" \|\| c.Data != "data]" {
	463	+	t.Fatalf("v1: %+v %v", c, ok)
	464	+	}
	465	+	}
	466	+
	467	+	func TestStopCommands(t *testing.T) {
	468	+	needTools(t, "bash")
	469	+	rec, _ := runLocal(t, wfHead+` steps:
	470	+	- run: \|
	471	+	echo "::stop-commands::tok12345"
	472	+	echo "::warning::not-processed"
	473	+	echo "::tok12345::"
	474	+	echo "::warning::processed"
	475	+	`, "j", nil)
	476	+	got := logOf(rec.steps[1])
	477	+	if !strings.Contains(got, "::warning::not-processed") \|\| !strings.Contains(got, "##[warning]processed") \|\| !strings.Contains(got, "::stop-commands::tok12345") {
	478	+	t.Errorf("%s", got)
	479	+	}
	480	+	}
	481	+
	482	+	func TestJobOutputsOverOneMiBFailTheJob(t *testing.T) {
	483	+	needTools(t, "bash")
	484	+	rec, res := runLocal(t, `on: push
	485	+	jobs:
	486	+	j:
	487	+	runs-on: x
	488	+	outputs:
	489	+	big: ${{ steps.s.outputs.big }}
	490	+	other: ${{ steps.s.outputs.big }}
	491	+	steps:
	492	+	- id: s
	493	+	run: echo "big=$(head -c 600000 /dev/zero \| tr '\0' x)" >> "$GITHUB_OUTPUT"
	494	+	`, "j", nil)
	495	+	if res.Conclusion != ConclusionFailure \|\| len(res.Outputs) != 0 {
	496	+	t.Fatalf("%+v", res.Conclusion)
	497	+	}
	498	+	last := rec.steps[len(rec.steps)-1]
	499	+	if last.Name != CompleteJobName \|\| last.Conclusion != ConclusionFailure \|\| !strings.Contains(logOf(last), "##[error]The job outputs are larger than 1 MiB in total.") {
	500	+	t.Fatalf("%s %s", last.Conclusion, logOf(last))
	501	+	}
	502	+	}
	503	+
	504	+	// A step that plants a link where the executor writes event.json (before the
	505	+	// directory exists, or between two steps) must not make it write elsewhere.
	506	+	func TestEventJSONIsNeverWrittenThroughALink(t *testing.T) {
	507	+	needTools(t, "bash")
	508	+	victim := filepath.Join(t.TempDir(), "victim")
	509	+	if err := os.WriteFile(victim, []byte("keep"), 0o600); err != nil {
	510	+	t.Fatal(err)
	511	+	}
	512	+	rec, res := runLocal(t, wfHead+` steps:
	513	+	- run: \|
	514	+	rm -f "$RUNNER_TEMP/_github_workflow/event.json"
	515	+	ln -s "`+victim+`" "$RUNNER_TEMP/_github_workflow/event.json"
	516	+	- run: cat "$GITHUB_EVENT_PATH"
	517	+	`, "j", nil)
	518	+	if res.Conclusion != ConclusionSuccess {
	519	+	t.Fatalf("%+v %s", res, logOf(rec.steps[2]))
	520	+	}
	521	+	if b, _ := os.ReadFile(victim); string(b) != "keep" {
	522	+	t.Fatalf("the target of the link was overwritten: %q", b)
	523	+	}
	524	+	if got := logOf(rec.steps[2]); !strings.Contains(got, `"ref": "refs/heads/scn/x/r1"`) {
	525	+	t.Fatalf("the second step does not see a fresh event.json:\n%s", got)
	526	+	}
	527	+	}
	528	+
	529	+	func TestLineWriterBoundsALineWithoutEnd(t *testing.T) {
	530	+	var got []string
	531	+	w := &lineWriter{fn: func(l string) { got = append(got, l) }}
	532	+	// 2.5 MiB of two-byte characters without a line feed, in odd-sized writes
	533	+	// that cut characters in two.
	534	+	blob := []byte(strings.Repeat("é", (5<<20)/4))
	535	+	for i := 0; i < len(blob); i += 7777 {
	536	+	_, _ = w.Write(blob[i:min(i+7777, len(blob))])
	537	+	}
	538	+	w.Flush()
	539	+	total := 0
	540	+	for _, l := range got {
	541	+	if len(l) > maxLineBytes {
	542	+	t.Fatalf("a line of %d bytes", len(l))
	543	+	}
	544	+	if strings.ContainsRune(l, '\ufffd') {
	545	+	t.Fatal("a character was cut in two")
	546	+	}
	547	+	total += len(l)
	548	+	}
	549	+	if len(got) < 3 \|\| total != len(blob) {
	550	+	t.Fatalf("%d lines, %d bytes of %d", len(got), total, len(blob))
	551	+	}
	552	+	}
	553	+
	554	+	func TestEnvFlagReadsTheStepEnvironmentNotTheExecutors(t *testing.T) {
	555	+	// ACTIONS_ALLOW_UNSECURE_COMMANDS is read from the environment the steps
	556	+	// get (BaseEnv), not from the process of the executor.
	557	+	t.Setenv("ACTIONS_ALLOW_UNSECURE_COMMANDS", "true")
	558	+	needTools(t, "bash")
	559	+	rec, res := runLocal(t, wfHead+" steps:\n - run: echo '::set-env name=X::1'\n", "j", nil)
	560	+	if res.Conclusion != ConclusionFailure \|\| !strings.Contains(logOf(rec.steps[1]), "The `set-env` command is disabled.") {
	561	+	t.Fatalf("%+v\n%s", res, logOf(rec.steps[1]))
	562	+	}
	563	+	rec, res = runLocal(t, wfHead+" steps:\n - run: echo '::set-env name=X::1'\n - run: echo \"X=$X\"\n", "j", func(c *Config) {
	564	+	c.BaseEnv = append(c.BaseEnv, "ACTIONS_ALLOW_UNSECURE_COMMANDS=true")
	565	+	})
	566	+	if res.Conclusion != ConclusionSuccess \|\| !strings.HasSuffix(logOf(rec.steps[2]), "X=1") {
	567	+	t.Fatalf("%+v\n%s", res, logOf(rec.steps[1]))
	568	+	}
	569	+	}
	570	+
	571	+	// SIGINT goes to the process the step started, like the official runner
	572	+	// (ProcessInvoker.SendSignal: kill(_proc.Id, SIGINT)), not to its group: bash
	573	+	// is interrupted but the foreground sleep it waits for is not signalled by it.
	574	+	func TestCancelSignalsTheShellNotTheGroup(t *testing.T) {
	575	+	needTools(t, "bash")
	576	+	c := NewCanceller()
	577	+	started := make(chan struct{})
	578	+	rec := newRecorder()
	579	+	rep := &signalReporter{recorder: rec, started: started, match: "ready"}
	580	+	go func() { <-started; c.Cancel(CancelUser) }()
	581	+	home := t.TempDir()
	582	+	layout := NewLayout(home, "hello", home+"/tc")
	583	+	marker := filepath.Join(home, "child-survived")
	584	+	var sigs []syscall.Signal
	585	+	res := Run(c, Config{
	586	+	WorkflowYAML: []byte(wfHead + " steps:\n - run: \|\n trap 'echo got-int' INT\n echo ready\n sleep 1 && touch " + marker + "\n"),
	587	+	JobKey: "j", ContextsJSON: testGitHubContext("j"), Layout: &layout, Reporter: rep,
	588	+	BaseEnv: []string{"PATH=" + envPath(), "HOME=" + home},
	589	+	TermAfter: 5 * time.Second, KillAfter: 5 * time.Second,
	590	+	Signal: func(pid int, sig syscall.Signal) {
	591	+	sigs = append(sigs, sig)
	592	+	if err := syscall.Kill(pid, sig); err != nil {
	593	+	t.Error(err)
	594	+	}
	595	+	},
	596	+	})
	597	+	if res.Conclusion != ConclusionCancelled \|\| len(sigs) != 1 \|\| sigs[0] != syscall.SIGINT {
	598	+	t.Fatalf("%+v signals %v", res, sigs)
	599	+	}
	600	+	// bash runs its trap after the foreground child ends: the sleep was not killed.
	601	+	if _, err := os.Stat(marker); err != nil {
	602	+	t.Fatalf("the foreground child was signalled with the shell: %v", err)
	603	+	}
	604	+	}
	605	+
	606	+	// Deadline is what is left of the timeout when the host sent the job: the
	607	+	// executor must not grant the whole timeout again.
	608	+	func TestDeadlineOverridesTheTimeout(t *testing.T) {
	609	+	needTools(t, "bash")
	610	+	start := time.Now()
	611	+	rec, res := runLocal(t, wfHead+" steps:\n - run: sleep 30\n", "j", func(c *Config) {
	612	+	c.TimeoutMinutes = 60
	613	+	c.Deadline = 500 * time.Millisecond
	614	+	})
	615	+	if res.Conclusion != ConclusionTimedOut \|\| !strings.Contains(res.Error, "60 minutes") {
	616	+	t.Fatalf("%+v", res)
	617	+	}
	618	+	if time.Since(start) > 10*time.Second \|\| rec.steps[1].Conclusion != ConclusionCancelled {
	619	+	t.Fatalf("took %v: %+v", time.Since(start), rec.steps[1])
	620	+	}
	621	+	}

A internal/executor/proc.go +413 -0

@@ -0,0 +1,413 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package executor
	4	+
	5	+	import (
	6	+	"bytes"
	7	+	"errors"
	8	+	"fmt"
	9	+	"io"
	10	+	"os"
	11	+	"os/exec"
	12	+	"sort"
	13	+	"strconv"
	14	+	"strings"
	15	+	"sync"
	16	+	"syscall"
	17	+	"time"
	18	+	"unicode/utf8"
	19	+	)
	20	+
	21	+	// Cancellation timings of the official runner (Runner.Sdk/ProcessInvoker.cs:32):
	22	+	// SIGINT at once, SIGTERM after 7.5 s, kill 2.5 s later.
	23	+	const (
	24	+	DefaultTermAfter = 7500 * time.Millisecond
	25	+	DefaultKillAfter = 2500 * time.Millisecond
	26	+	// DefaultWaitDelay is how long the executor waits for the output pipes to
	27	+	// close after the shell exited, before it gives up on background children
	28	+	// that still hold them: ProcessInvoker.ProcessExitedHandler waits 5 s
	29	+	// (ProcessInvoker.cs:491).
	30	+	DefaultWaitDelay = 5 * time.Second
	31	+	)
	32	+
	33	+	// lineWriter turns a byte stream into lines like StreamReader.ReadLine: "\n",
	34	+	// "\r" and "\r\n" end a line, and invalid UTF-8 becomes U+FFFD. It calls fn
	35	+	// for every complete line, in order, from the goroutine that writes.
	36	+	type lineWriter struct {
	37	+	fn func(line string)
	38	+	buf []byte
	39	+	pending bool // the last byte seen was "\r": a following "\n" is part of it
	40	+	}
	41	+
	42	+	// maxLineBytes bounds what a line may hold before it is emitted unfinished: a
	43	+	// step that prints gigabytes without a line feed must not fill the memory of
	44	+	// the executor. The official runner reads lines with StreamReader.ReadLine and
	45	+	// has no such bound; a line cut here is sent as consecutive lines.
	46	+	const maxLineBytes = 1 << 20
	47	+
	48	+	func (w *lineWriter) Write(p []byte) (int, error) {
	49	+	n := len(p)
	50	+	for len(p) > 0 {
	51	+	if w.pending {
	52	+	w.pending = false
	53	+	if p[0] == '\n' {
	54	+	p = p[1:]
	55	+	continue
	56	+	}
	57	+	}
	58	+	i := bytes.IndexAny(p, "\r\n")
	59	+	if i < 0 {
	60	+	for len(w.buf)+len(p) > maxLineBytes {
	61	+	room := maxLineBytes - len(w.buf)
	62	+	w.buf = append(w.buf, p[:room]...)
	63	+	p = p[room:]
	64	+	w.emitCut()
	65	+	}
	66	+	w.buf = append(w.buf, p...)
	67	+	break
	68	+	}
	69	+	w.buf = append(w.buf, p[:i]...)
	70	+	w.emit()
	71	+	w.pending = p[i] == '\r'
	72	+	p = p[i+1:]
	73	+	}
	74	+	return n, nil
	75	+	}
	76	+
	77	+	// emitCut emits the buffer without a line end, cutting before an incomplete
	78	+	// UTF-8 character so that it is not replaced by U+FFFD.
	79	+	func (w *lineWriter) emitCut() {
	80	+	cut := len(w.buf)
	81	+	for back := 1; back <= utf8.UTFMax && back <= len(w.buf); back++ {
	82	+	if utf8.RuneStart(w.buf[len(w.buf)-back]) {
	83	+	if !utf8.FullRune(w.buf[len(w.buf)-back:]) {
	84	+	cut = len(w.buf) - back
	85	+	}
	86	+	break
	87	+	}
	88	+	}
	89	+	rest := append([]byte(nil), w.buf[cut:]...)
	90	+	w.buf = w.buf[:cut]
	91	+	w.emit()
	92	+	w.buf = append(w.buf, rest...)
	93	+	}
	94	+
	95	+	func (w *lineWriter) emit() {
	96	+	b := w.buf
	97	+	if !utf8.Valid(b) {
	98	+	b = bytes.ToValidUTF8(b, []byte("\ufffd"))
	99	+	}
	100	+	line := string(b)
	101	+	w.buf = w.buf[:0]
	102	+	w.fn(line)
	103	+	}
	104	+
	105	+	// Flush emits the last line when the stream ended without a terminator.
	106	+	func (w *lineWriter) Flush() {
	107	+	if len(w.buf) > 0 {
	108	+	w.emit()
	109	+	}
	110	+	}
	111	+
	112	+	// dispatchBatchDelay is how long the dispatcher lets lines pile up after the
	113	+	// first one before it hands a batch over. The official runner hands over
	114	+	// whatever its reader threads queued when its loop wakes up (a thread-pool
	115	+	// wake-up, about a millisecond); Go wakes in microseconds, so without this
	116	+	// pause the interleaving of a fast script's stdout and stderr would differ
	117	+	// from the official runner's more often.
	118	+	const dispatchBatchDelay = 2 * time.Millisecond
	119	+
	120	+	// dispatcher reproduces how ProcessInvoker delivers output: stdout and stderr
	121	+	// are read by two readers into two queues, and a loop that wakes up on a
	122	+	// signal hands over all queued stderr lines first and then all queued stdout
	123	+	// lines (ProcessInvoker.ProcessOutput, ProcessInvoker.cs:392-436). Within a
	124	+	// stream the order is kept; between the streams a batch puts stderr first,
	125	+	// which is visible with `bash -x` (the corpus recording of shell-custom).
	126	+	type dispatcher struct {
	127	+	fn func(string)
	128	+	mu sync.Mutex
	129	+	outQ []string
	130	+	errQ []string
	131	+	signal chan struct{}
	132	+	closed chan struct{}
	133	+	finished chan struct{}
	134	+	out, err *lineWriter
	135	+	}
	136	+
	137	+	func newDispatcher(fn func(string)) *dispatcher {
	138	+	d := &dispatcher{fn: fn, signal: make(chan struct{}, 1), closed: make(chan struct{}), finished: make(chan struct{})}
	139	+	push := func(q *[]string) func(string) {
	140	+	return func(line string) {
	141	+	d.mu.Lock()
	142	+	q = append(q, line)
	143	+	d.mu.Unlock()
	144	+	select {
	145	+	case d.signal <- struct{}{}:
	146	+	default:
	147	+	}
	148	+	}
	149	+	}
	150	+	d.out = &lineWriter{fn: push(&d.outQ)}
	151	+	d.err = &lineWriter{fn: push(&d.errQ)}
	152	+	go d.loop()
	153	+	return d
	154	+	}
	155	+
	156	+	func (d *dispatcher) drain() {
	157	+	d.mu.Lock()
	158	+	errs, outs := d.errQ, d.outQ
	159	+	d.errQ, d.outQ = nil, nil
	160	+	d.mu.Unlock()
	161	+	for _, l := range errs {
	162	+	d.fn(l)
	163	+	}
	164	+	for _, l := range outs {
	165	+	d.fn(l)
	166	+	}
	167	+	}
	168	+
	169	+	func (d *dispatcher) loop() {
	170	+	defer close(d.finished)
	171	+	for {
	172	+	select {
	173	+	case <-d.signal:
	174	+	time.Sleep(dispatchBatchDelay)
	175	+	d.drain()
	176	+	case <-d.closed:
	177	+	d.drain()
	178	+	return
	179	+	}
	180	+	}
	181	+	}
	182	+
	183	+	// Close flushes the readers and delivers what is left. Call it once the
	184	+	// process and its pipes are done.
	185	+	func (d *dispatcher) Close() {
	186	+	d.err.Flush()
	187	+	d.out.Flush()
	188	+	close(d.closed)
	189	+	<-d.finished
	190	+	}
	191	+
	192	+	// procSpec describes one process to run.
	193	+	type procSpec struct {
	194	+	Argv []string
	195	+	Dir string
	196	+	Env []string
	197	+	// Credential, when set, is the user the process runs as.
	198	+	Credential *syscall.Credential
	199	+	// OnLine receives the lines of stdout and stderr, one call at a time, in
	200	+	// the order of dispatcher.
	201	+	OnLine func(line string)
	202	+	// Cancel starts the SIGINT, SIGTERM, SIGKILL escalation when closed.
	203	+	Cancel <-chan struct{}
	204	+
	205	+	TermAfter, KillAfter, WaitDelay time.Duration
	206	+	// Signal delivers a signal to the process the step started (the shell);
	207	+	// tests replace it.
	208	+	Signal func(pid int, sig syscall.Signal)
	209	+	}
	210	+
	211	+	// procResult is how a process ended.
	212	+	type procResult struct {
	213	+	ExitCode int
	214	+	Cancelled bool // Cancel was closed while the process ran
	215	+	}
	216	+
	217	+	// runProcess starts the process in its own process group, streams its output
	218	+	// and waits. It is ProcessInvoker.ExecuteAsync for one process: stdin is
	219	+	// closed, and a closed Cancel starts the escalation. After the shell exits the
	220	+	// group is not touched (background children survive, like on a hosted runner
	221	+	// until the orphan cleanup of "Complete job").
	222	+	func runProcess(spec procSpec) (procResult, error) {
	223	+	if spec.TermAfter == 0 {
	224	+	spec.TermAfter = DefaultTermAfter
	225	+	}
	226	+	if spec.KillAfter == 0 {
	227	+	spec.KillAfter = DefaultKillAfter
	228	+	}
	229	+	if spec.WaitDelay == 0 {
	230	+	spec.WaitDelay = DefaultWaitDelay
	231	+	}
	232	+	if spec.Signal == nil {
	233	+	// Like the official runner, which signals the process it started and
	234	+	// not its group: ProcessInvoker.SendSignal calls kill(_proc.Id, signal)
	235	+	// for SIGINT and SIGTERM (Runner.Sdk/ProcessInvoker.cs:829-833), and the
	236	+	// last resort is Process.Kill() of that process alone (:855-869, which
	237	+	// is named "KillProcessTree" but does not kill the tree). What the shell
	238	+	// started is left to the shell: bash forwards nothing to a foreground
	239	+	// child it waits for, and the children that outlive the step are
	240	+	// killed by "Complete job" (the orphan cleanup, JobExtension.cs:891-939).
	241	+	// The step is still started in its own process group, so that a signal
	242	+	// sent to the executor's group (init forwarding a SIGTERM) does not reach
	243	+	// it before the executor decides.
	244	+	spec.Signal = func(pid int, sig syscall.Signal) { _ = syscall.Kill(pid, sig) }
	245	+	}
	246	+	cmd := exec.Command(spec.Argv[0], spec.Argv[1:]...)
	247	+	cmd.Dir = spec.Dir
	248	+	cmd.Env = spec.Env
	249	+	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true, Credential: spec.Credential}
	250	+	disp := newDispatcher(spec.OnLine)
	251	+	cmd.Stdout, cmd.Stderr = disp.out, disp.err
	252	+	cmd.WaitDelay = spec.WaitDelay
	253	+	if err := cmd.Start(); err != nil {
	254	+	return procResult{}, err
	255	+	}
	256	+	pid := cmd.Process.Pid
	257	+	setOOMScoreAdj(pid)
	258	+
	259	+	var mu sync.Mutex
	260	+	done := false
	261	+	cancelled := false
	262	+	var timers []*time.Timer
	263	+	stopWatch := make(chan struct{})
	264	+	watchDone := make(chan struct{})
	265	+	go func() {
	266	+	defer close(watchDone)
	267	+	select {
	268	+	case <-spec.Cancel:
	269	+	case <-stopWatch:
	270	+	return
	271	+	}
	272	+	mu.Lock()
	273	+	defer mu.Unlock()
	274	+	if done {
	275	+	return
	276	+	}
	277	+	cancelled = true
	278	+	spec.Signal(pid, syscall.SIGINT)
	279	+	send := func(sig syscall.Signal) func() {
	280	+	return func() {
	281	+	mu.Lock()
	282	+	defer mu.Unlock()
	283	+	if !done {
	284	+	spec.Signal(pid, sig)
	285	+	}
	286	+	}
	287	+	}
	288	+	timers = append(timers,
	289	+	time.AfterFunc(spec.TermAfter, send(syscall.SIGTERM)),
	290	+	time.AfterFunc(spec.TermAfter+spec.KillAfter, send(syscall.SIGKILL)))
	291	+	}()
	292	+
	293	+	err := cmd.Wait()
	294	+	mu.Lock()
	295	+	done = true
	296	+	for _, t := range timers {
	297	+	t.Stop()
	298	+	}
	299	+	was := cancelled
	300	+	mu.Unlock()
	301	+	close(stopWatch)
	302	+	<-watchDone
	303	+	disp.Close()
	304	+
	305	+	res := procResult{Cancelled: was}
	306	+	var ee *exec.ExitError
	307	+	switch {
	308	+	case err == nil, errors.Is(err, exec.ErrWaitDelay):
	309	+	case errors.As(err, &ee):
	310	+	res.ExitCode = ee.ExitCode()
	311	+	if res.ExitCode < 0 { // killed by a signal: 128+signal, as .NET reports it
	312	+	if ws, ok := ee.Sys().(syscall.WaitStatus); ok && ws.Signaled() {
	313	+	res.ExitCode = 128 + int(ws.Signal())
	314	+	} else {
	315	+	res.ExitCode = 1
	316	+	}
	317	+	}
	318	+	default:
	319	+	return res, err
	320	+	}
	321	+	return res, nil
	322	+	}
	323	+
	324	+	// setOOMScoreAdj makes the step process a likelier OOM victim than the
	325	+	// executor (ProcessInvoker.DecreaseProcessPriority, ProcessInvoker.cs:575).
	326	+	func setOOMScoreAdj(pid int) {
	327	+	_ = os.WriteFile(fmt.Sprintf("/proc/%d/oom_score_adj", pid), []byte("500"), 0o644)
	328	+	}
	329	+
	330	+	// startError turns the error of a failed start into the text of .NET's
	331	+	// Win32Exception, which the official runner logs (the corpus recording
	332	+	// "working directory that does not exist" has it).
	333	+	func startError(shellPath, dir string, err error) string {
	334	+	if errors.Is(err, os.ErrNotExist) \|\| errors.Is(err, syscall.ENOENT) {
	335	+	return fmt.Sprintf("An error occurred trying to start process '%s' with working directory '%s'. No such file or directory", shellPath, dir)
	336	+	}
	337	+	if errors.Is(err, syscall.ENOTDIR) {
	338	+	return fmt.Sprintf("An error occurred trying to start process '%s' with working directory '%s'. Not a directory", shellPath, dir)
	339	+	}
	340	+	if errors.Is(err, os.ErrPermission) {
	341	+	return fmt.Sprintf("An error occurred trying to start process '%s' with working directory '%s'. Permission denied", shellPath, dir)
	342	+	}
	343	+	return fmt.Sprintf("An error occurred trying to start process '%s' with working directory '%s'. %v", shellPath, dir, err)
	344	+	}
	345	+
	346	+	// ProcessTrackingID is the variable that marks every process of a job so that
	347	+	// "Complete job" can find the orphans (Constants.ProcessTrackingId).
	348	+	const ProcessTrackingID = "RUNNER_TRACKING_ID"
	349	+
	350	+	// procEntry is a process seen in /proc.
	351	+	type procEntry struct {
	352	+	PID int
	353	+	Name string // comm, as .NET's ProcessName
	354	+	}
	355	+
	356	+	func (p procEntry) key() string { return strconv.Itoa(p.PID) + "_" + p.Name }
	357	+
	358	+	// snapshotProcesses lists the processes (JobExtension.SnapshotProcesses).
	359	+	func snapshotProcesses() []procEntry {
	360	+	ents, err := os.ReadDir("/proc")
	361	+	if err != nil {
	362	+	return nil
	363	+	}
	364	+	var out []procEntry
	365	+	for _, e := range ents {
	366	+	pid, err := strconv.Atoi(e.Name())
	367	+	if err != nil {
	368	+	continue
	369	+	}
	370	+	comm, err := os.ReadFile("/proc/" + e.Name() + "/comm")
	371	+	if err != nil {
	372	+	continue
	373	+	}
	374	+	out = append(out, procEntry{pid, strings.TrimSuffix(string(comm), "\n")})
	375	+	}
	376	+	sort.Slice(out, func(i, j int) bool { return out[i].PID < out[j].PID })
	377	+	return out
	378	+	}
	379	+
	380	+	// processTrackingValue reads RUNNER_TRACKING_ID from the environment of a
	381	+	// process; "" when it cannot be read.
	382	+	func processTrackingValue(pid int) string {
	383	+	b, err := os.ReadFile("/proc/" + strconv.Itoa(pid) + "/environ")
	384	+	if err != nil {
	385	+	return ""
	386	+	}
	387	+	prefix := []byte(ProcessTrackingID + "=")
	388	+	for _, kv := range bytes.Split(b, []byte{0}) {
	389	+	if bytes.HasPrefix(kv, prefix) {
	390	+	return string(kv[len(prefix):])
	391	+	}
	392	+	}
	393	+	return ""
	394	+	}
	395	+
	396	+	// killOrphans kills the processes that carry the tracking id and were not
	397	+	// there before the job (JobExtension.FinalizeJob, JobExtension.cs:891-939).
	398	+	// logf receives the "Terminate orphan process" lines.
	399	+	func killOrphans(id string, before map[string]bool, logf func(string)) {
	400	+	self := os.Getpid()
	401	+	for _, p := range snapshotProcesses() {
	402	+	if p.PID == self \|\| before[p.key()] {
	403	+	continue
	404	+	}
	405	+	if !strings.EqualFold(processTrackingValue(p.PID), id) {
	406	+	continue
	407	+	}
	408	+	logf(fmt.Sprintf("Terminate orphan process: pid (%d) (%s)", p.PID, p.Name))
	409	+	_ = syscall.Kill(p.PID, syscall.SIGKILL)
	410	+	}
	411	+	}
	412	+
	413	+	var _ io.Writer = (*lineWriter)(nil)

A internal/executor/report.go +50 -0

@@ -0,0 +1,50 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package executor
	4	+
	5	+	import "time"
	6	+
	7	+	// Conclusion is how a step or a job ended, with the names of the protocol
	8	+	// (CiConclusion in ermite-api).
	9	+	type Conclusion string
	10	+
	11	+	// Conclusions.
	12	+	const (
	13	+	ConclusionSuccess Conclusion = "success"
	14	+	ConclusionFailure Conclusion = "failure"
	15	+	ConclusionCancelled Conclusion = "cancelled"
	16	+	ConclusionSkipped Conclusion = "skipped"
	17	+	ConclusionTimedOut Conclusion = "timed_out" // jobs only
	18	+	)
	19	+
	20	+	// Reporter receives what happens in a job. The executor calls it from several
	21	+	// goroutines but never concurrently: calls are serialised.
	22	+	//
	23	+	// Steps are numbered like GitHub's log: "Set up job" is 1, the steps of the
	24	+	// workflow follow, and "Complete job" is last. Every log line belongs to the
	25	+	// step that is running.
	26	+	type Reporter interface {
	27	+	// StepStarted announces a step that begins to run, with its display name.
	28	+	// A step skipped by its condition gets StepStarted and StepCompleted too.
	29	+	StepStarted(number int, name string, at time.Time)
	30	+	// Log is one line of the job log, without its line terminator.
	31	+	Log(number int, at time.Time, line string)
	32	+	// StepCompleted ends a step.
	33	+	StepCompleted(number int, name string, conclusion Conclusion, at time.Time)
	34	+	// JobCompleted is the last call: the result of the job.
	35	+	JobCompleted(Result)
	36	+	}
	37	+
	38	+	// Result is the outcome of a job.
	39	+	type Result struct {
	40	+	Conclusion Conclusion
	41	+	// Outputs are the job outputs, evaluated at the end of the job. Empty values
	42	+	// are left out, like the official runner does.
	43	+	Outputs map[string]string
	44	+	// Error is a job-level error (a timeout, a workflow that cannot be run);
	45	+	// step errors are in the log.
	46	+	Error string
	47	+	// RunnerShutdown is true when the job was cut short because the runner was
	48	+	// told to stop (see ErrShutdown).
	49	+	RunnerShutdown bool
	50	+	}

A internal/executor/shell.go +317 -0

@@ -0,0 +1,317 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package executor
	4	+
	5	+	import (
	6	+	"fmt"
	7	+	"os"
	8	+	"path/filepath"
	9	+	"strings"
	10	+
	11	+	"github.com/google/uuid"
	12	+	)
	13	+
	14	+	// Shells and script files of `run:` steps: a port of ScriptHandlerHelpers.cs
	15	+	// and of the parts of ScriptHandler.cs that build the command line
	16	+	// (Runner.Worker/Handlers). Only the Linux behaviour is ported.
	17	+
	18	+	// defaultArguments is ScriptHandlerHelpers._defaultArguments
	19	+	// (ScriptHandlerHelpers.cs:13). Keys are compared case-insensitively.
	20	+	var defaultArguments = map[string]string{
	21	+	"cmd": `/D /E:ON /V:OFF /S /C "CALL "{0}""`,
	22	+	"pwsh": `-command ". '{0}'"`,
	23	+	"powershell": `-command ". '{0}'"`,
	24	+	"bash": "--noprofile --norc -e -o pipefail {0}",
	25	+	"sh": "-e {0}",
	26	+	"python": "{0}",
	27	+	}
	28	+
	29	+	// scriptExtensions is ScriptHandlerHelpers._extensions (:23).
	30	+	var scriptExtensions = map[string]string{
	31	+	"cmd": ".cmd",
	32	+	"pwsh": ".ps1",
	33	+	"powershell": ".ps1",
	34	+	"bash": ".sh",
	35	+	"sh": ".sh",
	36	+	"python": ".py",
	37	+	}
	38	+
	39	+	// ScriptArgumentsFormat is GetScriptArgumentsFormat (:33): the argument format
	40	+	// of a built-in shell, "" for any other command.
	41	+	func ScriptArgumentsFormat(shellType string) string {
	42	+	return defaultArguments[strings.ToLower(shellType)]
	43	+	}
	44	+
	45	+	// ScriptFileExtension is GetScriptFileExtension (:42): ".sh", ".py", ... or "".
	46	+	func ScriptFileExtension(shellType string) string {
	47	+	return scriptExtensions[strings.ToLower(shellType)]
	48	+	}
	49	+
	50	+	// FixUpScriptContents is ScriptHandlerHelpers.FixUpScriptContents (:51). The
	51	+	// switch is case-sensitive, unlike the two lookups above.
	52	+	func FixUpScriptContents(shellType, contents string) string {
	53	+	switch shellType {
	54	+	case "cmd":
	55	+	// Use @echo off rather than /Q: with /Q, echo can't be turned on.
	56	+	return "@echo off\n" + contents
	57	+	case "powershell", "pwsh":
	58	+	return "$ErrorActionPreference = 'stop'\n" + contents + "\n" +
	59	+	`if ((Test-Path -LiteralPath variable:\LASTEXITCODE)) { exit $LASTEXITCODE }`
	60	+	}
	61	+	return contents
	62	+	}
	63	+
	64	+	// ParseShellOption is ParseShellOptionString (:70): "COMMAND [..ARGS]" split at
	65	+	// the first space.
	66	+	func ParseShellOption(shell string) (command, args string) {
	67	+	command, args, _ = strings.Cut(shell, " ")
	68	+	return command, args
	69	+	}
	70	+
	71	+	// ErrInvalidShell is the error of a shell whose argument format has no {0}
	72	+	// (ScriptHandler.cs:245).
	73	+	var ErrInvalidShell error = msgError("Invalid shell option. Shell must be a valid built-in (bash, sh, cmd, powershell, pwsh) or a format string containing '{0}'")
	74	+
	75	+	// Lookup finds executables like WhichUtil.Which (Runner.Sdk/Util/WhichUtil.cs):
	76	+	// in PATH with the directories added by GITHUB_PATH in front.
	77	+	type Lookup struct {
	78	+	// Path is the PATH of the executor process (not of the step).
	79	+	Path string
	80	+	// Prepend is Global.PrependPath in the order of the AddPath calls.
	81	+	Prepend []string
	82	+	}
	83	+
	84	+	// NotFoundError is the FileNotFoundException of Which(require: true).
	85	+	type NotFoundError struct{ Command string }
	86	+
	87	+	func (e *NotFoundError) Error() string { return e.Command + ": command not found" }
	88	+
	89	+	// Which returns the path of command, or "" when it is not found.
	90	+	func (l Lookup) Which(command string) string {
	91	+	if filepath.IsAbs(command) {
	92	+	if isFile(command) {
	93	+	return command
	94	+	}
	95	+	}
	96	+	path := PrependedPath(l.Prepend, l.Path)
	97	+	for _, seg := range strings.Split(path, ":") {
	98	+	if seg == "" {
	99	+	continue
	100	+	}
	101	+	seg = os.ExpandEnv(seg)
	102	+	// Directory.EnumerateFiles(seg, command): a plain file name in the directory.
	103	+	if strings.ContainsRune(command, '/') {
	104	+	continue
	105	+	}
	106	+	p := filepath.Join(seg, command)
	107	+	if isFile(p) {
	108	+	return p
	109	+	}
	110	+	}
	111	+	return ""
	112	+	}
	113	+
	114	+	// isFile reports whether p is a file, or a symbolic link whose target exists
	115	+	// (WhichUtil.IsPathValid).
	116	+	func isFile(p string) bool {
	117	+	st, err := os.Stat(p)
	118	+	return err == nil && !st.IsDir()
	119	+	}
	120	+
	121	+	// Require is Which(command, require: true).
	122	+	func (l Lookup) Require(command string) (string, error) {
	123	+	if p := l.Which(command); p != "" {
	124	+	return p, nil
	125	+	}
	126	+	return "", &NotFoundError{command}
	127	+	}
	128	+
	129	+	// ShellPlan is how a step's script is run: the shell chosen by the step, the
	130	+	// job defaults, or the runner's default.
	131	+	type ShellPlan struct {
	132	+	// Command is the shell command: "sh" for the default shell (whose path is
	133	+	// bash when found), else the first word of the shell option.
	134	+	Command string
	135	+	// CommandPath is the executable that is started, and shown on the
	136	+	// "shell:" line of the log.
	137	+	CommandPath string
	138	+	// ArgFormat is the argument format with {0} for the script path.
	139	+	ArgFormat string
	140	+	}
	141	+
	142	+	// PlanShell chooses the shell like ScriptHandler.RunAsync (:170-234) and
	143	+	// PrintActionDetails (:65-117). shell is the step's `shell` or, when empty,
	144	+	// defaults.run.shell (the caller does that merge, :171-180).
	145	+	//
	146	+	// With no shell: bash if it is in PATH, else sh, with the arguments of "sh"
	147	+	// (-e {0}). "sh" is only the name used for the arguments and the extension.
	148	+	func PlanShell(shell string, l Lookup) (ShellPlan, error) {
	149	+	if shell == "" {
	150	+	p, err := l.lookupBashOrSh()
	151	+	if err != nil {
	152	+	return ShellPlan{}, err
	153	+	}
	154	+	return checkFormat(ShellPlan{Command: "sh", CommandPath: p, ArgFormat: ScriptArgumentsFormat("sh")})
	155	+	}
	156	+	cmd, args := ParseShellOption(shell)
	157	+	p, err := l.Require(cmd)
	158	+	if err != nil {
	159	+	return ShellPlan{}, err
	160	+	}
	161	+	format := strings.TrimLeft(args, " \t\n\v\f\r")
	162	+	if format == "" {
	163	+	format = ScriptArgumentsFormat(cmd)
	164	+	}
	165	+	return checkFormat(ShellPlan{Command: cmd, CommandPath: p, ArgFormat: format})
	166	+	}
	167	+
	168	+	func (l Lookup) lookupBashOrSh() (string, error) {
	169	+	if p := l.Which("bash"); p != "" {
	170	+	return p, nil
	171	+	}
	172	+	return l.Require("sh")
	173	+	}
	174	+
	175	+	func checkFormat(p ShellPlan) (ShellPlan, error) {
	176	+	if p.ArgFormat == "" \|\| !strings.Contains(p.ArgFormat, "{0}") {
	177	+	return p, ErrInvalidShell
	178	+	}
	179	+	return p, nil
	180	+	}
	181	+
	182	+	// ShellLine is the text after "shell: " in the step header (:121-126).
	183	+	func (p ShellPlan) ShellLine() string { return p.CommandPath + " " + p.ArgFormat }
	184	+
	185	+	// Extension is the extension of the script file, from the shell command.
	186	+	func (p ShellPlan) Extension() string { return ScriptFileExtension(p.Command) }
	187	+
	188	+	// FormatArguments is string.Format(argFormat, path) of .NET with one argument:
	189	+	// {0} is the path, {{ and }} are braces; anything else is a format error. The
	190	+	// script path has its double quotes escaped first (ScriptHandler.cs:252).
	191	+	func FormatArguments(format, scriptPath string) (string, error) {
	192	+	path := strings.ReplaceAll(scriptPath, `"`, `\"`)
	193	+	var b strings.Builder
	194	+	for i := 0; i < len(format); i++ {
	195	+	switch c := format[i]; {
	196	+	case c == '{' && i+1 < len(format) && format[i+1] == '{':
	197	+	b.WriteByte('{')
	198	+	i++
	199	+	case c == '}' && i+1 < len(format) && format[i+1] == '}':
	200	+	b.WriteByte('}')
	201	+	i++
	202	+	case c == '{':
	203	+	end := strings.IndexByte(format[i:], '}')
	204	+	if end < 0 {
	205	+	return "", ghErrorf("Input string was not in a correct format.")
	206	+	}
	207	+	spec := format[i+1 : i+end]
	208	+	index, _, _ := strings.Cut(spec, ",")
	209	+	index, _, _ = strings.Cut(index, ":")
	210	+	if strings.TrimSpace(index) != "0" {
	211	+	return "", ghErrorf("Index (zero based) must be greater than or equal to zero and less than the size of the argument list.")
	212	+	}
	213	+	b.WriteString(path)
	214	+	i += end
	215	+	case c == '}':
	216	+	return "", ghErrorf("Input string was not in a correct format.")
	217	+	default:
	218	+	b.WriteByte(c)
	219	+	}
	220	+	}
	221	+	return b.String(), nil
	222	+	}
	223	+
	224	+	// ParseArguments splits an argument string into arguments the way .NET does
	225	+	// for Process.Start on Unix (Process.Unix.cs ParseArgumentsIntoList): spaces
	226	+	// and tabs separate; double quotes group; backslashes only count before a
	227	+	// quote.
	228	+	func ParseArguments(s string) []string {
	229	+	var out []string
	230	+	for i := 0; i < len(s); i++ {
	231	+	for i < len(s) && (s[i] == ' ' \|\| s[i] == '\t') {
	232	+	i++
	233	+	}
	234	+	if i == len(s) {
	235	+	break
	236	+	}
	237	+	var cur strings.Builder
	238	+	inQuotes := false
	239	+	for i < len(s) {
	240	+	n := 0
	241	+	for i < len(s) && s[i] == '\\' {
	242	+	i++
	243	+	n++
	244	+	}
	245	+	if n > 0 {
	246	+	if i >= len(s) \|\| s[i] != '"' {
	247	+	cur.WriteString(strings.Repeat(`\`, n))
	248	+	} else {
	249	+	cur.WriteString(strings.Repeat(`\`, n/2))
	250	+	if n%2 != 0 {
	251	+	cur.WriteByte('"')
	252	+	i++
	253	+	}
	254	+	}
	255	+	continue
	256	+	}
	257	+	c := s[i]
	258	+	if c == '"' {
	259	+	if inQuotes && i < len(s)-1 && s[i+1] == '"' {
	260	+	cur.WriteByte('"')
	261	+	i++
	262	+	} else {
	263	+	inQuotes = !inQuotes
	264	+	}
	265	+	i++
	266	+	continue
	267	+	}
	268	+	if (c == ' ' \|\| c == '\t') && !inQuotes {
	269	+	break
	270	+	}
	271	+	cur.WriteByte(c)
	272	+	i++
	273	+	}
	274	+	out = append(out, cur.String())
	275	+	}
	276	+	return out
	277	+	}
	278	+
	279	+	// Script is a script written to the temp directory, ready to run.
	280	+	type Script struct {
	281	+	// Path is the script file.
	282	+	Path string
	283	+	// Argv is the command: the shell and its arguments with the script path.
	284	+	Argv []string
	285	+	}
	286	+
	287	+	// WriteScript writes the script of a step to the directory of root (RUNNER_TEMP)
	288	+	// under a random name with the extension of the shell, and builds the command
	289	+	// line (ScriptHandler.RunAsync :247-285). The file has no BOM and is not
	290	+	// executable: the shell reads it. The directory belongs to the user of the steps,
	291	+	// so the file is created through root, exclusively and without following links
	292	+	// (see createExcl).
	293	+	func WriteScript(root *os.Root, p ShellPlan, contents string) (Script, error) {
	294	+	name := uuid.NewString() + p.Extension()
	295	+	path := filepath.Join(root.Name(), name)
	296	+	args, err := FormatArguments(p.ArgFormat, path)
	297	+	if err != nil {
	298	+	return Script{}, err
	299	+	}
	300	+	contents = FixUpScriptContents(p.Command, contents)
	301	+	if err := createExcl(root, name, []byte(contents), 0o644, nil); err != nil {
	302	+	return Script{}, fmt.Errorf("write script: %w", err)
	303	+	}
	304	+	return Script{Path: path, Argv: append([]string{p.CommandPath}, ParseArguments(args)...)}, nil
	305	+	}
	306	+
	307	+	// ResolveWorkingDirectory is Path.Combine(workspace, workingDirectory ?? "")
	308	+	// (ScriptHandler.cs:168): an absolute directory replaces the workspace.
	309	+	func ResolveWorkingDirectory(workspace, dir string) string {
	310	+	if filepath.IsAbs(dir) {
	311	+	return dir
	312	+	}
	313	+	if dir == "" {
	314	+	return workspace
	315	+	}
	316	+	return strings.TrimRight(workspace, "/") + "/" + dir
	317	+	}

A internal/executor/shell_test.go +281 -0

@@ -0,0 +1,281 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package executor
	4	+
	5	+	import (
	6	+	"os"
	7	+	"path/filepath"
	8	+	"reflect"
	9	+	"strings"
	10	+	"testing"
	11	+	)
	12	+
	13	+	func fakeBin(t *testing.T, names ...string) string {
	14	+	t.Helper()
	15	+	dir := t.TempDir()
	16	+	for _, n := range names {
	17	+	if err := os.WriteFile(filepath.Join(dir, n), []byte("#!/bin/sh\n"), 0o755); err != nil {
	18	+	t.Fatal(err)
	19	+	}
	20	+	}
	21	+	return dir
	22	+	}
	23	+
	24	+	func TestBuiltinFormatsAndExtensions(t *testing.T) {
	25	+	// ScriptHandlerHelpers.cs _defaultArguments and _extensions (Linux rows).
	26	+	for shell, want := range map[string][2]string{
	27	+	"bash": {"--noprofile --norc -e -o pipefail {0}", ".sh"},
	28	+	"sh": {"-e {0}", ".sh"},
	29	+	"python": {"{0}", ".py"},
	30	+	"pwsh": {`-command ". '{0}'"`, ".ps1"},
	31	+	"powershell": {`-command ". '{0}'"`, ".ps1"},
	32	+	"cmd": {`/D /E:ON /V:OFF /S /C "CALL "{0}""`, ".cmd"},
	33	+	"BASH": {"--noprofile --norc -e -o pipefail {0}", ".sh"}, // case-insensitive tables
	34	+	"python3": {"", ""},
	35	+	"ruby": {"", ""},
	36	+	} {
	37	+	if got := ScriptArgumentsFormat(shell); got != want[0] {
	38	+	t.Errorf("format(%s) = %q", shell, got)
	39	+	}
	40	+	if got := ScriptFileExtension(shell); got != want[1] {
	41	+	t.Errorf("extension(%s) = %q", shell, got)
	42	+	}
	43	+	}
	44	+	}
	45	+
	46	+	func TestFixUpScriptContents(t *testing.T) {
	47	+	if got := FixUpScriptContents("cmd", "echo hi"); got != "@echo off\necho hi" {
	48	+	t.Fatal(got)
	49	+	}
	50	+	want := "$ErrorActionPreference = 'stop'\nGet-Date\nif ((Test-Path -LiteralPath variable:\\LASTEXITCODE)) { exit $LASTEXITCODE }"
	51	+	for _, s := range []string{"pwsh", "powershell"} {
	52	+	if got := FixUpScriptContents(s, "Get-Date"); got != want {
	53	+	t.Fatalf("%s: %q", s, got)
	54	+	}
	55	+	}
	56	+	// The switch is case-sensitive: PWSH is left alone.
	57	+	for _, s := range []string{"bash", "sh", "python", "PWSH", "Cmd"} {
	58	+	if got := FixUpScriptContents(s, "x"); got != "x" {
	59	+	t.Fatalf("%s: %q", s, got)
	60	+	}
	61	+	}
	62	+	}
	63	+
	64	+	func TestParseShellOption(t *testing.T) {
	65	+	for in, want := range map[string][2]string{
	66	+	"bash": {"bash", ""},
	67	+	"bash -x {0}": {"bash", "-x {0}"},
	68	+	"python3 {0}": {"python3", "{0}"},
	69	+	"perl -e {0}": {"perl", " -e {0}"}, // Split(" ", 2) keeps the second space
	70	+	"": {"", ""},
	71	+	"node -e \"x y\" {0}": {"node", "-e \"x y\" {0}"},
	72	+	} {
	73	+	c, a := ParseShellOption(in)
	74	+	if c != want[0] \|\| a != want[1] {
	75	+	t.Errorf("%q: %q %q", in, c, a)
	76	+	}
	77	+	}
	78	+	}
	79	+
	80	+	func TestPlanShell(t *testing.T) {
	81	+	both := fakeBin(t, "bash", "sh", "python", "python3", "perl")
	82	+	onlySh := fakeBin(t, "sh")
	83	+	l := Lookup{Path: both}
	84	+	tests := []struct {
	85	+	name string
	86	+	shell string
	87	+	l Lookup
	88	+	want ShellPlan
	89	+	err string
	90	+	}{
	91	+	{"default uses bash with the arguments of sh", "", l, ShellPlan{"sh", filepath.Join(both, "bash"), "-e {0}"}, ""},
	92	+	{"default falls back to sh", "", Lookup{Path: onlySh}, ShellPlan{"sh", filepath.Join(onlySh, "sh"), "-e {0}"}, ""},
	93	+	{"bash", "bash", l, ShellPlan{"bash", filepath.Join(both, "bash"), "--noprofile --norc -e -o pipefail {0}"}, ""},
	94	+	{"sh", "sh", l, ShellPlan{"sh", filepath.Join(both, "sh"), "-e {0}"}, ""},
	95	+	{"python", "python", l, ShellPlan{"python", filepath.Join(both, "python"), "{0}"}, ""},
	96	+	{"custom with args", "python3 {0}", l, ShellPlan{"python3", filepath.Join(both, "python3"), "{0}"}, ""},
	97	+	{"custom args are left-trimmed", "perl -w {0}", l, ShellPlan{"perl", filepath.Join(both, "perl"), "-w {0}"}, ""},
	98	+	{"bash with explicit args", "bash -x {0}", l, ShellPlan{"bash", filepath.Join(both, "bash"), "-x {0}"}, ""},
	99	+	{"unknown command", "nope {0}", l, ShellPlan{}, "nope: command not found"},
	100	+	{"pwsh not installed", "pwsh", l, ShellPlan{}, "pwsh: command not found"},
	101	+	{"no sh at all", "", Lookup{Path: t.TempDir()}, ShellPlan{}, "sh: command not found"},
	102	+	{"custom without {0}", "perl -w", l, ShellPlan{}, "Invalid shell option"},
	103	+	{"no format for an unknown shell", "perl", l, ShellPlan{}, "Invalid shell option"},
	104	+	}
	105	+	for _, tc := range tests {
	106	+	t.Run(tc.name, func(t *testing.T) {
	107	+	got, err := PlanShell(tc.shell, tc.l)
	108	+	if tc.err != "" {
	109	+	if err == nil \|\| !strings.Contains(err.Error(), tc.err) {
	110	+	t.Fatalf("err = %v, want %q", err, tc.err)
	111	+	}
	112	+	return
	113	+	}
	114	+	if err != nil \|\| got != tc.want {
	115	+	t.Fatalf("got %+v, %v; want %+v", got, err, tc.want)
	116	+	}
	117	+	})
	118	+	}
	119	+	}
	120	+
	121	+	func TestPlanShellUsesPrependedPath(t *testing.T) {
	122	+	sys, extra := fakeBin(t, "sh"), fakeBin(t, "bash")
	123	+	p, err := PlanShell("", Lookup{Path: sys, Prepend: []string{extra}})
	124	+	if err != nil \|\| p.CommandPath != filepath.Join(extra, "bash") {
	125	+	t.Fatalf("%+v %v", p, err)
	126	+	}
	127	+	}
	128	+
	129	+	func TestShellLine(t *testing.T) {
	130	+	p := ShellPlan{"sh", "/usr/bin/bash", "-e {0}"}
	131	+	if p.ShellLine() != "/usr/bin/bash -e {0}" \|\| p.Extension() != ".sh" {
	132	+	t.Fatal(p.ShellLine(), p.Extension())
	133	+	}
	134	+	}
	135	+
	136	+	func TestFormatArguments(t *testing.T) {
	137	+	for _, tc := range []struct{ format, path, want, err string }{
	138	+	{"-e {0}", "/t/a.sh", "-e /t/a.sh", ""},
	139	+	{"{0}", "/t/a b.sh", "/t/a b.sh", ""},
	140	+	{"-c '{0}' {0}", "/t/x", "-c '/t/x' /t/x", ""},
	141	+	{"{{0}} {0}", "/t/x", "{0} /t/x", ""},
	142	+	{"x }}", "/t/x", "x }", ""},
	143	+	{"{0}", `/t/a"b`, `/t/a\"b`, ""}, // double quotes in the path are escaped
	144	+	{"{0:D}", "/t/x", "/t/x", ""},
	145	+	{"{1}", "/t/x", "", "Index"},
	146	+	{"{0", "/t/x", "", "not in a correct format"},
	147	+	{"a } b", "/t/x", "", "not in a correct format"},
	148	+	} {
	149	+	got, err := FormatArguments(tc.format, tc.path)
	150	+	if tc.err != "" {
	151	+	if err == nil \|\| !strings.Contains(err.Error(), tc.err) {
	152	+	t.Errorf("%q: err = %v", tc.format, err)
	153	+	}
	154	+	continue
	155	+	}
	156	+	if err != nil \|\| got != tc.want {
	157	+	t.Errorf("%q: %q %v", tc.format, got, err)
	158	+	}
	159	+	}
	160	+	}
	161	+
	162	+	func TestParseArguments(t *testing.T) {
	163	+	for in, want := range map[string][]string{
	164	+	"": nil,
	165	+	" ": nil,
	166	+	"-e /t/a.sh": {"-e", "/t/a.sh"},
	167	+	"--noprofile --norc -e -o pipefail /t/a.sh": {"--noprofile", "--norc", "-e", "-o", "pipefail", "/t/a.sh"},
	168	+	`-c "a b" c`: {"-c", "a b", "c"},
	169	+	`a\"b`: {`a"b`},
	170	+	`a\\"b c"`: {`a\b c`},
	171	+	`a\\b`: {`a\\b`},
	172	+	`"a""b"`: {`a"b`},
	173	+	"a\tb": {"a", "b"},
	174	+	`-command ". '/t/x.ps1'"`: {"-command", ". '/t/x.ps1'"},
	175	+	`""`: {""},
	176	+	} {
	177	+	if got := ParseArguments(in); !reflect.DeepEqual(got, want) {
	178	+	t.Errorf("%q: %q, want %q", in, got, want)
	179	+	}
	180	+	}
	181	+	}
	182	+
	183	+	func TestWriteScript(t *testing.T) {
	184	+	dir := t.TempDir()
	185	+	root, err := os.OpenRoot(dir)
	186	+	if err != nil {
	187	+	t.Fatal(err)
	188	+	}
	189	+	defer func() { _ = root.Close() }()
	190	+	bin := fakeBin(t, "bash", "sh", "python")
	191	+	l := Lookup{Path: bin}
	192	+	for _, tc := range []struct {
	193	+	shell, ext string
	194	+	args []string
	195	+	}{
	196	+	{"", ".sh", []string{"-e"}},
	197	+	{"bash", ".sh", []string{"--noprofile", "--norc", "-e", "-o", "pipefail"}},
	198	+	{"python", ".py", nil},
	199	+	} {
	200	+	plan, err := PlanShell(tc.shell, l)
	201	+	if err != nil {
	202	+	t.Fatal(err)
	203	+	}
	204	+	s, err := WriteScript(root, plan, "echo hi\n")
	205	+	if err != nil {
	206	+	t.Fatal(err)
	207	+	}
	208	+	if filepath.Dir(s.Path) != dir \|\| filepath.Ext(s.Path) != tc.ext {
	209	+	t.Fatalf("%s: path %s", tc.shell, s.Path)
	210	+	}
	211	+	want := append(append([]string{plan.CommandPath}, tc.args...), s.Path)
	212	+	if !reflect.DeepEqual(s.Argv, want) {
	213	+	t.Fatalf("%s: argv %q, want %q", tc.shell, s.Argv, want)
	214	+	}
	215	+	b, _ := os.ReadFile(s.Path)
	216	+	if string(b) != "echo hi\n" {
	217	+	t.Fatalf("contents %q", b)
	218	+	}
	219	+	}
	220	+	// A custom shell without an extension in the table: no extension.
	221	+	plan, _ := PlanShell("python {0}", l)
	222	+	if s, _ := WriteScript(root, plan, "x"); filepath.Ext(s.Path) != ".py" {
	223	+	t.Fatalf("python {0}: %s", s.Path)
	224	+	}
	225	+	bin2 := fakeBin(t, "perl")
	226	+	plan, _ = PlanShell("perl {0}", Lookup{Path: bin2})
	227	+	if s, _ := WriteScript(root, plan, "x"); filepath.Ext(s.Path) != "" {
	228	+	t.Fatalf("perl: %s", s.Path)
	229	+	}
	230	+	}
	231	+
	232	+	func TestResolveWorkingDirectory(t *testing.T) {
	233	+	for _, tc := range [][3]string{
	234	+	{"/w/r/r", "", "/w/r/r"},
	235	+	{"/w/r/r", "sub", "/w/r/r/sub"},
	236	+	{"/w/r/r", "a/b", "/w/r/r/a/b"},
	237	+	{"/w/r/r", "/tmp", "/tmp"},
	238	+	{"/w/r/r", "..", "/w/r/r/.."},
	239	+	} {
	240	+	if got := ResolveWorkingDirectory(tc[0], tc[1]); got != tc[2] {
	241	+	t.Errorf("%v: %q", tc, got)
	242	+	}
	243	+	}
	244	+	}
	245	+
	246	+	// The script goes through the os.Root of RUNNER_TEMP: a directory swapped for a
	247	+	// link to elsewhere after the root was opened does not redirect it (GO-002).
	248	+	func TestWriteScriptStaysInTheOpenedDirectory(t *testing.T) {
	249	+	parent := t.TempDir()
	250	+	dir := filepath.Join(parent, "tmp")
	251	+	if err := os.Mkdir(dir, 0o755); err != nil {
	252	+	t.Fatal(err)
	253	+	}
	254	+	root, err := os.OpenRoot(dir)
	255	+	if err != nil {
	256	+	t.Fatal(err)
	257	+	}
	258	+	defer func() { _ = root.Close() }()
	259	+	outside := filepath.Join(parent, "outside")
	260	+	if err := os.Mkdir(outside, 0o755); err != nil {
	261	+	t.Fatal(err)
	262	+	}
	263	+	// The step swaps the directory for a link to another one.
	264	+	if err := os.Rename(dir, dir+".moved"); err != nil {
	265	+	t.Fatal(err)
	266	+	}
	267	+	if err := os.Symlink(outside, dir); err != nil {
	268	+	t.Fatal(err)
	269	+	}
	270	+	bin := fakeBin(t, "bash", "sh")
	271	+	plan, _ := PlanShell("", Lookup{Path: bin})
	272	+	if _, err := WriteScript(root, plan, "echo hi\n"); err != nil {
	273	+	t.Fatal(err)
	274	+	}
	275	+	if ents, _ := os.ReadDir(outside); len(ents) != 0 {
	276	+	t.Fatalf("the script was written through the swapped link: %v", ents)
	277	+	}
	278	+	if ents, _ := os.ReadDir(dir + ".moved"); len(ents) != 1 {
	279	+	t.Fatalf("the script is not in the directory that root was opened on: %v", ents)
	280	+	}
	281	+	}

A internal/executor/step.go +275 -0

@@ -0,0 +1,275 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package executor
	4	+
	5	+	import (
	6	+	"context"
	7	+	"fmt"
	8	+	"strings"
	9	+	"sync"
	10	+
	11	+	"bnema.dev/ermite-actions/template"
	12	+	"bnema.dev/ermite-actions/workflow"
	13	+	)
	14	+
	15	+	// ---------------------------------------------------------------- steps
	16	+
	17	+	// stepRun is the state of one step being run; it implements Sink and
	18	+	// commandHost.
	19	+	type stepRun struct {
	20	+	j *job
	21	+	num int
	22	+	st *workflow.Step
	23	+	env *ordered // the step's environment: job env + step env
	24	+	cmdResult *stepResult
	25	+	outcome *stepResult
	26	+	cmds *commandManager
	27	+	}
	28	+
	29	+	func (s *stepRun) Out(text string) { s.j.out(s.num, text) }
	30	+	func (s *stepRun) Tagged(tag, msg string) { s.j.tagged(s.num, tag, msg) }
	31	+	func (s *stepRun) Debug(msg string) { s.j.debug(s.num, msg) }
	32	+	func (s *stepRun) Error(msg string) { s.j.errorf(s.num, "%s", msg) }
	33	+	func (s *stepRun) CommandFailed() { f := resFailed; s.cmdResult = &f }
	34	+	func (s *stepRun) SetOutput(name, value string) {
	35	+	s.j.mu.Lock()
	36	+	s.j.state.SetOutput(s.st.ID, name, value)
	37	+	s.j.mu.Unlock()
	38	+	}
	39	+	func (s *stepRun) SaveState(name, value string) {}
	40	+	func (s *stepRun) SetEnv(name, value string) {
	41	+	s.j.mu.Lock()
	42	+	s.j.state.SetEnv(name, value)
	43	+	s.j.mu.Unlock()
	44	+	}
	45	+	func (s *stepRun) AddPath(p string) {
	46	+	s.j.mu.Lock()
	47	+	s.j.state.AddPath(p)
	48	+	s.j.mu.Unlock()
	49	+	}
	50	+
	51	+	// EnvFlag reads a boolean variable of the env context, then of the process
	52	+	// (ActionCommandManager.cs:236-246).
	53	+	func (s *stepRun) EnvFlag(name string) bool {
	54	+	v := strings.ToLower(envValue(s.j.baseEnv, name))
	55	+	if v != "true" {
	56	+	if e, ok := s.env.Get(name); ok {
	57	+	v = strings.ToLower(e)
	58	+	}
	59	+	}
	60	+	return v == "true"
	61	+	}
	62	+
	63	+	func firstLine(contents string) string {
	64	+	l := strings.TrimLeft(contents, " \t\r\n")
	65	+	if i := strings.IndexAny(l, "\r\n"); i >= 0 {
	66	+	l = l[:i]
	67	+	}
	68	+	return l
	69	+	}
	70	+
	71	+	// displayName is the name shown for the step (ActionRunner.GenerateDisplayName,
	72	+	// ActionRunner.cs:363): the evaluated name, or "Run " and the first line of
	73	+	// the script or the action reference.
	74	+	func (j job) displayName(st workflow.Step, env *ordered) string {
	75	+	tok := st.Name
	76	+	prefix := ""
	77	+	switch {
	78	+	case tok != nil:
	79	+	case st.Run != nil:
	80	+	tok, prefix = st.Run, "Run "
	81	+	case st.Uses != nil:
	82	+	tok, prefix = st.Uses, "Run "
	83	+	default:
	84	+	return ""
	85	+	}
	86	+	name, err := j.evalString(tok, env)
	87	+	if err != nil {
	88	+	name = tokenSource(tok)
	89	+	}
	90	+	return formatStepName(prefix, name)
	91	+	}
	92	+
	93	+	func tokenSource(t template.Token) string {
	94	+	if be, ok := t.(*template.BasicExpression); ok && be.Source != "" {
	95	+	return be.Source
	96	+	}
	97	+	return t.String()
	98	+	}
	99	+
	100	+	// formatStepName is ActionRunner.FormatStepName (ActionRunner.cs:447).
	101	+	func formatStepName(prefix, name string) string {
	102	+	if name == "" {
	103	+	return ""
	104	+	}
	105	+	return prefix + firstLine(name)
	106	+	}
	107	+
	108	+	func (j *job) statusFuncs() workflow.StatusFuncs {
	109	+	st := j.state.Status()
	110	+	return workflow.StatusFuncs{Success: st == "success", Failure: st == "failure", Cancelled: st == "cancelled"}
	111	+	}
	112	+
	113	+	// markCancelled records the cancellation of the job: job result Canceled and
	114	+	// job.status cancelled (StepsRunner.cs:152-155 and :190-196).
	115	+	func (j *job) markCancelledLocked() {
	116	+	if j.cancelled {
	117	+	return
	118	+	}
	119	+	j.cancelled = true
	120	+	c := resCanceled
	121	+	j.result = &c
	122	+	j.state.SetStatus("cancelled")
	123	+	}
	124	+
	125	+	func (j job) runStep(num int, st workflow.Step) {
	126	+	rep := j.cfg.Reporter
	127	+	s := &stepRun{j: j, num: num, st: st}
	128	+	s.cmds = newCommandManager(s, j.cfg.Debug)
	129	+
	130	+	// Environment of the step (StepsRunner.cs:108-139): job env, then the step's
	131	+	// own, evaluated against the job env.
	132	+	j.mu.Lock()
	133	+	jobEnv := j.state.JobEnv()
	134	+	j.mu.Unlock()
	135	+	s.env = jobEnv.Clone()
	136	+	pairs, envErr := j.evalMap(st.Env, jobEnv)
	137	+	if envErr == nil {
	138	+	for _, p := range pairs {
	139	+	s.env.Set(p.Key, p.Value)
	140	+	}
	141	+	}
	142	+	name := j.displayName(st, s.env)
	143	+	rep.StepStarted(num, name, j.now())
	144	+
	145	+	complete := func(r stepResult) {
	146	+	res := r
	147	+	j.mu.Lock()
	148	+	// Update the steps context (ExecutionContext.UpdateGlobalStepsContext).
	149	+	outcome := res
	150	+	if s.outcome != nil {
	151	+	outcome = *s.outcome
	152	+	}
	153	+	j.state.SetStepResult(st.ID, string(outcome.conclusion()), string(res.conclusion()))
	154	+	if res == resFailed {
	155	+	m := merge(j.result, resFailed)
	156	+	j.result = &m
	157	+	j.state.SetStatus(statusName(m))
	158	+	}
	159	+	j.mu.Unlock()
	160	+	rep.StepCompleted(num, name, res.conclusion(), j.now())
	161	+	}
	162	+
	163	+	if envErr != nil {
	164	+	j.errorf(num, "%s", envErr.Error())
	165	+	complete(resFailed)
	166	+	return
	167	+	}
	168	+
	169	+	// Cancellation seen before the step starts (StepsRunner.cs:188-197).
	170	+	cancelledAtStart := false
	171	+	select {
	172	+	case <-j.cancel.Done():
	173	+	cancelledAtStart = true
	174	+	j.mu.Lock()
	175	+	j.markCancelledLocked()
	176	+	j.mu.Unlock()
	177	+	default:
	178	+	}
	179	+
	180	+	// Condition (StepsRunner.cs:200-261). On runner shutdown conditions are not
	181	+	// evaluated and the step is skipped.
	182	+	cond := func() (bool, error) {
	183	+	j.mu.Lock()
	184	+	defer j.mu.Unlock()
	185	+	return st.EvaluateIf(context.Background(), j.evalEnv(s.env), j.statusFuncs())
	186	+	}
	187	+	if cancelledAtStart && j.cancel.Reason() == CancelShutdown {
	188	+	complete(resSkipped)
	189	+	return
	190	+	}
	191	+	ok, err := cond()
	192	+	switch {
	193	+	case err != nil:
	194	+	j.errorf(num, "%s", err.Error())
	195	+	complete(resFailed)
	196	+	return
	197	+	case !ok:
	198	+	complete(resSkipped)
	199	+	return
	200	+	}
	201	+
	202	+	// Cancellation while the step runs (StepsRunner.cs:149-186): the job becomes
	203	+	// cancelled and the step is stopped unless its condition still holds
	204	+	// (always()).
	205	+	stepCancel := make(chan struct{})
	206	+	stopWatch := make(chan struct{})
	207	+	var watch sync.WaitGroup
	208	+	if !cancelledAtStart {
	209	+	watch.Add(1)
	210	+	go func() {
	211	+	defer watch.Done()
	212	+	select {
	213	+	case <-j.cancel.Done():
	214	+	case <-stopWatch:
	215	+	return
	216	+	}
	217	+	j.mu.Lock()
	218	+	j.markCancelledLocked()
	219	+	j.mu.Unlock()
	220	+	keep := false
	221	+	if j.cancel.Reason() != CancelShutdown {
	222	+	keep, _ = cond()
	223	+	}
	224	+	if !keep {
	225	+	close(stepCancel)
	226	+	}
	227	+	}()
	228	+	}
	229	+
	230	+	r := j.execute(s, stepCancel)
	231	+	close(stopWatch)
	232	+	watch.Wait()
	233	+
	234	+	// Merge the result of the commands (StepsRunner.cs:347-351).
	235	+	if s.cmdResult != nil {
	236	+	m := merge(&r, *s.cmdResult)
	237	+	r = m
	238	+	}
	239	+	// continue-on-error (ExecutionContext.ApplyContinueOnError, :1422).
	240	+	if r == resFailed && st.ContinueOnError != nil {
	241	+	coe, cerr := template.EvaluateBool(context.Background(), st.ContinueOnError, j.evalEnv(s.env))
	242	+	if cerr != nil {
	243	+	j.errorf(num, "The step failed and an error occurred when attempting to determine whether to continue on error.")
	244	+	j.errorf(num, "%s", cerr.Error())
	245	+	}
	246	+	if coe && cerr == nil {
	247	+	o := resFailed
	248	+	s.outcome = &o
	249	+	r = resSucceeded
	250	+	}
	251	+	}
	252	+	complete(r)
	253	+	}
	254	+
	255	+	// statusName is the job.status for a result (TaskResult.ToActionResult).
	256	+	func statusName(r stepResult) string {
	257	+	switch r {
	258	+	case resFailed:
	259	+	return "failure"
	260	+	case resCanceled:
	261	+	return "cancelled"
	262	+	}
	263	+	return "success"
	264	+	}
	265	+
	266	+	// notSupported is the failure of a step that uses a feature of a later phase.
	267	+	func notSupported(st *workflow.Step) string {
	268	+	switch {
	269	+	case st.Uses != nil:
	270	+	return fmt.Sprintf("not supported yet: uses (%s)", st.Uses.Value)
	271	+	case st.TimeoutMinutes != nil:
	272	+	return "not supported yet: timeout-minutes"
	273	+	}
	274	+	return ""
	275	+	}

M internal/guest/exec.go +54 -21


@@ -38,6 +38,13 @@ type ExecConfig struct {
38	38		Version string
39	39		// Shell runs the script as `Shell -c script`; default /bin/sh.
40	40		Shell string
	41	+	// StepUser is the account the steps of a job run as when the executor is
	42	+	// root; default "runner".
	43	+	StepUser string
	44	+	// AllowRootSteps lets the steps run as root when StepUser does not exist.
	45	+	// For test images only; set by ERMITE_RUNNER_ALLOW_ROOT_STEPS=1 in the
	46	+	// executor's environment.
	47	+	AllowRootSteps bool
41	48		// ConnectTimeout bounds the connection attempts (default 30 s).
42	49		ConnectTimeout time.Duration
43	50		// RunTimeout bounds the wait for the host's Run frame (default 60 s).

@@ -57,6 +64,9 @@ func (c *ExecConfig) defaults() {
57	64		if c.Shell == "" {
58	65		c.Shell = "/bin/sh"
59	66		}
	67	+	if c.StepUser == "" {
	68	+	c.StepUser = "runner"
	69	+	}
60	70		if c.ConnectTimeout == 0 {
61	71		c.ConnectTimeout = 30 * time.Second
62	72		}

@@ -77,8 +87,9 @@ func (c *ExecConfig) defaults() {
77	87		}
78	88		}
79	89
80		-	// Exec is `ermite-runner exec`, the CI-1 stub executor: it connects back to the
81		-	// host, receives one Run, runs the script with the shell, streams stdout and
	90	+	// Exec is `ermite-runner exec`: it connects back to the host and receives one
	91	+	// Run. A Run with a job is executed by package executor (see runJob); a Run
	92	+	// without one is the stub of sandbox-check, which runs the script with the shell, streams stdout and
82	93		// stderr as Output frames, and ends with an Exit frame. It returns the exit code
83	94		// it reported. A Cancel frame, or the end of ctx, stops the script's process
84	95		// group with escalating signals (see ExecConfig).

@@ -179,29 +190,47 @@ func serve(ctx context.Context, rw io.ReadWriteCloser, cfg ExecConfig) (int, err
179	190		}}}); err != nil {
180	191		return 1, fmt.Errorf("send hello: %w", err)
181	192		}
	193	+	run, err := waitForRun(ctx, conn, cfg.RunTimeout)
	194	+	if err != nil {
	195	+	return 1, err
	196	+	}
	197	+	if run.GetJob() != nil {
	198	+	return sendExit(conn, runJob(ctx, conn, cfg, run.GetJob()), nil)
	199	+	}
	200	+	return runStub(ctx, conn, cfg, run)
	201	+	}
182	202
183		-	// Wait for Run, bounded: a host that never answers must not pin the sandbox.
	203	+	// waitForRun waits for the Run frame of the host, bounded: a host that never
	204	+	// answers must not pin the sandbox.
	205	+	func waitForRun(ctx context.Context, conn execstream.Conn, timeout time.Duration) (execpb.Run, error) {
184	206		type recv struct {
185	207		f *execpb.Frame
186	208		err error
187	209		}
188	210		first := make(chan recv, 1)
189	211		go func() { f, err := conn.Recv(); first <- recv{f, err} }()
190		-	var run *execpb.Run
	212	+	timer := time.NewTimer(timeout)
	213	+	defer timer.Stop()
191	214		select {
192	215		case r := <-first:
193	216		if r.err != nil {
194		-	return 1, fmt.Errorf("waiting for run: %w", r.err)
	217	+	return nil, fmt.Errorf("waiting for run: %w", r.err)
195	218		}
196		-	if run = r.f.GetRun(); run == nil {
197		-	return 1, errors.New("first frame from host is not run")
	219	+	run := r.f.GetRun()
	220	+	if run == nil {
	221	+	return nil, errors.New("first frame from host is not run")
198	222		}
199		-	case <-time.After(cfg.RunTimeout):
200		-	return 1, errors.New("timed out waiting for the host's run frame")
	223	+	return run, nil
	224	+	case <-timer.C:
	225	+	return nil, errors.New("timed out waiting for the host's run frame")
201	226		case <-ctx.Done():
202		-	return 1, ctx.Err()
	227	+	return nil, ctx.Err()
203	228		}
	229	+	}
204	230
	231	+	// runStub is the stub executor of sandbox-check: it runs the script of a Run
	232	+	// without a job with the shell and streams its output.
	233	+	func runStub(ctx context.Context, conn execstream.Conn, cfg ExecConfig, run execpb.Run) (int, error) {
205	234		script := run.GetScript()
206	235		if script == "" {
207	236		script = DefaultScript

@@ -242,26 +271,30 @@ func serve(ctx context.Context, rw io.ReadWriteCloser, cfg ExecConfig) (int, err
242	271
243	272		waitErr := cmd.Wait()
244	273		esc.finish()
	274	+	code, err := exitCode(waitErr)
	275	+	return sendExit(conn, code, err)
	276	+	}
245	277
246		-	code := 0
	278	+	// exitCode is the exit code to report for the error of cmd.Wait: 0 for a clean
	279	+	// exit (also when a background child kept the pipes open past WaitDelay: the
	280	+	// group was killed by esc.finish), the code of the process (128+signal when it
	281	+	// was killed, like a shell), or 1 with the error for anything else.
	282	+	func exitCode(waitErr error) (int, error) {
247	283		var ee *exec.ExitError
248	284		switch {
249	285		case waitErr == nil, errors.Is(waitErr, exec.ErrWaitDelay):
250		-	// ErrWaitDelay: the shell exited cleanly but a background child kept the
251		-	// pipes open; the group was killed above.
	286	+	return 0, nil
252	287		case errors.As(waitErr, &ee):
253		-	code = ee.ExitCode()
254		-	if code < 0 { // killed by a signal: 128+signal like a shell
	288	+	code := ee.ExitCode()
	289	+	if code < 0 {
255	290		if ws, ok := ee.Sys().(syscall.WaitStatus); ok && ws.Signaled() {
256		-	code = 128 + int(ws.Signal())
257		-	} else {
258		-	code = 1
	291	+	return 128 + int(ws.Signal()), nil
259	292		}
	293	+	return 1, nil
260	294		}
261		-	default:
262		-	return sendExit(conn, 1, waitErr)
	295	+	return code, nil
263	296		}
264		-	return sendExit(conn, code, nil)
	297	+	return 1, waitErr
265	298		}
266	299
267	300		func sendExit(c *execstream.Conn, code int, cause error) (int, error) {

A internal/guest/jobexec.go +226 -0

@@ -0,0 +1,226 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package guest
	4	+
	5	+	import (
	6	+	"context"
	7	+	"fmt"
	8	+	"os"
	9	+	"os/user"
	10	+	"strconv"
	11	+	"strings"
	12	+	"syscall"
	13	+	"time"
	14	+
	15	+	"bnema.dev/ermite-runner/internal/execstream"
	16	+	"bnema.dev/ermite-runner/internal/execstream/execpb"
	17	+	"bnema.dev/ermite-runner/internal/executor"
	18	+	)
	19	+
	20	+	// frameReporter is the executor.Reporter that writes frames on the stream.
	21	+	type frameReporter struct {
	22	+	conn *execstream.Conn
	23	+	// err keeps the first send error; the job goes on without a host.
	24	+	failed chan struct{}
	25	+	}
	26	+
	27	+	func ts(t time.Time) int64 { return t.UnixNano() }
	28	+
	29	+	func conclusionProto(c executor.Conclusion) execpb.StepConclusion {
	30	+	switch c {
	31	+	case executor.ConclusionSuccess:
	32	+	return execpb.StepConclusion_STEP_CONCLUSION_SUCCESS
	33	+	case executor.ConclusionFailure:
	34	+	return execpb.StepConclusion_STEP_CONCLUSION_FAILURE
	35	+	case executor.ConclusionCancelled:
	36	+	return execpb.StepConclusion_STEP_CONCLUSION_CANCELLED
	37	+	case executor.ConclusionSkipped:
	38	+	return execpb.StepConclusion_STEP_CONCLUSION_SKIPPED
	39	+	case executor.ConclusionTimedOut:
	40	+	return execpb.StepConclusion_STEP_CONCLUSION_TIMED_OUT
	41	+	}
	42	+	return execpb.StepConclusion_STEP_CONCLUSION_UNSPECIFIED
	43	+	}
	44	+
	45	+	func (r frameReporter) send(f execpb.Frame) {
	46	+	if err := r.conn.Send(f); err != nil {
	47	+	select {
	48	+	case <-r.failed:
	49	+	default:
	50	+	close(r.failed)
	51	+	}
	52	+	}
	53	+	}
	54	+
	55	+	func (r *frameReporter) StepStarted(n int, name string, at time.Time) {
	56	+	r.send(&execpb.Frame{Body: &execpb.Frame_StepStarted{StepStarted: &execpb.StepStarted{Number: int32(n), Name: name, UnixNano: ts(at)}}})
	57	+	}
	58	+
	59	+	func (r *frameReporter) Log(n int, at time.Time, line string) {
	60	+	for i, piece := range execstream.SplitLine(line, execstream.MaxLogLine) {
	61	+	r.send(&execpb.Frame{Body: &execpb.Frame_Log{Log: &execpb.Log{Step: int32(n), UnixNano: ts(at), Text: piece, Continued: i > 0}}})
	62	+	}
	63	+	}
	64	+
	65	+	func (r *frameReporter) StepCompleted(n int, name string, c executor.Conclusion, at time.Time) {
	66	+	r.send(&execpb.Frame{Body: &execpb.Frame_StepCompleted{StepCompleted: &execpb.StepCompleted{Number: int32(n), Name: name, Conclusion: conclusionProto(c), UnixNano: ts(at)}}})
	67	+	}
	68	+
	69	+	func (r *frameReporter) JobCompleted(res executor.Result) {
	70	+	r.send(&execpb.Frame{Body: &execpb.Frame_JobResult{JobResult: &execpb.JobResult{
	71	+	Conclusion: conclusionProto(res.Conclusion), Outputs: res.Outputs, Error: res.Error, RunnerShutdown: res.RunnerShutdown,
	72	+	}}})
	73	+	}
	74	+
	75	+	// StepUser is the account the steps run as: uid, gid and supplementary groups,
	76	+	// like the hosted runners' "runner" user. Nil credential means the steps run as
	77	+	// the executor itself.
	78	+	type StepUser struct {
	79	+	Credential *syscall.Credential
	80	+	Name, Home string
	81	+	}
	82	+
	83	+	// LookupStepUser finds the user the steps run as. When the executor is not
	84	+	// root it cannot switch user: the steps run as the current user (a development
	85	+	// or test setup, never the sandbox of a runner, whose init is root). When it is
	86	+	// root and the user does not exist, the lookup fails: running a job's code as
	87	+	// root of the sandbox, with the docker daemon and the whole image at hand,
	88	+	// because an image lacks the account would hide a broken image. A test image
	89	+	// opts in with allowRoot.
	90	+	func LookupStepUser(name string, allowRoot bool) (StepUser, error) {
	91	+	return lookupStepUser(os.Geteuid(), user.Lookup, name, allowRoot)
	92	+	}
	93	+
	94	+	func lookupStepUser(euid int, lookup func(string) (*user.User, error), name string, allowRoot bool) (StepUser, error) {
	95	+	if euid != 0 \|\| name == "" {
	96	+	u := StepUser{Home: os.Getenv("HOME")}
	97	+	if cu, err := user.Current(); err == nil {
	98	+	u.Name = cu.Username
	99	+	if u.Home == "" {
	100	+	u.Home = cu.HomeDir
	101	+	}
	102	+	}
	103	+	return u, nil
	104	+	}
	105	+	u, err := lookup(name)
	106	+	if err != nil {
	107	+	if allowRoot {
	108	+	return StepUser{Name: "root", Home: "/root"}, nil
	109	+	}
	110	+	return StepUser{}, fmt.Errorf("the image has no user %q to run the steps as (%v): refusing to run them as root", name, err)
	111	+	}
	112	+	uid, _ := strconv.ParseUint(u.Uid, 10, 32)
	113	+	gid, _ := strconv.ParseUint(u.Gid, 10, 32)
	114	+	cred := &syscall.Credential{Uid: uint32(uid), Gid: uint32(gid)}
	115	+	if ids, err := u.GroupIds(); err == nil {
	116	+	for _, g := range ids {
	117	+	if n, err := strconv.ParseUint(g, 10, 32); err == nil {
	118	+	cred.Groups = append(cred.Groups, uint32(n))
	119	+	}
	120	+	}
	121	+	}
	122	+	return StepUser{Credential: cred, Name: u.Username, Home: u.HomeDir}, nil
	123	+	}
	124	+
	125	+	// stepEnv is the environment the steps inherit: the executor's, without the
	126	+	// variables of the stream, with HOME, USER and LOGNAME of the step user.
	127	+	func stepEnv(environ []string, u StepUser) []string {
	128	+	var out []string
	129	+	for _, kv := range environ {
	130	+	k, _, _ := strings.Cut(kv, "=")
	131	+	switch {
	132	+	case strings.HasPrefix(k, "ERMITE_"), k == "RUNNER_TOOL_CACHE", k == "HOME", k == "USER", k == "LOGNAME":
	133	+	continue
	134	+	}
	135	+	out = append(out, kv)
	136	+	}
	137	+	if u.Home != "" {
	138	+	out = append(out, "HOME="+u.Home)
	139	+	}
	140	+	if u.Name != "" {
	141	+	out = append(out, "USER="+u.Name, "LOGNAME="+u.Name)
	142	+	}
	143	+	return out
	144	+	}
	145	+
	146	+	func cancelKind(k execpb.CancelKind) executor.CancelReason {
	147	+	switch k {
	148	+	case execpb.CancelKind_CANCEL_KIND_TIMEOUT:
	149	+	return executor.CancelTimeout
	150	+	case execpb.CancelKind_CANCEL_KIND_SHUTDOWN:
	151	+	return executor.CancelShutdown
	152	+	}
	153	+	return executor.CancelUser
	154	+	}
	155	+
	156	+	// runJob runs the job of a Run frame with the executor and reports on conn.
	157	+	// It returns the exit code to report.
	158	+	func runJob(ctx context.Context, conn execstream.Conn, cfg ExecConfig, spec execpb.JobSpec) int {
	159	+	su, err := LookupStepUser(cfg.StepUser, cfg.AllowRootSteps)
	160	+	if err != nil {
	161	+	// Reported like any failure of "Set up job": the job fails, with the reason
	162	+	// in its log, and nothing of the workflow runs.
	163	+	return failSetup(conn, err.Error())
	164	+	}
	165	+	canceller := executor.NewCanceller()
	166	+	rep := &frameReporter{conn: conn, failed: make(chan struct{})}
	167	+
	168	+	// Frames from the host while the job runs: a Cancel stops the job; a lost
	169	+	// host does too, like a runner shutdown. The reader ends with the stream: the
	170	+	// caller closes it after runJob, and the goroutine that reports a failed
	171	+	// send ends with the job (done).
	172	+	done := make(chan struct{})
	173	+	defer close(done)
	174	+	go func() {
	175	+	for {
	176	+	f, err := conn.Recv()
	177	+	if err != nil {
	178	+	canceller.Cancel(executor.CancelShutdown)
	179	+	return
	180	+	}
	181	+	if c := f.GetCancel(); c != nil {
	182	+	canceller.Cancel(cancelKind(c.GetKind()))
	183	+	}
	184	+	}
	185	+	}()
	186	+	// The executor's own context (SIGTERM forwarded by init: the sandbox is
	187	+	// being stopped) is a shutdown.
	188	+	stop := context.AfterFunc(ctx, func() { canceller.Cancel(executor.CancelShutdown) })
	189	+	defer stop()
	190	+	go func() {
	191	+	select {
	192	+	case <-rep.failed:
	193	+	canceller.Cancel(executor.CancelShutdown)
	194	+	case <-canceller.Done():
	195	+	case <-done:
	196	+	}
	197	+	}()
	198	+
	199	+	env := spec.GetRunnerEnvironment()
	200	+	res := executor.Run(canceller, executor.Config{
	201	+	WorkflowPath: spec.GetWorkflowPath(), WorkflowYAML: []byte(spec.GetWorkflowYaml()), JobKey: spec.GetJobKey(),
	202	+	ContextsJSON: spec.GetContexts(), MatrixJSON: spec.GetMatrix(), Repo: spec.GetRepo(),
	203	+	TimeoutMinutes: int(spec.GetTimeoutMinutes()), Deadline: time.Duration(spec.GetDeadlineMs()) * time.Millisecond,
	204	+	Runner: executor.RunnerInfo{Name: spec.GetRunnerName(), Environment: env},
	205	+	RunnerVersion: spec.GetRunnerVersion(),
	206	+	Home: su.Home, ToolCache: os.Getenv("RUNNER_TOOL_CACHE"), BaseEnv: stepEnv(os.Environ(), su), Credential: su.Credential,
	207	+	Debug: spec.GetDebug(), Reporter: rep,
	208	+	TermAfter: cfg.TermAfter, KillAfter: cfg.KillAfter, WaitDelay: cfg.WaitDelay,
	209	+	})
	210	+	if res.Conclusion == executor.ConclusionSuccess {
	211	+	return 0
	212	+	}
	213	+	return 1
	214	+	}
	215	+
	216	+	// failSetup ends a job that cannot start, like a failed "Set up job": step 1
	217	+	// starts, logs the reason as an error, and fails; the job result is a failure.
	218	+	func failSetup(conn *execstream.Conn, msg string) int {
	219	+	rep := &frameReporter{conn: conn, failed: make(chan struct{})}
	220	+	now := time.Now()
	221	+	rep.StepStarted(1, executor.SetUpJobName, now)
	222	+	rep.Log(1, now, "##[error]"+msg)
	223	+	rep.StepCompleted(1, executor.SetUpJobName, executor.ConclusionFailure, time.Now())
	224	+	rep.JobCompleted(executor.Result{Conclusion: executor.ConclusionFailure, Error: msg})
	225	+	return 1
	226	+	}

A internal/guest/jobexec_test.go +259 -0

@@ -0,0 +1,259 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package guest
	4	+
	5	+	import (
	6	+	"context"
	7	+	"net"
	8	+	"os/exec"
	9	+	"os/user"
	10	+	"path/filepath"
	11	+	"strings"
	12	+	"sync"
	13	+	"testing"
	14	+	"time"
	15	+
	16	+	"bnema.dev/ermite-runner/internal/execstream"
	17	+	"bnema.dev/ermite-runner/internal/execstream/execpb"
	18	+	)
	19	+
	20	+	// runJobSession starts Exec on a unix socket and drives it with a job.
	21	+	func runJobSession(t testing.T, yaml string, cancelOn string, tweak func(ExecConfig)) (steps []string, logs []execpb.Log, res execpb.JobResult, exit *execpb.Exit) {
	22	+	t.Helper()
	23	+	sock := filepath.Join(t.TempDir(), "s")
	24	+	ln, err := net.Listen("unix", sock)
	25	+	if err != nil {
	26	+	t.Skip(err)
	27	+	}
	28	+	defer func() { _ = ln.Close() }()
	29	+	home := t.TempDir()
	30	+	t.Setenv("HOME", home)
	31	+	t.Setenv("RUNNER_TOOL_CACHE", filepath.Join(home, "toolcache"))
	32	+	t.Setenv("ERMITE_CONNECT", "unix:"+sock)
	33	+	done := make(chan struct{})
	34	+	go func() {
	35	+	defer close(done)
	36	+	cfg := ExecConfig{Target: Target{"unix", sock}, Sandbox: "container", Version: "test", RunTimeout: 5 * time.Second,
	37	+	TermAfter: 200 * time.Millisecond, KillAfter: 200 * time.Millisecond, WaitDelay: 300 * time.Millisecond}
	38	+	if tweak != nil {
	39	+	tweak(&cfg)
	40	+	}
	41	+	_, _ = Exec(context.Background(), cfg)
	42	+	}()
	43	+	conn, err := ln.Accept()
	44	+	if err != nil {
	45	+	t.Fatal(err)
	46	+	}
	47	+	defer func() { _ = conn.Close() }()
	48	+	// With cancelOn, the job is cancelled when it logs that line (the script
	49	+	// prints it once it is ready to be interrupted): no timer to guess.
	50	+	var cancel chan *execpb.Cancel
	51	+	if cancelOn != "" {
	52	+	cancel = make(chan *execpb.Cancel, 1)
	53	+	}
	54	+	var cancelOnce sync.Once
	55	+	var mu sync.Mutex
	56	+	_, exit, err = execstream.DriveJob(conn, &execpb.Run{Job: &execpb.JobSpec{
	57	+	WorkflowPath: ".github/workflows/ci.yml", WorkflowYaml: yaml, JobKey: "j", Repo: "hello", RunnerName: "r", RunnerVersion: "v",
	58	+	Contexts: `{"github":{"repository":"octo/hello","event_name":"push","job":"j"}}`, Matrix: "{}",
	59	+	}}, cancel, execstream.JobHandler{
	60	+	StepStarted: func(s *execpb.StepStarted) {
	61	+	mu.Lock()
	62	+	steps = append(steps, s.Name)
	63	+	mu.Unlock()
	64	+	},
	65	+	Log: func(l *execpb.Log) {
	66	+	logs = append(logs, l)
	67	+	if cancelOn != "" && l.Text == cancelOn {
	68	+	cancelOnce.Do(func() { cancel <- &execpb.Cancel{Reason: "test", Kind: execpb.CancelKind_CANCEL_KIND_USER} })
	69	+	}
	70	+	},
	71	+	JobResult: func(r *execpb.JobResult) { res = r },
	72	+	})
	73	+	if err != nil {
	74	+	t.Fatal(err)
	75	+	}
	76	+	<-done
	77	+	return
	78	+	}
	79	+
	80	+	func requireBash(t *testing.T) {
	81	+	t.Helper()
	82	+	if _, err := lookPath("bash"); err != nil {
	83	+	t.Skip("bash not installed")
	84	+	}
	85	+	}
	86	+
	87	+	func TestExecRunsAJob(t *testing.T) {
	88	+	requireBash(t)
	89	+	steps, logs, res, exit := runJobSession(t, "on: push\njobs:\n j:\n runs-on: x\n outputs:\n o: ${{ steps.s.outputs.v }}\n steps:\n - id: s\n run: \|\n echo hello-from-job\n echo v=42 >> \"$GITHUB_OUTPUT\"\n", "", nil)
	90	+	if res == nil \|\| res.Conclusion != execpb.StepConclusion_STEP_CONCLUSION_SUCCESS \|\| res.Outputs["o"] != "42" \|\| exit.Code != 0 {
	91	+	t.Fatalf("res=%v exit=%v", res, exit)
	92	+	}
	93	+	if strings.Join(steps, "\|") != "Set up job\|Run echo hello-from-job\|Complete job" {
	94	+	t.Fatalf("steps %v", steps)
	95	+	}
	96	+	found := false
	97	+	for _, l := range logs {
	98	+	if l.Text == "hello-from-job" && l.Step == 2 && l.UnixNano > 0 && !l.Continued {
	99	+	found = true
	100	+	}
	101	+	}
	102	+	if !found {
	103	+	t.Fatalf("log line missing: %d lines", len(logs))
	104	+	}
	105	+	}
	106	+
	107	+	func TestExecJobFailureExitsNonZero(t *testing.T) {
	108	+	requireBash(t)
	109	+	_, _, res, exit := runJobSession(t, "on: push\njobs:\n j:\n runs-on: x\n steps:\n - run: exit 4\n", "", nil)
	110	+	if res.Conclusion != execpb.StepConclusion_STEP_CONCLUSION_FAILURE \|\| exit.Code == 0 {
	111	+	t.Fatalf("res=%v exit=%v", res, exit)
	112	+	}
	113	+	}
	114	+
	115	+	func TestExecJobLongLinesAreContinued(t *testing.T) {
	116	+	requireBash(t)
	117	+	_, logs, res, _ := runJobSession(t, "on: push\njobs:\n j:\n runs-on: x\n steps:\n - run: head -c 150000 /dev/zero \| tr '\\0' y; echo\n", "", nil)
	118	+	if res.Conclusion != execpb.StepConclusion_STEP_CONCLUSION_SUCCESS {
	119	+	t.Fatalf("%v", res)
	120	+	}
	121	+	var total int
	122	+	var pieces int
	123	+	for _, l := range logs {
	124	+	if strings.HasPrefix(l.Text, "yyyy") {
	125	+	pieces++
	126	+	total += len(l.Text)
	127	+	if len(l.Text) > execstream.MaxLogLine \|\| (pieces > 1) != l.Continued {
	128	+	t.Fatalf("piece %d: len %d continued %v", pieces, len(l.Text), l.Continued)
	129	+	}
	130	+	}
	131	+	}
	132	+	if pieces != 3 \|\| total != 150000 {
	133	+	t.Fatalf("pieces=%d total=%d", pieces, total)
	134	+	}
	135	+	}
	136	+
	137	+	func TestExecJobCancel(t *testing.T) {
	138	+	requireBash(t)
	139	+	start := time.Now()
	140	+	_, _, res, _ := runJobSession(t, "on: push\njobs:\n j:\n runs-on: x\n steps:\n - run: \|\n echo ready\n sleep 30\n", "ready", nil)
	141	+	if res.Conclusion != execpb.StepConclusion_STEP_CONCLUSION_CANCELLED \|\| res.RunnerShutdown {
	142	+	t.Fatalf("%v", res)
	143	+	}
	144	+	if time.Since(start) > 8*time.Second {
	145	+	t.Fatal("cancel too slow")
	146	+	}
	147	+	}
	148	+
	149	+	func TestExecJobHostGoneIsAShutdown(t *testing.T) {
	150	+	requireBash(t)
	151	+	sock := filepath.Join(t.TempDir(), "s")
	152	+	ln, err := net.Listen("unix", sock)
	153	+	if err != nil {
	154	+	t.Skip(err)
	155	+	}
	156	+	defer func() { _ = ln.Close() }()
	157	+	t.Setenv("HOME", t.TempDir())
	158	+	t.Setenv("RUNNER_TOOL_CACHE", filepath.Join(t.TempDir(), "toolcache"))
	159	+	done := make(chan int, 1)
	160	+	go func() {
	161	+	c, _ := Exec(context.Background(), ExecConfig{Target: Target{"unix", sock}, RunTimeout: 5 * time.Second,
	162	+	TermAfter: 100 * time.Millisecond, KillAfter: 100 * time.Millisecond, WaitDelay: 200 * time.Millisecond})
	163	+	done <- c
	164	+	}()
	165	+	conn, _ := ln.Accept()
	166	+	c := execstream.NewConn(conn)
	167	+	if _, err := c.Recv(); err != nil {
	168	+	t.Fatal(err)
	169	+	}
	170	+	_ = c.Send(&execpb.Frame{Body: &execpb.Frame_Run{Run: &execpb.Run{Job: &execpb.JobSpec{
	171	+	WorkflowYaml: "on: push\njobs:\n j:\n runs-on: x\n steps:\n - run: echo running; sleep 60\n", JobKey: "j", Repo: "hello",
	172	+	Contexts: `{"github":{"repository":"o/hello"}}`,
	173	+	}}}})
	174	+	// Close the stream once the step is running (its first line arrived).
	175	+	for {
	176	+	f, err := c.Recv()
	177	+	if err != nil {
	178	+	t.Fatal(err)
	179	+	}
	180	+	if l := f.GetLog(); l != nil && l.Text == "running" {
	181	+	break
	182	+	}
	183	+	}
	184	+	_ = conn.Close()
	185	+	select {
	186	+	case <-done:
	187	+	case <-time.After(10 * time.Second):
	188	+	t.Fatal("the executor kept running without a host")
	189	+	}
	190	+	}
	191	+
	192	+	func TestStepEnv(t *testing.T) {
	193	+	got := stepEnv([]string{"PATH=/bin", "HOME=/root", "ERMITE_CONNECT=x", "USER=root", "RUNNER_TOOL_CACHE=/x", "FOO=1"}, StepUser{Name: "runner", Home: "/home/runner"})
	194	+	want := "PATH=/bin FOO=1 HOME=/home/runner USER=runner LOGNAME=runner"
	195	+	if strings.Join(got, " ") != want {
	196	+	t.Fatalf("%v", got)
	197	+	}
	198	+	}
	199	+
	200	+	func lookPath(name string) (string, error) { return exec.LookPath(name) }
	201	+
	202	+	func TestMissingStepUserFailsTheJobUnlessTestOptionSet(t *testing.T) {
	203	+	none := func(string) (*user.User, error) { return nil, user.UnknownUserError("runner") }
	204	+	// As root with no such user: refuse.
	205	+	if _, err := lookupStepUser(0, none, "runner", false); err == nil \|\| !strings.Contains(err.Error(), `no user "runner"`) \|\| !strings.Contains(err.Error(), "refusing to run them as root") {
	206	+	t.Fatalf("%v", err)
	207	+	}
	208	+	// The test option allows root.
	209	+	if su, err := lookupStepUser(0, none, "runner", true); err != nil \|\| su.Name != "root" \|\| su.Credential != nil {
	210	+	t.Fatalf("%+v %v", su, err)
	211	+	}
	212	+	// Not root: the steps run as the current user, whatever the name.
	213	+	if su, err := lookupStepUser(1000, none, "runner", false); err != nil \|\| su.Credential != nil {
	214	+	t.Fatalf("%+v %v", su, err)
	215	+	}
	216	+	// As root with the user: switch to it.
	217	+	found := func(string) (*user.User, error) {
	218	+	return &user.User{Uid: "1001", Gid: "1001", Username: "runner", HomeDir: "/home/runner"}, nil
	219	+	}
	220	+	su, err := lookupStepUser(0, found, "runner", false)
	221	+	if err != nil \|\| su.Credential == nil \|\| su.Credential.Uid != 1001 \|\| su.Credential.Gid != 1001 \|\| su.Home != "/home/runner" {
	222	+	t.Fatalf("%+v %v", su, err)
	223	+	}
	224	+	}
	225	+
	226	+	// The failure is reported like a failed "Set up job".
	227	+	func TestFailSetupReportsAFailedFirstStep(t *testing.T) {
	228	+	host, guest := net.Pipe()
	229	+	got := make(chan []string, 1)
	230	+	go func() {
	231	+	var frames []string
	232	+	c := execstream.NewConn(host)
	233	+	for {
	234	+	f, err := c.Recv()
	235	+	if err != nil {
	236	+	got <- frames
	237	+	return
	238	+	}
	239	+	switch b := f.Body.(type) {
	240	+	case *execpb.Frame_StepStarted:
	241	+	frames = append(frames, "start:"+b.StepStarted.Name)
	242	+	case *execpb.Frame_Log:
	243	+	frames = append(frames, "log:"+b.Log.Text)
	244	+	case *execpb.Frame_StepCompleted:
	245	+	frames = append(frames, "done:"+b.StepCompleted.Conclusion.String())
	246	+	case *execpb.Frame_JobResult:
	247	+	frames = append(frames, "result:"+b.JobResult.Conclusion.String())
	248	+	}
	249	+	}
	250	+	}()
	251	+	if code := failSetup(execstream.NewConn(guest), "no user"); code != 1 {
	252	+	t.Fatal(code)
	253	+	}
	254	+	_ = guest.Close()
	255	+	want := "start:Set up job\|log:##[error]no user\|done:STEP_CONCLUSION_FAILURE\|result:STEP_CONCLUSION_FAILURE"
	256	+	if g := strings.Join(<-got, "\|"); g != want {
	257	+	t.Fatalf("%s", g)
	258	+	}
	259	+	}

A internal/host/config.go +220 -0

@@ -0,0 +1,220 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package host
	4	+
	5	+	import (
	6	+	"encoding/json"
	7	+	"errors"
	8	+	"fmt"
	9	+	"io"
	10	+	"net/http"
	11	+	"os"
	12	+	"path/filepath"
	13	+	"regexp"
	14	+	"strings"
	15	+
	16	+	"bnema.dev/ermite-api/gen/ermite/ermiteconnect"
	17	+	)
	18	+
	19	+	// Environment variables of the host process. The sandbox settings
	20	+	// (ERMITE_RUNNER_ISOLATION, _CAPACITY, _IMAGE, _JOB_*) are read by package
	21	+	// sandbox.
	22	+	const (
	23	+	EnvURL = "ERMITE_RUNNER_URL"
	24	+	EnvName = "ERMITE_RUNNER_NAME"
	25	+	EnvLabels = "ERMITE_RUNNER_LABELS"
	26	+	EnvRegisterToken = "ERMITE_RUNNER_REGISTER_TOKEN"
	27	+	// EnvInsecureHTTP allows a plain http server URL that is not loopback. The
	28	+	// runner token and the job tokens then travel in clear: tests only.
	29	+	EnvInsecureHTTP = "ERMITE_RUNNER_INSECURE_HTTP"
	30	+	)
	31	+
	32	+	// Config is the host part of the runner configuration.
	33	+	type Config struct {
	34	+	// URL is the base URL of the ermite server.
	35	+	URL string
	36	+	// Name is the runner name: 1 to 64 letters, digits, ".", "_" and "-".
	37	+	Name string
	38	+	// Labels are the extra labels of the runner.
	39	+	Labels []string
	40	+	// StateDir holds the runner token (mode 0600).
	41	+	StateDir string
	42	+	// Capacity is the number of jobs in parallel.
	43	+	Capacity int
	44	+	// Version is the version of the runner binary.
	45	+	Version string
	46	+	// InsecureHTTP allows a non-loopback http URL (ERMITE_RUNNER_INSECURE_HTTP=1).
	47	+	InsecureHTTP bool
	48	+	}
	49	+
	50	+	// Client returns the client for a token, honouring InsecureHTTP.
	51	+	func (c Config) Client(url, token string) ermiteconnect.RunnerServiceClient {
	52	+	if c.InsecureHTTP {
	53	+	return NewInsecureClient(url, token, http.DefaultTransport)
	54	+	}
	55	+	return NewClient(url, token, http.DefaultTransport)
	56	+	}
	57	+
	58	+	var nameRe = regexp.MustCompile(`^[A-Za-z0-9._-]{1,64}$`)
	59	+
	60	+	// LoadConfig reads the host settings. hostname is the default runner name.
	61	+	// stateDir and capacity come from the sandbox configuration, so that both
	62	+	// agree.
	63	+	func LoadConfig(getenv func(string) string, hostname, stateDir string, capacity int, version string) (Config, error) {
	64	+	c := Config{URL: strings.TrimSpace(getenv(EnvURL)), Name: strings.TrimSpace(getenv(EnvName)), StateDir: stateDir, Capacity: capacity, Version: version}
	65	+	if c.Name == "" {
	66	+	c.Name = sanitizeName(hostname)
	67	+	}
	68	+	switch strings.ToLower(strings.TrimSpace(getenv(EnvInsecureHTTP))) {
	69	+	case "", "0", "false", "no":
	70	+	case "1", "true", "yes":
	71	+	c.InsecureHTTP = true
	72	+	default:
	73	+	return c, fmt.Errorf("%s: %q is not 0 or 1", EnvInsecureHTTP, getenv(EnvInsecureHTTP))
	74	+	}
	75	+	if c.URL != "" {
	76	+	if err := CheckURL(c.URL, c.InsecureHTTP); err != nil {
	77	+	return c, fmt.Errorf("%s: %w", EnvURL, err)
	78	+	}
	79	+	}
	80	+	if !nameRe.MatchString(c.Name) {
	81	+	return c, fmt.Errorf("%s: %q is not a valid runner name (1 to 64 letters, digits, '.', '_' and '-')", EnvName, c.Name)
	82	+	}
	83	+	seen := map[string]bool{}
	84	+	for _, l := range strings.Split(getenv(EnvLabels), ",") {
	85	+	l = strings.TrimSpace(l)
	86	+	if l == "" {
	87	+	continue
	88	+	}
	89	+	if !nameRe.MatchString(l) {
	90	+	return c, fmt.Errorf("%s: %q is not a valid label (1 to 64 letters, digits, '.', '_' and '-')", EnvLabels, l)
	91	+	}
	92	+	if k := strings.ToLower(l); !seen[k] {
	93	+	seen[k] = true
	94	+	c.Labels = append(c.Labels, l)
	95	+	}
	96	+	}
	97	+	return c, nil
	98	+	}
	99	+
	100	+	func sanitizeName(s string) string {
	101	+	b := []byte(s)
	102	+	for i, ch := range b {
	103	+	if ok := ch >= 'a' && ch <= 'z' \|\| ch >= 'A' && ch <= 'Z' \|\| ch >= '0' && ch <= '9' \|\| ch == '.' \|\| ch == '_' \|\| ch == '-'; !ok {
	104	+	b[i] = '-'
	105	+	}
	106	+	}
	107	+	if len(b) > 64 {
	108	+	b = b[:64]
	109	+	}
	110	+	if len(b) == 0 {
	111	+	return "runner"
	112	+	}
	113	+	return string(b)
	114	+	}
	115	+
	116	+	// State is what register stores: the runner token and who it belongs to. The
	117	+	// file is mode 0600 and the token is never logged.
	118	+	type State struct {
	119	+	URL string `json:"url"`
	120	+	RunnerID string `json:"runner_id"`
	121	+	Name string `json:"name"`
	122	+	RunnerToken string `json:"runner_token"`
	123	+	}
	124	+
	125	+	// StateFile is the path of the state file in dir.
	126	+	func StateFile(dir string) string { return filepath.Join(dir, "runner.json") }
	127	+
	128	+	// ErrNotRegistered is returned by LoadState when no runner is registered.
	129	+	var ErrNotRegistered = errors.New("this runner is not registered: run `ermite-runner register` first")
	130	+
	131	+	// SaveState writes the state atomically with mode 0600.
	132	+	func SaveState(dir string, s State) error {
	133	+	if err := os.MkdirAll(dir, 0o700); err != nil {
	134	+	return err
	135	+	}
	136	+	// MkdirAll leaves an existing directory as it is; the directory holds the
	137	+	// runner token, so it is made private (the VM images live in it too, and
	138	+	// the runner runs as root: nobody else needs to enter).
	139	+	if err := os.Chmod(dir, 0o700); err != nil {
	140	+	return fmt.Errorf("make the state directory private: %w", err)
	141	+	}
	142	+	b, err := json.MarshalIndent(s, "", " ")
	143	+	if err != nil {
	144	+	return err
	145	+	}
	146	+	tmp, err := os.CreateTemp(dir, ".runner-*.json")
	147	+	if err != nil {
	148	+	return err
	149	+	}
	150	+	defer func() { _ = os.Remove(tmp.Name()) }()
	151	+	if err := tmp.Chmod(0o600); err != nil {
	152	+	_ = tmp.Close()
	153	+	return err
	154	+	}
	155	+	if _, err := tmp.Write(append(b, '\n')); err != nil {
	156	+	_ = tmp.Close()
	157	+	return err
	158	+	}
	159	+	if err := tmp.Sync(); err != nil {
	160	+	_ = tmp.Close()
	161	+	return err
	162	+	}
	163	+	if err := tmp.Close(); err != nil {
	164	+	return err
	165	+	}
	166	+	return os.Rename(tmp.Name(), StateFile(dir))
	167	+	}
	168	+
	169	+	// LoadState reads the state; it refuses a file readable by others, because the
	170	+	// runner token is a secret.
	171	+	func LoadState(dir string) (State, error) {
	172	+	p := StateFile(dir)
	173	+	st, err := os.Stat(p)
	174	+	if errors.Is(err, os.ErrNotExist) {
	175	+	return State{}, ErrNotRegistered
	176	+	}
	177	+	if err != nil {
	178	+	return State{}, err
	179	+	}
	180	+	if st.Mode().Perm()&0o077 != 0 {
	181	+	return State{}, fmt.Errorf("%s is accessible by other users (mode %o): chmod 600 it", p, st.Mode().Perm())
	182	+	}
	183	+	b, err := os.ReadFile(p)
	184	+	if err != nil {
	185	+	return State{}, err
	186	+	}
	187	+	var s State
	188	+	if err := json.Unmarshal(b, &s); err != nil {
	189	+	return State{}, fmt.Errorf("%s: %w", p, err)
	190	+	}
	191	+	if s.RunnerToken == "" {
	192	+	return State{}, fmt.Errorf("%s has no runner token", p)
	193	+	}
	194	+	return s, nil
	195	+	}
	196	+
	197	+	// ResolveRegisterToken returns the token for `register`, from the first source
	198	+	// that has one: standard input (--token-stdin, read to the end and trimmed),
	199	+	// the environment (ERMITE_RUNNER_REGISTER_TOKEN), then the --token flag. The
	200	+	// flag is last because the arguments of a process are visible to every user of
	201	+	// the host (ps, /proc/<pid>/cmdline); the other two are not.
	202	+	func ResolveRegisterToken(flagValue string, fromStdin bool, stdin io.Reader, getenv func(string) string) (string, error) {
	203	+	if fromStdin {
	204	+	b, err := io.ReadAll(io.LimitReader(stdin, 4096))
	205	+	if err != nil {
	206	+	return "", fmt.Errorf("reading the token from standard input: %w", err)
	207	+	}
	208	+	if t := strings.TrimSpace(string(b)); t != "" {
	209	+	return t, nil
	210	+	}
	211	+	return "", errors.New("standard input has no token")
	212	+	}
	213	+	if t := strings.TrimSpace(getenv(EnvRegisterToken)); t != "" {
	214	+	return t, nil
	215	+	}
	216	+	if t := strings.TrimSpace(flagValue); t != "" {
	217	+	return t, nil
	218	+	}
	219	+	return "", fmt.Errorf("give the token in %s or on standard input (--token-stdin); --token works too but is visible in the process list", EnvRegisterToken)
	220	+	}

A internal/host/fake_test.go +562 -0

@@ -0,0 +1,562 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package host
	4	+
	5	+	import (
	6	+	"context"
	7	+	"io"
	8	+	"net"
	9	+	"net/http"
	10	+	"net/http/httptest"
	11	+	"strings"
	12	+	"sync"
	13	+	"testing"
	14	+	"time"
	15	+
	16	+	"connectrpc.com/connect"
	17	+	"github.com/bnema/zerowrap"
	18	+
	19	+	ermite "bnema.dev/ermite-api/gen/ermite"
	20	+	"bnema.dev/ermite-api/gen/ermite/ermiteconnect"
	21	+
	22	+	"bnema.dev/ermite-runner/internal/execstream"
	23	+	"bnema.dev/ermite-runner/internal/execstream/execpb"
	24	+	"bnema.dev/ermite-runner/internal/host/hosttest"
	25	+	"bnema.dev/ermite-runner/internal/sandbox"
	26	+	)
	27	+
	28	+	// fakeServer is an in-memory RunnerService with the contract's rules for
	29	+	// AppendLog (offsets), tokens and the end of a job. Hooks let a test inject
	30	+	// failures.
	31	+	type fakeServer struct {
	32	+	ermiteconnect.UnimplementedRunnerServiceHandler
	33	+
	34	+	mu sync.Mutex
	35	+	runnerTok string
	36	+	jobs []*ermite.Job // queued, handed out in order
	37	+	assigned map[string]*ermite.Job
	38	+	polls []string // poll ids seen
	39	+	pollHook func(n int, req ermite.FetchJobRequest) (ermite.FetchJobResponse, error)
	40	+	pollCalls int
	41	+	retryAfter time.Duration
	42	+
	43	+	beats []time.Time
	44	+	beatHook func(n int) error
	45	+	registers []*ermite.RegisterRunnerRequest
	46	+
	47	+	store *hosttest.LogStore
	48	+	violations []string
	49	+	lines []string // log text, one per line (mirror of store)
	50	+	cont []bool
	51	+	steps map[int32]*ermite.StepUpdate
	52	+	appendReq []*ermite.AppendLogRequest
	53	+	appendHook func(n int, req *ermite.AppendLogRequest) (applyThenFail bool, err error)
	54	+	appendN int
	55	+
	56	+	completes []*ermite.CompleteJobRequest
	57	+	completeHook func(n int) (applyThenFail bool, err error)
	58	+	completeN int
	59	+	ended *ermite.JobEnded // when set, job calls answer JobEnded
	60	+	completed map[string]bool
	61	+
	62	+	watchMsgs chan *ermite.WatchJobResponse
	63	+	watches int
	64	+	}
	65	+
	66	+	func newFakeServer() *fakeServer {
	67	+	s := &fakeServer{
	68	+	runnerTok: "ermiterun_tok", assigned: map[string]ermite.Job{}, completed: map[string]bool{}, steps: map[int32]ermite.StepUpdate{},
	69	+	watchMsgs: make(chan *ermite.WatchJobResponse, 16),
	70	+	}
	71	+	s.store = &hosttest.LogStore{OnViolation: func(m string) { s.violations = append(s.violations, m) }}
	72	+	return s
	73	+	}
	74	+
	75	+	func endedErr(je *ermite.JobEnded) error {
	76	+	e := connect.NewError(connect.CodeFailedPrecondition, io.EOF)
	77	+	if d, err := connect.NewErrorDetail(je); err == nil {
	78	+	e.AddDetail(d)
	79	+	}
	80	+	return e
	81	+	}
	82	+
	83	+	func (s *fakeServer) checkRunner(h http.Header) error {
	84	+	if h.Get("Authorization") != "Bearer "+s.runnerTok {
	85	+	return connect.NewError(connect.CodeUnauthenticated, io.EOF)
	86	+	}
	87	+	return nil
	88	+	}
	89	+
	90	+	func (s fakeServer) checkJob(h http.Header, id string) (ermite.Job, error) {
	91	+	tok := strings.TrimPrefix(h.Get("Authorization"), "Bearer ")
	92	+	for _, j := range s.assigned {
	93	+	if j.JobToken == tok {
	94	+	if j.JobId != id {
	95	+	return nil, connect.NewError(connect.CodePermissionDenied, io.EOF)
	96	+	}
	97	+	return j, nil
	98	+	}
	99	+	}
	100	+	return nil, connect.NewError(connect.CodeUnauthenticated, io.EOF)
	101	+	}
	102	+
	103	+	func (s fakeServer) RegisterRunner(_ context.Context, r connect.Request[ermite.RegisterRunnerRequest]) (*connect.Response[ermite.RegisterRunnerResponse], error) {
	104	+	s.mu.Lock()
	105	+	defer s.mu.Unlock()
	106	+	s.registers = append(s.registers, r.Msg)
	107	+	if r.Header().Get("Authorization") != "Bearer reg-token" {
	108	+	return nil, connect.NewError(connect.CodePermissionDenied, io.EOF)
	109	+	}
	110	+	return connect.NewResponse(&ermite.RegisterRunnerResponse{RunnerId: "rid1", RunnerToken: "ermiterun_new"}), nil
	111	+	}
	112	+
	113	+	func (s fakeServer) FetchJob(_ context.Context, r connect.Request[ermite.FetchJobRequest]) (*connect.Response[ermite.FetchJobResponse], error) {
	114	+	s.mu.Lock()
	115	+	defer s.mu.Unlock()
	116	+	if err := s.checkRunner(r.Header()); err != nil {
	117	+	return nil, err
	118	+	}
	119	+	s.pollCalls++
	120	+	s.polls = append(s.polls, r.Msg.PollId)
	121	+	if s.pollHook != nil {
	122	+	if resp, err := s.pollHook(s.pollCalls, r.Msg); resp != nil \|\| err != nil {
	123	+	if resp != nil && resp.Job != nil {
	124	+	s.assigned[resp.Job.JobId] = resp.Job
	125	+	}
	126	+	return connect.NewResponse(resp), err
	127	+	}
	128	+	}
	129	+	if len(s.jobs) == 0 {
	130	+	resp := &ermite.FetchJobResponse{}
	131	+	if s.retryAfter > 0 {
	132	+	resp.RetryAfter = durationpbOf(s.retryAfter)
	133	+	}
	134	+	return connect.NewResponse(resp), nil
	135	+	}
	136	+	j := s.jobs[0]
	137	+	s.jobs = s.jobs[1:]
	138	+	s.assigned[j.JobId] = j
	139	+	return connect.NewResponse(&ermite.FetchJobResponse{Job: j}), nil
	140	+	}
	141	+
	142	+	func (s fakeServer) Heartbeat(_ context.Context, r connect.Request[ermite.HeartbeatRequest]) (*connect.Response[ermite.HeartbeatResponse], error) {
	143	+	s.mu.Lock()
	144	+	defer s.mu.Unlock()
	145	+	if _, err := s.checkJob(r.Header(), r.Msg.JobId); err != nil {
	146	+	return nil, err
	147	+	}
	148	+	if s.ended != nil {
	149	+	return nil, endedErr(s.ended)
	150	+	}
	151	+	s.beats = append(s.beats, time.Now())
	152	+	if s.beatHook != nil {
	153	+	if err := s.beatHook(len(s.beats)); err != nil {
	154	+	return nil, err
	155	+	}
	156	+	}
	157	+	return connect.NewResponse(&ermite.HeartbeatResponse{}), nil
	158	+	}
	159	+
	160	+	func (s fakeServer) UpdateJob(_ context.Context, r connect.Request[ermite.UpdateJobRequest]) (*connect.Response[ermite.UpdateJobResponse], error) {
	161	+	s.mu.Lock()
	162	+	defer s.mu.Unlock()
	163	+	if _, err := s.checkJob(r.Header(), r.Msg.JobId); err != nil {
	164	+	return nil, err
	165	+	}
	166	+	if s.ended != nil \|\| s.completed[r.Msg.JobId] {
	167	+	je := s.ended
	168	+	if je == nil {
	169	+	je = &ermite.JobEnded{Conclusion: ermite.CiConclusion_CI_CONCLUSION_SUCCESS}
	170	+	}
	171	+	return nil, endedErr(je)
	172	+	}
	173	+	for _, u := range r.Msg.Steps {
	174	+	s.steps[u.Number] = u
	175	+	}
	176	+	return connect.NewResponse(&ermite.UpdateJobResponse{}), nil
	177	+	}
	178	+
	179	+	func (s fakeServer) AppendLog(_ context.Context, r connect.Request[ermite.AppendLogRequest]) (*connect.Response[ermite.AppendLogResponse], error) {
	180	+	s.mu.Lock()
	181	+	defer s.mu.Unlock()
	182	+	if _, err := s.checkJob(r.Header(), r.Msg.JobId); err != nil {
	183	+	return nil, err
	184	+	}
	185	+	if s.ended != nil \|\| s.completed[r.Msg.JobId] {
	186	+	je := s.ended
	187	+	if je == nil {
	188	+	je = &ermite.JobEnded{Conclusion: ermite.CiConclusion_CI_CONCLUSION_SUCCESS}
	189	+	}
	190	+	return nil, endedErr(je)
	191	+	}
	192	+	s.appendN++
	193	+	s.appendReq = append(s.appendReq, r.Msg)
	194	+	// The rules of runner.proto are written once, in hosttest.LogStore.
	195	+	var failAfter error
	196	+	if s.appendHook != nil {
	197	+	thenFail, err := s.appendHook(s.appendN, r.Msg)
	198	+	if err != nil && !thenFail {
	199	+	return nil, err
	200	+	}
	201	+	failAfter = err
	202	+	}
	203	+	next, err := s.store.Apply(r.Msg)
	204	+	if err != nil {
	205	+	return nil, err
	206	+	}
	207	+	s.lines, s.cont = s.lines[:0], s.cont[:0]
	208	+	for _, l := range s.store.Lines() {
	209	+	s.lines = append(s.lines, l.Text)
	210	+	s.cont = append(s.cont, l.Continued)
	211	+	}
	212	+	if failAfter != nil {
	213	+	return nil, failAfter
	214	+	}
	215	+	return connect.NewResponse(&ermite.AppendLogResponse{NextOffset: next}), nil
	216	+	}
	217	+
	218	+	func (s fakeServer) CompleteJob(_ context.Context, r connect.Request[ermite.CompleteJobRequest]) (*connect.Response[ermite.CompleteJobResponse], error) {
	219	+	s.mu.Lock()
	220	+	defer s.mu.Unlock()
	221	+	if _, err := s.checkJob(r.Header(), r.Msg.JobId); err != nil {
	222	+	return nil, err
	223	+	}
	224	+	s.completeN++
	225	+	s.completes = append(s.completes, r.Msg)
	226	+	if s.completeHook != nil {
	227	+	thenFail, err := s.completeHook(s.completeN)
	228	+	if err != nil {
	229	+	if thenFail {
	230	+	s.completed[r.Msg.JobId] = true
	231	+	}
	232	+	return nil, err
	233	+	}
	234	+	}
	235	+	if s.completed[r.Msg.JobId] \|\| s.ended != nil {
	236	+	c := ermite.CiConclusion_CI_CONCLUSION_FAILURE
	237	+	if s.ended != nil {
	238	+	c = s.ended.Conclusion
	239	+	}
	240	+	return connect.NewResponse(&ermite.CompleteJobResponse{Conclusion: c, Applied: false}), nil
	241	+	}
	242	+	s.completed[r.Msg.JobId] = true
	243	+	return connect.NewResponse(&ermite.CompleteJobResponse{Conclusion: r.Msg.Conclusion, Applied: true}), nil
	244	+	}
	245	+
	246	+	func (s fakeServer) WatchJob(ctx context.Context, r connect.Request[ermite.WatchJobRequest], st *connect.ServerStream[ermite.WatchJobResponse]) error {
	247	+	s.mu.Lock()
	248	+	_, err := s.checkJob(r.Header(), r.Msg.JobId)
	249	+	s.watches++
	250	+	ended := s.ended
	251	+	s.mu.Unlock()
	252	+	if err != nil {
	253	+	return err
	254	+	}
	255	+	if ended != nil {
	256	+	return endedErr(ended)
	257	+	}
	258	+	for {
	259	+	select {
	260	+	case m := <-s.watchMsgs:
	261	+	if err := st.Send(m); err != nil {
	262	+	return err
	263	+	}
	264	+	if m.GetEnded() != nil {
	265	+	return nil
	266	+	}
	267	+	case <-ctx.Done():
	268	+	return ctx.Err()
	269	+	}
	270	+	}
	271	+	}
	272	+
	273	+	// start serves the handler over HTTP/1.1 on a local port (Connect streams server messages over it) and returns its URL.
	274	+	func (s fakeServer) start(t testing.T) string {
	275	+	t.Helper()
	276	+	_, h := ermiteconnect.NewRunnerServiceHandler(s)
	277	+	srv := httptest.NewServer(h)
	278	+	t.Cleanup(srv.Close)
	279	+	return srv.URL
	280	+	}
	281	+
	282	+	func (s *fakeServer) snapshotLines() []string {
	283	+	s.mu.Lock()
	284	+	defer s.mu.Unlock()
	285	+	return append([]string(nil), s.lines...)
	286	+	}
	287	+
	288	+	// ---- fake sandbox: the "executor" is a function on the other end of a pipe
	289	+
	290	+	type script func(t testing.T, c execstream.Conn, run execpb.Run, cancels <-chan execpb.Cancel)
	291	+
	292	+	type fakeDriver struct {
	293	+	mu sync.Mutex
	294	+	script script
	295	+	t *testing.T
	296	+	news int
	297	+	stops int
	298	+	started int
	299	+	reaps int
	300	+	reapErr error
	301	+	startErr error
	302	+	stopErr error
	303	+	capacity int
	304	+	live int
	305	+	blockStart chan struct{} // when set, Start waits on it
	306	+	specs []sandbox.JobSpec // what each Start was asked for
	307	+	iso sandbox.Isolation
	308	+	}
	309	+
	310	+	func (d *fakeDriver) Mode() sandbox.Mode { return sandbox.Container }
	311	+	func (d *fakeDriver) Isolation() sandbox.Isolation { return d.iso }
	312	+	func (d *fakeDriver) Close() error { return nil }
	313	+	func (d *fakeDriver) Reap(context.Context) error {
	314	+	d.mu.Lock()
	315	+	defer d.mu.Unlock()
	316	+	d.reaps++
	317	+	return d.reapErr
	318	+	}
	319	+	func (d *fakeDriver) New() (sandbox.Sandbox, error) {
	320	+	d.mu.Lock()
	321	+	defer d.mu.Unlock()
	322	+	if d.capacity > 0 && d.live >= d.capacity {
	323	+	return nil, sandbox.ErrCapacity
	324	+	}
	325	+	d.news++
	326	+	d.live++
	327	+	return &fakeSandbox{d: d}, nil
	328	+	}
	329	+
	330	+	type fakeSandbox struct {
	331	+	d *fakeDriver
	332	+	mu sync.Mutex
	333	+	conn net.Conn
	334	+	stoped bool
	335	+	}
	336	+
	337	+	func (s *fakeSandbox) ID() string { return "er00000000" }
	338	+	func (s *fakeSandbox) Start(ctx context.Context, spec sandbox.JobSpec) (sandbox.Conn, error) {
	339	+	d := s.d
	340	+	d.mu.Lock()
	341	+	d.specs = append(d.specs, spec)
	342	+	d.mu.Unlock()
	343	+	if d.blockStart != nil {
	344	+	select {
	345	+	case <-d.blockStart:
	346	+	case <-ctx.Done():
	347	+	return nil, ctx.Err()
	348	+	}
	349	+	}
	350	+	d.mu.Lock()
	351	+	if d.startErr != nil {
	352	+	err := d.startErr
	353	+	d.mu.Unlock()
	354	+	return nil, err
	355	+	}
	356	+	d.started++
	357	+	sc := d.script
	358	+	d.mu.Unlock()
	359	+	host, guest := net.Pipe()
	360	+	s.mu.Lock()
	361	+	s.conn = guest
	362	+	s.mu.Unlock()
	363	+	go func() {
	364	+	defer func() { _ = guest.Close() }()
	365	+	c := execstream.NewConn(guest)
	366	+	_ = c.Send(&execpb.Frame{Body: &execpb.Frame_Hello{Hello: &execpb.Hello{Version: "test", Sandbox: "container"}}})
	367	+	f, err := c.Recv()
	368	+	if err != nil \|\| f.GetRun() == nil {
	369	+	return
	370	+	}
	371	+	cancels := make(chan *execpb.Cancel, 4)
	372	+	go func() {
	373	+	for {
	374	+	f, err := c.Recv()
	375	+	if err != nil {
	376	+	close(cancels)
	377	+	return
	378	+	}
	379	+	if cn := f.GetCancel(); cn != nil {
	380	+	cancels <- cn
	381	+	}
	382	+	}
	383	+	}()
	384	+	sc(d.t, c, f.GetRun(), cancels)
	385	+	}()
	386	+	return host, nil
	387	+	}
	388	+
	389	+	func (s *fakeSandbox) Stop(context.Context) error {
	390	+	s.mu.Lock()
	391	+	if s.conn != nil {
	392	+	_ = s.conn.Close()
	393	+	}
	394	+	first := !s.stoped
	395	+	s.stoped = true
	396	+	s.mu.Unlock()
	397	+	if first {
	398	+	s.d.mu.Lock()
	399	+	s.d.stops++
	400	+	s.d.live--
	401	+	err := s.d.stopErr
	402	+	s.d.mu.Unlock()
	403	+	return err
	404	+	}
	405	+	return nil
	406	+	}
	407	+
	408	+	// ---- script helpers
	409	+
	410	+	func send(c execstream.Conn, b execpb.Frame) { _ = c.Send(b) }
	411	+
	412	+	func stepStart(c *execstream.Conn, n int, name string) {
	413	+	send(c, &execpb.Frame{Body: &execpb.Frame_StepStarted{StepStarted: &execpb.StepStarted{Number: int32(n), Name: name, UnixNano: time.Now().UnixNano()}}})
	414	+	}
	415	+	func stepDone(c *execstream.Conn, n int, name string, concl execpb.StepConclusion) {
	416	+	send(c, &execpb.Frame{Body: &execpb.Frame_StepCompleted{StepCompleted: &execpb.StepCompleted{Number: int32(n), Name: name, Conclusion: concl, UnixNano: time.Now().UnixNano()}}})
	417	+	}
	418	+	func logLineFrame(c *execstream.Conn, n int, text string, continued bool) {
	419	+	send(c, &execpb.Frame{Body: &execpb.Frame_Log{Log: &execpb.Log{Step: int32(n), UnixNano: time.Now().UnixNano(), Text: text, Continued: continued}}})
	420	+	}
	421	+	func jobDone(c *execstream.Conn, concl execpb.StepConclusion, outputs map[string]string, shutdown bool) {
	422	+	send(c, &execpb.Frame{Body: &execpb.Frame_JobResult{JobResult: &execpb.JobResult{Conclusion: concl, Outputs: outputs, RunnerShutdown: shutdown}}})
	423	+	send(c, &execpb.Frame{Body: &execpb.Frame_Exit{Exit: &execpb.Exit{}}})
	424	+	}
	425	+
	426	+	const (
	427	+	cOK = execpb.StepConclusion_STEP_CONCLUSION_SUCCESS
	428	+	cFail = execpb.StepConclusion_STEP_CONCLUSION_FAILURE
	429	+	cCancel = execpb.StepConclusion_STEP_CONCLUSION_CANCELLED
	430	+	)
	431	+
	432	+	// simpleScript runs a one-step job that logs the given lines.
	433	+	func simpleScript(lines ...string) script {
	434	+	return func(_ testing.T, c execstream.Conn, _ execpb.Run, _ <-chan execpb.Cancel) {
	435	+	stepStart(c, 1, "Set up job")
	436	+	stepDone(c, 1, "Set up job", cOK)
	437	+	stepStart(c, 2, "Run it")
	438	+	for _, l := range lines {
	439	+	logLineFrame(c, 2, l, false)
	440	+	}
	441	+	stepDone(c, 2, "Run it", cOK)
	442	+	stepStart(c, 3, "Complete job")
	443	+	stepDone(c, 3, "Complete job", cOK)
	444	+	jobDone(c, cOK, map[string]string{"out": "v"}, false)
	445	+	}
	446	+	}
	447	+
	448	+	// ---- harness
	449	+
	450	+	func testJob(id string) *ermite.Job {
	451	+	return &ermite.Job{
	452	+	JobId: id, RunId: 7, Attempt: 1, JobToken: "ermitejob_" + id, WorkflowPath: ".github/workflows/ci.yml",
	453	+	WorkflowYaml: "on: push\njobs:\n j:\n runs-on: x\n steps: [{run: echo}]\n", JobKey: "j", Matrix: "{}",
	454	+	Contexts: `{"github":{"repository":"o/r"}}`, Repo: "r", TimeoutMinutes: 5,
	455	+	}
	456	+	}
	457	+
	458	+	func fastTiming() Timing {
	459	+	return Timing{
	460	+	PollTimeout: 5 * time.Second, HeartbeatEvery: 100 * time.Millisecond, HeartbeatRetry: 30 * time.Millisecond,
	461	+	CallTimeout: 5 * time.Second, BackoffMin: 10 * time.Millisecond, BackoffMax: 40 * time.Millisecond,
	462	+	LogFlush: 5 * time.Millisecond, WatchRetry: 20 * time.Millisecond, ShutdownGrace: 2 * time.Second,
	463	+	CompleteDeadline: 10 * time.Second, NoJobPause: 10 * time.Millisecond,
	464	+	}
	465	+	}
	466	+
	467	+	type harness struct {
	468	+	srv *fakeServer
	469	+	url string
	470	+	driver *fakeDriver
	471	+	runner *Runner
	472	+	logs *logBuf
	473	+	}
	474	+
	475	+	// logBuf collects what the runner logs (zerowrap, console format without
	476	+	// colors), so that tests can look at it.
	477	+	type logBuf struct {
	478	+	mu sync.Mutex
	479	+	b strings.Builder
	480	+	}
	481	+
	482	+	func (l *logBuf) Write(p []byte) (int, error) {
	483	+	l.mu.Lock()
	484	+	defer l.mu.Unlock()
	485	+	return l.b.Write(p)
	486	+	}
	487	+
	488	+	func (l *logBuf) String() string { l.mu.Lock(); defer l.mu.Unlock(); return l.b.String() }
	489	+
	490	+	// ctx returns a context with a logger that writes to the buffer (JSON, one
	491	+	// event per line, debug level).
	492	+	func (l *logBuf) ctx() context.Context {
	493	+	return zerowrap.WithCtx(context.Background(), zerowrap.New(zerowrap.Config{Level: "debug", Format: "json", Output: l}))
	494	+	}
	495	+
	496	+	func newHarness(t testing.T, sc script) harness {
	497	+	t.Helper()
	498	+	srv := newFakeServer()
	499	+	url := srv.start(t)
	500	+	drv := &fakeDriver{script: sc, t: t, iso: sandbox.IsolationVM}
	501	+	lb := &logBuf{}
	502	+	h := &harness{srv: srv, url: url, driver: drv, logs: lb}
	503	+	t.Cleanup(func() {
	504	+	srv.mu.Lock()
	505	+	defer srv.mu.Unlock()
	506	+	if len(srv.violations) > 0 {
	507	+	t.Errorf("the host broke the contract of AppendLog: %q", srv.violations)
	508	+	}
	509	+	})
	510	+	h.runner = &Runner{
	511	+	Cfg: Config{URL: url, Name: "r1", Capacity: 1, Version: "test"},
	512	+	Driver: drv,
	513	+	Poll: NewClient(url, "ermiterun_tok", http.DefaultTransport),
	514	+	Job: func(tok string) ermiteconnect.RunnerServiceClient { return NewClient(url, tok, http.DefaultTransport) },
	515	+	Timing: fastTiming(),
	516	+	}
	517	+	return h
	518	+	}
	519	+
	520	+	// run starts the loop; stop cancels it and waits for Run to return.
	521	+	func (h harness) run(t testing.T) (stop func() error) {
	522	+	t.Helper()
	523	+	ctx, cancel := context.WithCancel(h.logs.ctx())
	524	+	done := make(chan error, 1)
	525	+	go func() { done <- h.runner.Run(ctx) }()
	526	+	var once sync.Once
	527	+	var res error
	528	+	stop = func() error {
	529	+	once.Do(func() {
	530	+	cancel()
	531	+	select {
	532	+	case res = <-done:
	533	+	case <-time.After(20 * time.Second):
	534	+	t.Error("Run did not return after shutdown")
	535	+	}
	536	+	})
	537	+	return res
	538	+	}
	539	+	t.Cleanup(func() { _ = stop() })
	540	+	return stop
	541	+	}
	542	+
	543	+	func eventually(t *testing.T, what string, cond func() bool) {
	544	+	t.Helper()
	545	+	deadline := time.Now().Add(15 * time.Second)
	546	+	for time.Now().Before(deadline) {
	547	+	if cond() {
	548	+	return
	549	+	}
	550	+	time.Sleep(10 * time.Millisecond)
	551	+	}
	552	+	t.Fatalf("timed out waiting for %s", what)
	553	+	}
	554	+
	555	+	func (s *fakeServer) completeCount() int { s.mu.Lock(); defer s.mu.Unlock(); return len(s.completes) }
	556	+
	557	+	// Result returns what a job reported, like hosttest.Server.Result.
	558	+	func (s *fakeServer) Result(jobID string) hosttest.Result {
	559	+	s.mu.Lock()
	560	+	defer s.mu.Unlock()
	561	+	return hosttest.Result{Steps: s.steps}
	562	+	}

A internal/host/hosttest/hosttest.go +229 -0

@@ -0,0 +1,229 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	// Package hosttest is an in-memory RunnerService for tests that run the real
	4	+	// host loop against a real sandbox. It enforces the rules of runner.proto that
	5	+	// matter to a client: tokens per kind and per job, the offsets of AppendLog,
	6	+	// the limits of a request, and the end of a job after CompleteJob.
	7	+	package hosttest
	8	+
	9	+	import (
	10	+	"context"
	11	+	"io"
	12	+	"net/http"
	13	+	"net/http/httptest"
	14	+	"strings"
	15	+	"sync"
	16	+
	17	+	"connectrpc.com/connect"
	18	+
	19	+	ermite "bnema.dev/ermite-api/gen/ermite"
	20	+	"bnema.dev/ermite-api/gen/ermite/ermiteconnect"
	21	+	)
	22	+
	23	+	// RunnerToken is the runner token the server accepts.
	24	+	const RunnerToken = "ermiterun_test"
	25	+
	26	+	// Line is one stored log line.
	27	+	type Line struct {
	28	+	Step int32
	29	+	Text string
	30	+	Continued bool
	31	+	}
	32	+
	33	+	// Result is what a job reported.
	34	+	type Result struct {
	35	+	Complete *ermite.CompleteJobRequest
	36	+	Lines []Line
	37	+	Steps map[int32]*ermite.StepUpdate
	38	+	}
	39	+
	40	+	// Server is the fake. Create it with New; URL is where to point the runner.
	41	+	type Server struct {
	42	+	ermiteconnect.UnimplementedRunnerServiceHandler
	43	+
	44	+	URL string
	45	+	srv *httptest.Server
	46	+
	47	+	mu sync.Mutex
	48	+	queue []*ermite.Job
	49	+	jobs map[string]*jobState
	50	+	assigned map[string]*ermite.Job // by job token
	51	+	// Errors lists contract violations seen (a gap in AppendLog, a request over
	52	+	// the limits, a call after the end): a test fails when it is not empty.
	53	+	Errors []string
	54	+	done chan string
	55	+	}
	56	+
	57	+	type jobState struct {
	58	+	log *LogStore
	59	+	steps map[int32]*ermite.StepUpdate
	60	+	complete *ermite.CompleteJobRequest
	61	+	}
	62	+
	63	+	// New starts the server; Close stops it.
	64	+	func New() *Server {
	65	+	s := &Server{jobs: map[string]jobState{}, assigned: map[string]ermite.Job{}, done: make(chan string, 256)}
	66	+	_, h := ermiteconnect.NewRunnerServiceHandler(s)
	67	+	s.srv = httptest.NewServer(h)
	68	+	s.URL = s.srv.URL
	69	+	return s
	70	+	}
	71	+
	72	+	// Close stops the server.
	73	+	func (s *Server) Close() { s.srv.Close() }
	74	+
	75	+	// Queue adds jobs to hand out, in order.
	76	+	func (s Server) Queue(jobs ...ermite.Job) {
	77	+	s.mu.Lock()
	78	+	defer s.mu.Unlock()
	79	+	s.queue = append(s.queue, jobs...)
	80	+	}
	81	+
	82	+	// Violations returns the contract violations seen so far.
	83	+	func (s *Server) Violations() []string {
	84	+	s.mu.Lock()
	85	+	defer s.mu.Unlock()
	86	+	return append([]string(nil), s.Errors...)
	87	+	}
	88	+
	89	+	// Done receives the id of each job when its CompleteJob has been applied.
	90	+	func (s *Server) Done() <-chan string { return s.done }
	91	+
	92	+	// Result returns what a job reported so far.
	93	+	func (s *Server) Result(jobID string) Result {
	94	+	s.mu.Lock()
	95	+	defer s.mu.Unlock()
	96	+	st := s.jobs[jobID]
	97	+	if st == nil {
	98	+	return Result{}
	99	+	}
	100	+	steps := map[int32]*ermite.StepUpdate{}
	101	+	for k, v := range st.steps {
	102	+	steps[k] = v
	103	+	}
	104	+	return Result{Complete: st.complete, Lines: st.log.Lines(), Steps: steps}
	105	+	}
	106	+
	107	+	func (s *Server) violation(msg string) { s.Errors = append(s.Errors, msg) } // called with s.mu held
	108	+
	109	+	func unauth() error { return connect.NewError(connect.CodeUnauthenticated, io.EOF) }
	110	+
	111	+	func (s Server) job(h http.Header, id string) (jobState, error) {
	112	+	j := s.assigned[strings.TrimPrefix(h.Get("Authorization"), "Bearer ")]
	113	+	if j == nil {
	114	+	return nil, unauth()
	115	+	}
	116	+	if j.JobId != id {
	117	+	return nil, connect.NewError(connect.CodePermissionDenied, io.EOF)
	118	+	}
	119	+	return s.jobs[id], nil
	120	+	}
	121	+
	122	+	func ended(c ermite.CiConclusion) error {
	123	+	e := connect.NewError(connect.CodeFailedPrecondition, io.EOF)
	124	+	if d, err := connect.NewErrorDetail(&ermite.JobEnded{Conclusion: c}); err == nil {
	125	+	e.AddDetail(d)
	126	+	}
	127	+	return e
	128	+	}
	129	+
	130	+	// FetchJob hands out the queued jobs.
	131	+	func (s Server) FetchJob(_ context.Context, r connect.Request[ermite.FetchJobRequest]) (*connect.Response[ermite.FetchJobResponse], error) {
	132	+	s.mu.Lock()
	133	+	defer s.mu.Unlock()
	134	+	if r.Header().Get("Authorization") != "Bearer "+RunnerToken {
	135	+	return nil, unauth()
	136	+	}
	137	+	if len(s.queue) == 0 {
	138	+	return connect.NewResponse(&ermite.FetchJobResponse{}), nil
	139	+	}
	140	+	j := s.queue[0]
	141	+	s.queue = s.queue[1:]
	142	+	s.assigned[j.JobToken] = j
	143	+	s.jobs[j.JobId] = &jobState{log: &LogStore{OnViolation: s.violation}, steps: map[int32]*ermite.StepUpdate{}}
	144	+	return connect.NewResponse(&ermite.FetchJobResponse{Job: j}), nil
	145	+	}
	146	+
	147	+	// Heartbeat accepts heartbeats of running jobs.
	148	+	func (s Server) Heartbeat(_ context.Context, r connect.Request[ermite.HeartbeatRequest]) (*connect.Response[ermite.HeartbeatResponse], error) {
	149	+	s.mu.Lock()
	150	+	defer s.mu.Unlock()
	151	+	st, err := s.job(r.Header(), r.Msg.JobId)
	152	+	if err != nil {
	153	+	return nil, err
	154	+	}
	155	+	if st.complete != nil {
	156	+	return nil, ended(st.complete.Conclusion)
	157	+	}
	158	+	return connect.NewResponse(&ermite.HeartbeatResponse{}), nil
	159	+	}
	160	+
	161	+	// UpdateJob stores step snapshots.
	162	+	func (s Server) UpdateJob(_ context.Context, r connect.Request[ermite.UpdateJobRequest]) (*connect.Response[ermite.UpdateJobResponse], error) {
	163	+	s.mu.Lock()
	164	+	defer s.mu.Unlock()
	165	+	st, err := s.job(r.Header(), r.Msg.JobId)
	166	+	if err != nil {
	167	+	return nil, err
	168	+	}
	169	+	if st.complete != nil {
	170	+	return nil, ended(st.complete.Conclusion)
	171	+	}
	172	+	for _, u := range r.Msg.Steps {
	173	+	if old := st.steps[u.Number]; old != nil && old.State == ermite.StepState_STEP_STATE_COMPLETED {
	174	+	s.violation("UpdateJob: a completed step changed")
	175	+	}
	176	+	st.steps[u.Number] = u
	177	+	}
	178	+	return connect.NewResponse(&ermite.UpdateJobResponse{}), nil
	179	+	}
	180	+
	181	+	// AppendLog applies the rules of runner.proto through a LogStore.
	182	+	func (s Server) AppendLog(_ context.Context, r connect.Request[ermite.AppendLogRequest]) (*connect.Response[ermite.AppendLogResponse], error) {
	183	+	s.mu.Lock()
	184	+	defer s.mu.Unlock()
	185	+	st, err := s.job(r.Header(), r.Msg.JobId)
	186	+	if err != nil {
	187	+	return nil, err
	188	+	}
	189	+	if st.complete != nil {
	190	+	s.violation("AppendLog after CompleteJob")
	191	+	return nil, ended(st.complete.Conclusion)
	192	+	}
	193	+	next, err := st.log.Apply(r.Msg)
	194	+	if err != nil {
	195	+	return nil, err
	196	+	}
	197	+	return connect.NewResponse(&ermite.AppendLogResponse{NextOffset: next}), nil
	198	+	}
	199	+
	200	+	// CompleteJob ends a job, idempotently.
	201	+	func (s Server) CompleteJob(_ context.Context, r connect.Request[ermite.CompleteJobRequest]) (*connect.Response[ermite.CompleteJobResponse], error) {
	202	+	s.mu.Lock()
	203	+	defer s.mu.Unlock()
	204	+	st, err := s.job(r.Header(), r.Msg.JobId)
	205	+	if err != nil {
	206	+	return nil, err
	207	+	}
	208	+	if st.complete != nil {
	209	+	return connect.NewResponse(&ermite.CompleteJobResponse{Conclusion: st.complete.Conclusion}), nil
	210	+	}
	211	+	st.complete = r.Msg
	212	+	for _, u := range r.Msg.Steps {
	213	+	st.steps[u.Number] = u
	214	+	}
	215	+	s.done <- r.Msg.JobId
	216	+	return connect.NewResponse(&ermite.CompleteJobResponse{Conclusion: r.Msg.Conclusion, Applied: true}), nil
	217	+	}
	218	+
	219	+	// WatchJob keeps the stream open until the job ends or the client leaves.
	220	+	func (s Server) WatchJob(ctx context.Context, r connect.Request[ermite.WatchJobRequest], _ *connect.ServerStream[ermite.WatchJobResponse]) error {
	221	+	s.mu.Lock()
	222	+	_, err := s.job(r.Header(), r.Msg.JobId)
	223	+	s.mu.Unlock()
	224	+	if err != nil {
	225	+	return err
	226	+	}
	227	+	<-ctx.Done()
	228	+	return ctx.Err()
	229	+	}

A internal/host/hosttest/logstore.go +89 -0

@@ -0,0 +1,89 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package hosttest
	4	+
	5	+	import (
	6	+	"io"
	7	+	"strings"
	8	+	"sync"
	9	+
	10	+	"connectrpc.com/connect"
	11	+
	12	+	ermite "bnema.dev/ermite-api/gen/ermite"
	13	+	)
	14	+
	15	+	// LogStore is the log of one job as the server of runner.proto keeps it, with
	16	+	// the rules of AppendLog (the one place where they are written down for tests):
	17	+	//
	18	+	// - a request has 1 to 1,000 lines, at most 64 KiB of text in total, no line
	19	+	// feed in a line, a time on every line, and a continued first line only if
	20	+	// it follows a line;
	21	+	// - offset is the number of lines the server holds: equal appends, below skips
	22	+	// the first (count - offset) lines of the request (a retry), above is
	23	+	// out_of_range;
	24	+	// - the step of a request is at least the step of the last stored line
	25	+	// (failed_precondition otherwise; lines skipped as already stored are not
	26	+	// checked).
	27	+	//
	28	+	// A request that breaks a rule the client must never break (the first group)
	29	+	// is reported through OnViolation as well as answered with the error.
	30	+	type LogStore struct {
	31	+	// OnViolation, when set, receives the description of each client violation.
	32	+	OnViolation func(string)
	33	+
	34	+	mu sync.Mutex
	35	+	lines []Line
	36	+	}
	37	+
	38	+	// Lines returns a copy of the stored lines.
	39	+	func (s *LogStore) Lines() []Line {
	40	+	s.mu.Lock()
	41	+	defer s.mu.Unlock()
	42	+	return append([]Line(nil), s.lines...)
	43	+	}
	44	+
	45	+	// Apply applies a request and returns the number of lines held afterwards.
	46	+	func (s LogStore) Apply(r ermite.AppendLogRequest) (int64, error) {
	47	+	s.mu.Lock()
	48	+	defer s.mu.Unlock()
	49	+	bad := func(code connect.Code, why string) (int64, error) {
	50	+	if s.OnViolation != nil {
	51	+	s.OnViolation("AppendLog: " + why)
	52	+	}
	53	+	return 0, connect.NewError(code, io.EOF)
	54	+	}
	55	+	if r.Step < 1 {
	56	+	return bad(connect.CodeInvalidArgument, "step below 1")
	57	+	}
	58	+	n, bytes := len(r.Lines), 0
	59	+	if n == 0 \|\| n > 1000 {
	60	+	return bad(connect.CodeInvalidArgument, "bad number of lines")
	61	+	}
	62	+	have := int64(len(s.lines))
	63	+	for i, l := range r.Lines {
	64	+	bytes += len(l.Text)
	65	+	if strings.Contains(l.Text, "\n") \|\| l.Time == nil {
	66	+	return bad(connect.CodeInvalidArgument, "invalid line")
	67	+	}
	68	+	if l.Continued && i == 0 && have == 0 && r.Offset == 0 {
	69	+	return bad(connect.CodeInvalidArgument, "the first line of the job is continued")
	70	+	}
	71	+	}
	72	+	if bytes > 64*1024 {
	73	+	return bad(connect.CodeInvalidArgument, "more than 64 KiB")
	74	+	}
	75	+	if r.Offset > have {
	76	+	return bad(connect.CodeOutOfRange, "a gap in the offsets")
	77	+	}
	78	+	skip := have - r.Offset
	79	+	if skip < int64(n) && have > 0 && r.Step < s.lines[have-1].Step {
	80	+	return bad(connect.CodeFailedPrecondition, "a step went back")
	81	+	}
	82	+	for i, l := range r.Lines {
	83	+	if int64(i) < skip {
	84	+	continue
	85	+	}
	86	+	s.lines = append(s.lines, Line{Step: r.Step, Text: l.Text, Continued: l.Continued})
	87	+	}
	88	+	return int64(len(s.lines)), nil
	89	+	}

A internal/host/job.go +888 -0

@@ -0,0 +1,888 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package host
	4	+
	5	+	import (
	6	+	"context"
	7	+	"errors"
	8	+	"fmt"
	9	+	"sort"
	10	+	"strings"
	11	+	"sync"
	12	+	"time"
	13	+	"unicode/utf8"
	14	+
	15	+	"connectrpc.com/connect"
	16	+	"github.com/bnema/zerowrap"
	17	+	"google.golang.org/protobuf/proto"
	18	+	"google.golang.org/protobuf/types/known/timestamppb"
	19	+
	20	+	ermite "bnema.dev/ermite-api/gen/ermite"
	21	+	"bnema.dev/ermite-api/gen/ermite/ermiteconnect"
	22	+
	23	+	"bnema.dev/ermite-runner/internal/execstream"
	24	+	"bnema.dev/ermite-runner/internal/execstream/execpb"
	25	+	"bnema.dev/ermite-runner/internal/sandbox"
	26	+	)
	27	+
	28	+	// jobClient is what a job needs from the RunnerService client (job token).
	29	+	type jobClient interface {
	30	+	callClient
	31	+	Heartbeat(context.Context, connect.Request[ermite.HeartbeatRequest]) (connect.Response[ermite.HeartbeatResponse], error)
	32	+	WatchJob(context.Context, connect.Request[ermite.WatchJobRequest]) (connect.ServerStreamForClient[ermite.WatchJobResponse], error)
	33	+	CompleteJob(context.Context, connect.Request[ermite.CompleteJobRequest]) (connect.Response[ermite.CompleteJobResponse], error)
	34	+	}
	35	+
	36	+	var _ jobClient = ermiteconnect.RunnerServiceClient(nil)
	37	+
	38	+	// jobRun is the host's side of one job: the assignment, the sandbox, and the
	39	+	// calls to ermite with the job token.
	40	+	type jobRun struct {
	41	+	r *Runner
	42	+	job *ermite.Job
	43	+	client jobClient
	44	+	t Timing
	45	+	log zerowrap.Logger // with job_id, run_id and attempt
	46	+
	47	+	// ctx lives until the job is finished or abandoned. Shutdown does not end
	48	+	// it: a shutdown is a cancellation that the job reports.
	49	+	ctx context.Context
	50	+	cancel context.CancelCauseFunc
	51	+
	52	+	// receivedAt is when the host got the job: the job timeout counts from it
	53	+	// (runner.proto: Job.timeout_minutes).
	54	+	receivedAt time.Time
	55	+
	56	+	cancelCh chan *execpb.Cancel // the first value is sent to the executor
	57	+	cancelMu sync.Mutex
	58	+	cancelAt string // reason of the first cancel request, "" if none
	59	+	cancelKind execpb.CancelKind // kind of the first cancel request
	60	+	stopping bool // the runner is shutting down
	61	+
	62	+	mu sync.Mutex
	63	+	steps map[int32]*ermite.StepUpdate
	64	+	// What the guest has told us, for the validation of the next frame: the
	65	+	// guest is untrusted code, its step numbers and lines are checked before
	66	+	// they reach the server (see violate).
	67	+	maxStarted int32
	68	+	lastLogStep int32
	69	+	violation string
	70	+	// hard stops the sandbox at once; set before the session starts.
	71	+	hard func()
	72	+	result *execpb.JobResult
	73	+	outbox *outbox
	74	+	endedBy error // the server ended the job, or the assignment is gone
	75	+	}
	76	+
	77	+	var errAbandoned = errors.New("the server ended the job or revoked the assignment")
	78	+
	79	+	func (j *jobRun) abandon(err error) {
	80	+	j.mu.Lock()
	81	+	if j.endedBy == nil {
	82	+	j.endedBy = err
	83	+	}
	84	+	j.mu.Unlock()
	85	+	j.cancel(errAbandoned)
	86	+	}
	87	+
	88	+	func (j *jobRun) abandoned() error {
	89	+	j.mu.Lock()
	90	+	defer j.mu.Unlock()
	91	+	return j.endedBy
	92	+	}
	93	+
	94	+	// requestCancel asks the executor to stop; only the first request counts.
	95	+	func (j *jobRun) requestCancel(reason string, kind execpb.CancelKind) {
	96	+	j.cancelMu.Lock()
	97	+	defer j.cancelMu.Unlock()
	98	+	if j.cancelAt != "" {
	99	+	return
	100	+	}
	101	+	j.cancelAt = reason
	102	+	j.cancelKind = kind
	103	+	j.cancelCh <- &execpb.Cancel{Reason: reason, Kind: kind}
	104	+	}
	105	+
	106	+	// firstCancelKind is the kind of the first cancel request.
	107	+	func (j *jobRun) firstCancelKind() execpb.CancelKind {
	108	+	j.cancelMu.Lock()
	109	+	defer j.cancelMu.Unlock()
	110	+	return j.cancelKind
	111	+	}
	112	+
	113	+	func (j *jobRun) cancelRequested() bool {
	114	+	j.cancelMu.Lock()
	115	+	defer j.cancelMu.Unlock()
	116	+	return j.cancelAt != ""
	117	+	}
	118	+
	119	+	// heartbeat sends the acknowledgement. It retries every HeartbeatRetry until
	120	+	// it succeeds, so that the 30 s of the server are used well; it returns false
	121	+	// when the assignment is gone.
	122	+	func (j *jobRun) ack(ctx context.Context) bool {
	123	+	for {
	124	+	err := j.beat(ctx)
	125	+	if err == nil {
	126	+	return true
	127	+	}
	128	+	if ctx.Err() != nil {
	129	+	return false
	130	+	}
	131	+	switch classify(err) {
	132	+	case kindEnded, kindAbandoned:
	133	+	j.abandon(err)
	134	+	return false
	135	+	}
	136	+	j.log.Warn().Err(err).Msg("first heartbeat failed, retrying")
	137	+	if !sleep(ctx, j.t.HeartbeatRetry) {
	138	+	return false
	139	+	}
	140	+	}
	141	+	}
	142	+
	143	+	func (j *jobRun) beat(ctx context.Context) error {
	144	+	cctx, cancel := context.WithTimeout(ctx, j.t.CallTimeout)
	145	+	defer cancel()
	146	+	_, err := j.client.Heartbeat(cctx, connect.NewRequest(&ermite.HeartbeatRequest{JobId: j.job.JobId}))
	147	+	return err
	148	+	}
	149	+
	150	+	// heartbeats keeps the job alive: every HeartbeatEvery, and after a failure
	151	+	// every HeartbeatRetry until one succeeds (runner.proto: Heartbeat).
	152	+	func (j *jobRun) heartbeats(ctx context.Context) {
	153	+	wait := j.t.HeartbeatEvery
	154	+	for {
	155	+	if !sleep(ctx, wait) {
	156	+	return
	157	+	}
	158	+	if err := j.beat(ctx); err != nil {
	159	+	if ctx.Err() != nil {
	160	+	return
	161	+	}
	162	+	switch classify(err) {
	163	+	case kindEnded, kindAbandoned:
	164	+	j.abandon(err)
	165	+	return
	166	+	}
	167	+	j.log.Warn().Err(err).Dur("retry_in", j.t.HeartbeatRetry).Msg("heartbeat failed")
	168	+	wait = j.t.HeartbeatRetry
	169	+	continue
	170	+	}
	171	+	wait = j.t.HeartbeatEvery
	172	+	}
	173	+	}
	174	+
	175	+	// watch follows WatchJob: a cancel request stops the job, an Ended message or
	176	+	// the end-of-job error abandons it. The stream is reopened after any other
	177	+	// error and after a clean close without Ended.
	178	+	func (j *jobRun) watch(ctx context.Context) {
	179	+	for ctx.Err() == nil {
	180	+	stream, err := j.client.WatchJob(ctx, connect.NewRequest(&ermite.WatchJobRequest{JobId: j.job.JobId}))
	181	+	if err == nil {
	182	+	for stream.Receive() {
	183	+	switch m := stream.Msg().Msg.(type) {
	184	+	case *ermite.WatchJobResponse_Cancel:
	185	+	j.requestCancel(cancelText(m.Cancel), kindOf(m.Cancel.GetReason()))
	186	+	case *ermite.WatchJobResponse_Ended:
	187	+	_ = stream.Close()
	188	+	j.abandon(fmt.Errorf("job ended: %s", m.Ended.GetConclusion()))
	189	+	return
	190	+	}
	191	+	}
	192	+	err = stream.Err()
	193	+	_ = stream.Close()
	194	+	}
	195	+	if ctx.Err() != nil {
	196	+	return
	197	+	}
	198	+	if err != nil {
	199	+	switch classify(err) {
	200	+	case kindEnded, kindAbandoned:
	201	+	j.abandon(err)
	202	+	return
	203	+	}
	204	+	j.log.Warn().Err(err).Msg("WatchJob failed, reopening")
	205	+	}
	206	+	if !sleep(ctx, j.t.WatchRetry) {
	207	+	return
	208	+	}
	209	+	}
	210	+	}
	211	+
	212	+	func cancelText(c *ermite.CancelRequest) string {
	213	+	if c.GetMessage() != "" {
	214	+	return c.GetMessage()
	215	+	}
	216	+	return c.GetReason().String()
	217	+	}
	218	+
	219	+	func kindOf(r ermite.CancelReason) execpb.CancelKind {
	220	+	if r == ermite.CancelReason_CANCEL_REASON_TIMEOUT {
	221	+	return execpb.CancelKind_CANCEL_KIND_TIMEOUT
	222	+	}
	223	+	// User, fail-fast, and any reason this runner does not know: cancelled.
	224	+	return execpb.CancelKind_CANCEL_KIND_USER
	225	+	}
	226	+
	227	+	// ---- steps
	228	+
	229	+	// Limits on what the guest may send, which keep the memory of the host bounded
	230	+	// whatever the guest does: runner.proto allows 1,000 steps per job, names of
	231	+	// 1,024 characters, and lines of 64 KiB.
	232	+	const (
	233	+	maxSteps = 1000
	234	+	maxNameLen = 1024
	235	+	maxLineText = 64 * 1024
	236	+	)
	237	+
	238	+	// violate fails the job because the guest broke the protocol: it writes the
	239	+	// reason in the log, asks the executor to stop and destroys the sandbox. Only
	240	+	// the first violation counts; the frames that follow are dropped.
	241	+	func (j *jobRun) violate(format string, a ...any) {
	242	+	j.mu.Lock()
	243	+	if j.violation != "" {
	244	+	j.mu.Unlock()
	245	+	return
	246	+	}
	247	+	j.violation = fmt.Sprintf(format, a...)
	248	+	msg := j.violation
	249	+	j.mu.Unlock()
	250	+	j.log.Error().Str("violation", msg).Msg("the executor broke the protocol")
	251	+	j.sandboxError("The runner stopped the job: the executor broke the protocol: " + msg)
	252	+	if j.hard != nil {
	253	+	j.hard()
	254	+	}
	255	+	}
	256	+
	257	+	func (j *jobRun) violated() bool {
	258	+	j.mu.Lock()
	259	+	defer j.mu.Unlock()
	260	+	return j.violation != ""
	261	+	}
	262	+
	263	+	func (j jobRun) stepStarted(s execpb.StepStarted) {
	264	+	j.mu.Lock()
	265	+	switch {
	266	+	case j.violation != "":
	267	+	j.mu.Unlock()
	268	+	return
	269	+	case s.Number < 1 \|\| s.Number > maxSteps:
	270	+	j.mu.Unlock()
	271	+	j.violate("step number %d is outside 1..%d", s.Number, maxSteps)
	272	+	return
	273	+	case s.Number <= j.maxStarted:
	274	+	j.mu.Unlock()
	275	+	j.violate("step %d started after step %d", s.Number, j.maxStarted)
	276	+	return
	277	+	}
	278	+	j.maxStarted = s.Number
	279	+	u := &ermite.StepUpdate{Number: s.Number, Name: cutName(s.Name), State: ermite.StepState_STEP_STATE_IN_PROGRESS, StartedAt: tsOf(s.UnixNano)}
	280	+	j.steps[s.Number] = u
	281	+	j.mu.Unlock()
	282	+	j.outbox.Step(u)
	283	+	}
	284	+
	285	+	func (j jobRun) stepCompleted(s execpb.StepCompleted) {
	286	+	j.mu.Lock()
	287	+	if j.violation != "" {
	288	+	j.mu.Unlock()
	289	+	return
	290	+	}
	291	+	old := j.steps[s.Number]
	292	+	switch {
	293	+	case old == nil:
	294	+	j.mu.Unlock()
	295	+	j.violate("step %d completed without having started", s.Number)
	296	+	return
	297	+	case old.State == ermite.StepState_STEP_STATE_COMPLETED:
	298	+	j.mu.Unlock()
	299	+	j.violate("step %d completed twice", s.Number)
	300	+	return
	301	+	}
	302	+	u := cloneStep(old)
	303	+	if s.Name != "" {
	304	+	u.Name = cutName(s.Name)
	305	+	}
	306	+	u.State = ermite.StepState_STEP_STATE_COMPLETED
	307	+	u.Conclusion = conclusionOf(s.Conclusion)
	308	+	if u.Conclusion == ermite.CiConclusion_CI_CONCLUSION_UNSPECIFIED \|\| u.Conclusion == ermite.CiConclusion_CI_CONCLUSION_TIMED_OUT {
	309	+	u.Conclusion = ermite.CiConclusion_CI_CONCLUSION_FAILURE
	310	+	}
	311	+	u.CompletedAt = tsOf(s.UnixNano)
	312	+	j.steps[s.Number] = u
	313	+	j.mu.Unlock()
	314	+	j.outbox.Step(u)
	315	+	}
	316	+
	317	+	// logLine checks and queues a line of the log. The step must be one that was
	318	+	// started and not completed, and not lower than the step of the previous line
	319	+	// (the server refuses a log that goes back); the text is made valid for the
	320	+	// server: UTF-8, no line feed (a line feed in a line makes more lines).
	321	+	func (j jobRun) logLine(l execpb.Log) {
	322	+	j.mu.Lock()
	323	+	if j.violation != "" {
	324	+	j.mu.Unlock()
	325	+	return
	326	+	}
	327	+	st := j.steps[l.Step]
	328	+	switch {
	329	+	case l.Step < 1 \|\| l.Step > maxSteps \|\| st == nil:
	330	+	j.mu.Unlock()
	331	+	j.violate("a log line for step %d, which has not started", l.Step)
	332	+	return
	333	+	case st.State == ermite.StepState_STEP_STATE_COMPLETED:
	334	+	j.mu.Unlock()
	335	+	j.violate("a log line for step %d, which is completed", l.Step)
	336	+	return
	337	+	case l.Step < j.lastLogStep:
	338	+	j.mu.Unlock()
	339	+	j.violate("a log line for step %d after one for step %d", l.Step, j.lastLogStep)
	340	+	return
	341	+	case len(l.Text) > maxLineText:
	342	+	j.mu.Unlock()
	343	+	j.violate("a log line of %d bytes (the limit is %d)", len(l.Text), maxLineText)
	344	+	return
	345	+	}
	346	+	j.lastLogStep = l.Step
	347	+	j.mu.Unlock()
	348	+	text := l.Text
	349	+	if !utf8.ValidString(text) {
	350	+	text = strings.ToValidUTF8(text, "\ufffd")
	351	+	}
	352	+	for i, part := range strings.Split(text, "\n") {
	353	+	j.outbox.Log(logLine{step: l.Step, unixNano: l.UnixNano, text: part, continued: l.Continued && i == 0})
	354	+	}
	355	+	}
	356	+
	357	+	// cutName bounds a step name to what the server keeps, on a character boundary.
	358	+	func cutName(s string) string {
	359	+	if len(s) <= maxNameLen {
	360	+	return strings.ToValidUTF8(s, "\ufffd")
	361	+	}
	362	+	s = s[:maxNameLen]
	363	+	for !utf8.ValidString(s) && len(s) > 0 {
	364	+	s = s[:len(s)-1]
	365	+	}
	366	+	return s
	367	+	}
	368	+
	369	+	func cloneStep(u ermite.StepUpdate) ermite.StepUpdate {
	370	+	return proto.Clone(u).(*ermite.StepUpdate)
	371	+	}
	372	+
	373	+	func tsOf(unixNano int64) *timestamppb.Timestamp {
	374	+	if unixNano == 0 {
	375	+	return timestamppb.Now()
	376	+	}
	377	+	return timestamppb.New(time.Unix(0, unixNano))
	378	+	}
	379	+
	380	+	func conclusionOf(c execpb.StepConclusion) ermite.CiConclusion {
	381	+	switch c {
	382	+	case execpb.StepConclusion_STEP_CONCLUSION_SUCCESS:
	383	+	return ermite.CiConclusion_CI_CONCLUSION_SUCCESS
	384	+	case execpb.StepConclusion_STEP_CONCLUSION_FAILURE:
	385	+	return ermite.CiConclusion_CI_CONCLUSION_FAILURE
	386	+	case execpb.StepConclusion_STEP_CONCLUSION_CANCELLED:
	387	+	return ermite.CiConclusion_CI_CONCLUSION_CANCELLED
	388	+	case execpb.StepConclusion_STEP_CONCLUSION_SKIPPED:
	389	+	return ermite.CiConclusion_CI_CONCLUSION_SKIPPED
	390	+	case execpb.StepConclusion_STEP_CONCLUSION_TIMED_OUT:
	391	+	return ermite.CiConclusion_CI_CONCLUSION_TIMED_OUT
	392	+	}
	393	+	return ermite.CiConclusion_CI_CONCLUSION_UNSPECIFIED
	394	+	}
	395	+
	396	+	// finalSteps closes the steps that never completed (the executor died or was
	397	+	// stopped) with the conclusion of the job, and returns all snapshots in order.
	398	+	func (j jobRun) finalSteps(jobConclusion ermite.CiConclusion) []ermite.StepUpdate {
	399	+	j.mu.Lock()
	400	+	defer j.mu.Unlock()
	401	+	end := ermite.CiConclusion_CI_CONCLUSION_FAILURE
	402	+	if jobConclusion == ermite.CiConclusion_CI_CONCLUSION_CANCELLED \|\| jobConclusion == ermite.CiConclusion_CI_CONCLUSION_TIMED_OUT {
	403	+	end = ermite.CiConclusion_CI_CONCLUSION_CANCELLED
	404	+	}
	405	+	var out []*ermite.StepUpdate
	406	+	for _, u := range j.steps {
	407	+	if u.State != ermite.StepState_STEP_STATE_COMPLETED {
	408	+	u = cloneStep(u)
	409	+	u.State, u.Conclusion, u.CompletedAt = ermite.StepState_STEP_STATE_COMPLETED, end, timestamppb.Now()
	410	+	}
	411	+	out = append(out, u)
	412	+	}
	413	+	sort.Slice(out, func(a, b int) bool { return out[a].Number < out[b].Number })
	414	+	return out
	415	+	}
	416	+
	417	+	// ---- the job
	418	+
	419	+	// run runs the whole job: acknowledgement, sandbox, executor session, final
	420	+	// report. It returns when everything is over and the sandbox is gone.
	421	+	func (j jobRun) run(rt Runner, shutdown <-chan struct{}) {
	422	+	defer j.cancel(nil)
	423	+	ctx := j.ctx
	424	+
	425	+	if !j.acknowledge(ctx, shutdown) {
	426	+	return
	427	+	}
	428	+	var bg sync.WaitGroup
	429	+	bg.Add(2)
	430	+	go func() { defer bg.Done(); j.heartbeats(ctx) }()
	431	+	go func() { defer bg.Done(); j.watch(ctx) }()
	432	+	// The context is cancelled first, so that the heartbeat and watch loops end.
	433	+	defer func() { j.cancel(nil); bg.Wait() }()
	434	+
	435	+	j.outbox = newOutbox(j.client, j.job.JobId, j.t, func(err error) { j.abandon(err) }, j.log)
	436	+	j.outbox.start(ctx)
	437	+
	438	+	stopHard := j.startStopTimers(ctx, shutdown)
	439	+	out := j.execute(ctx, rt, stopHard)
	440	+	j.report(ctx, out, shutdown)
	441	+	}
	442	+
	443	+	// acknowledge sends the first heartbeat, before anything else. A shutdown ends
	444	+	// the wait: the job is not started and not acknowledged, and the server gives it
	445	+	// to another runner when no heartbeat comes within 30 s (runner.proto, FetchJob:
	446	+	// this is not a failure of the job). It reports whether the job goes on.
	447	+	func (j *jobRun) acknowledge(ctx context.Context, shutdown <-chan struct{}) bool {
	448	+	ackCtx, stopAck := context.WithCancel(ctx)
	449	+	defer stopAck()
	450	+	go func() {
	451	+	select {
	452	+	case <-shutdown:
	453	+	stopAck()
	454	+	case <-ackCtx.Done():
	455	+	}
	456	+	}()
	457	+	if j.ack(ackCtx) {
	458	+	return true
	459	+	}
	460	+	select {
	461	+	case <-shutdown:
	462	+	j.log.Warn().Msg("the runner is shutting down before the job was acknowledged: the server will give it to another runner")
	463	+	default:
	464	+	j.log.Warn().Msg("the assignment was lost before the job started")
	465	+	}
	466	+	return false
	467	+	}
	468	+
	469	+	// startStopTimers arms the two reasons to stop the job from the host, each a
	470	+	// cancellation followed by a hard stop of the sandbox (the returned channel is
	471	+	// closed) if the executor does not answer: a shutdown of the runner
	472	+	// (ShutdownGrace), and the job timeout, which the executor enforces itself but
	473	+	// a stuck executor would not (TimeoutGrace after the deadline: the escalation
	474	+	// SIGINT, SIGTERM, SIGKILL takes 10 s). The goroutines end with ctx.
	475	+	func (j *jobRun) startStopTimers(ctx context.Context, shutdown <-chan struct{}) <-chan struct{} {
	476	+	stopHard := make(chan struct{})
	477	+	var hardOnce sync.Once
	478	+	hard := func() { hardOnce.Do(func() { close(stopHard) }) }
	479	+	j.hard = hard
	480	+	// after waits grace, then stops the sandbox; it returns early with ctx.
	481	+	after := func(grace time.Duration) {
	482	+	timer := time.NewTimer(grace)
	483	+	defer timer.Stop()
	484	+	select {
	485	+	case <-timer.C:
	486	+	hard()
	487	+	case <-ctx.Done():
	488	+	}
	489	+	}
	490	+	go func() {
	491	+	select {
	492	+	case <-shutdown:
	493	+	case <-ctx.Done():
	494	+	return
	495	+	}
	496	+	j.cancelMu.Lock()
	497	+	j.stopping = true
	498	+	j.cancelMu.Unlock()
	499	+	j.requestCancel("the runner is shutting down", execpb.CancelKind_CANCEL_KIND_SHUTDOWN)
	500	+	after(j.t.ShutdownGrace)
	501	+	}()
	502	+	go func() {
	503	+	wait := time.NewTimer(time.Until(j.deadline()))
	504	+	defer wait.Stop()
	505	+	select {
	506	+	case <-wait.C:
	507	+	case <-ctx.Done():
	508	+	return
	509	+	}
	510	+	j.log.Warn().Int("timeout_minutes", j.timeoutMinutes()).Msg("the job timeout has passed")
	511	+	j.requestCancel("the job timeout has passed", execpb.CancelKind_CANCEL_KIND_TIMEOUT)
	512	+	after(j.t.TimeoutGrace)
	513	+	}()
	514	+	return stopHard
	515	+	}
	516	+
	517	+	// report ends the job towards the server: the log is flushed, then CompleteJob.
	518	+	// Nothing is sent to a job that the server has ended.
	519	+	func (j *jobRun) report(ctx context.Context, out outcome, shutdown <-chan struct{}) {
	520	+	if err := j.abandoned(); err != nil {
	521	+	j.log.Warn().Err(err).Msg("the server ended the job: no result is reported")
	522	+	return
	523	+	}
	524	+	// The report is made even when the runner is shutting down: it uses its own
	525	+	// deadline.
	526	+	budget := j.t.CompleteDeadline
	527	+	select {
	528	+	case <-shutdown:
	529	+	// The service manager waits only so long (see Timing): once the runner's
	530	+	// shutdown channel is closed, whatever the job's outcome, the report gets
	531	+	// what is left of the budget. The channel is read here, not the flag set
	532	+	// by the stop timers, which may not have run yet.
	533	+	budget = min(budget, j.t.ShutdownReport)
	534	+	default:
	535	+	}
	536	+	rctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), budget)
	537	+	defer cancel()
	538	+	// Output limits are checked before the log is flushed: a job whose outputs
	539	+	// the server would refuse fails without outputs, with a line in the log
	540	+	// (runner.proto: CompleteJobRequest.outputs), and lines sent after
	541	+	// CompleteJob are rejected.
	542	+	if msg := checkOutputs(out.outputs); msg != "" {
	543	+	j.sandboxError(msg)
	544	+	out = failure()
	545	+	}
	546	+	j.outbox.Close(rctx)
	547	+	if err := j.abandoned(); err != nil {
	548	+	return
	549	+	}
	550	+	j.complete(rctx, out.conclusion, out.outputs, out.shutdown)
	551	+	}
	552	+
	553	+	// outcome is how a job ended, as far as the host can tell.
	554	+	type outcome struct {
	555	+	conclusion ermite.CiConclusion
	556	+	outputs map[string]string
	557	+	// shutdown is true when the runner's shutdown cut the job short.
	558	+	shutdown bool
	559	+	}
	560	+
	561	+	func failure() outcome { return outcome{conclusion: ermite.CiConclusion_CI_CONCLUSION_FAILURE} }
	562	+
	563	+	// execute starts the sandbox, runs the job in it and returns how it ended.
	564	+	func (j jobRun) execute(ctx context.Context, rt Runner, stopHard <-chan struct{}) outcome {
	565	+	sb, err := rt.Driver.New()
	566	+	if err != nil {
	567	+	j.failEarly(fmt.Sprintf("The runner has no free sandbox: %v", err))
	568	+	return failure()
	569	+	}
	570	+	// Stop runs to completion even when the job context is done.
	571	+	defer j.stopSandbox(ctx, rt, sb)
	572	+
	573	+	conn, err := j.startSandbox(ctx, rt, sb, stopHard)
	574	+	if err != nil {
	575	+	if ctx.Err() != nil \|\| j.cancelRequested() {
	576	+	return outcome{conclusion: ermite.CiConclusion_CI_CONCLUSION_CANCELLED, shutdown: j.isStopping()}
	577	+	}
	578	+	j.failEarly(fmt.Sprintf("The sandbox did not start: %v", err))
	579	+	return failure()
	580	+	}
	581	+	defer func() { _ = conn.Close() }()
	582	+
	583	+	// Hard stops (grace elapsed, or the job was abandoned) close the stream. The
	584	+	// goroutine ends when this function returns, whatever happens to ctx.
	585	+	sessionDone := make(chan struct{})
	586	+	defer close(sessionDone)
	587	+	go func() {
	588	+	select {
	589	+	case <-stopHard:
	590	+	_ = conn.Close()
	591	+	case <-ctx.Done():
	592	+	_ = conn.Close()
	593	+	case <-sessionDone:
	594	+	}
	595	+	}()
	596	+
	597	+	exit, err := j.drive(conn, rt)
	598	+	return j.outcomeOf(ctx, exit, err)
	599	+	}
	600	+
	601	+	// stopSandbox destroys the sandbox; a cleanup that left something behind stops
	602	+	// the runner from taking jobs until it is reaped.
	603	+	func (j jobRun) stopSandbox(ctx context.Context, rt Runner, sb sandbox.Sandbox) {
	604	+	sctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), j.t.StopDeadline)
	605	+	defer cancel()
	606	+	if err := sb.Stop(sctx); err != nil {
	607	+	j.log.Error().Err(err).Msg("stopping the sandbox")
	608	+	if errors.Is(err, sandbox.ErrLeaked) {
	609	+	rt.markLeaked()
	610	+	}
	611	+	}
	612	+	}
	613	+
	614	+	// startSandbox starts the sandbox; the start is bounded by the job context and
	615	+	// by the hard stop.
	616	+	func (j jobRun) startSandbox(ctx context.Context, rt Runner, sb sandbox.Sandbox, stopHard <-chan struct{}) (sandbox.Conn, error) {
	617	+	startCtx, cancelStart := context.WithCancel(ctx)
	618	+	defer cancelStart()
	619	+	stopWatch := make(chan struct{})
	620	+	defer close(stopWatch)
	621	+	go func() {
	622	+	select {
	623	+	case <-stopHard:
	624	+	cancelStart()
	625	+	case <-stopWatch:
	626	+	}
	627	+	}()
	628	+	return sb.Start(startCtx, rt.spec(j.job))
	629	+	}
	630	+
	631	+	// drive sends the job to the executor and handles its frames until it exits.
	632	+	func (j jobRun) drive(conn sandbox.Conn, rt Runner) (*execpb.Exit, error) {
	633	+	run := &execpb.Run{Job: &execpb.JobSpec{
	634	+	WorkflowPath: j.job.WorkflowPath, WorkflowYaml: j.job.WorkflowYaml, JobKey: j.job.JobKey,
	635	+	Matrix: j.job.Matrix, Contexts: j.job.Contexts, Repo: j.job.Repo, TimeoutMinutes: int32(j.timeoutMinutes()),
	636	+	DeadlineMs: max(1, time.Until(j.deadline()).Milliseconds()),
	637	+	RunnerName: rt.Cfg.Name, RunnerVersion: rt.Cfg.Version, RunnerEnvironment: "self-hosted",
	638	+	}}
	639	+	_, exit, err := execstream.DriveJob(conn, run, j.cancelCh, execstream.JobHandler{
	640	+	StepStarted: j.stepStarted, StepCompleted: j.stepCompleted,
	641	+	Log: j.logLine,
	642	+	JobResult: func(r *execpb.JobResult) { j.mu.Lock(); j.result = r; j.mu.Unlock() },
	643	+	})
	644	+	return exit, err
	645	+	}
	646	+
	647	+	// outcomeOf decides how the job ended from what the executor reported and what
	648	+	// the host asked of it.
	649	+	func (j jobRun) outcomeOf(ctx context.Context, exit execpb.Exit, driveErr error) outcome {
	650	+	j.mu.Lock()
	651	+	res := j.result
	652	+	j.mu.Unlock()
	653	+
	654	+	switch {
	655	+	case j.violated():
	656	+	// The step numbers or lines of the executor were wrong: whatever it
	657	+	// reported is not trusted, the job fails (the reason is in the log).
	658	+	return failure()
	659	+	case res != nil:
	660	+	c := conclusionOf(res.Conclusion)
	661	+	if c == ermite.CiConclusion_CI_CONCLUSION_UNSPECIFIED \|\| c == ermite.CiConclusion_CI_CONCLUSION_SKIPPED {
	662	+	c = ermite.CiConclusion_CI_CONCLUSION_FAILURE
	663	+	}
	664	+	if res.Error != "" {
	665	+	j.log.Info().Str("error", res.Error).Msg("the job reported an error")
	666	+	}
	667	+	return outcome{conclusion: c, outputs: res.Outputs, shutdown: res.RunnerShutdown && c == ermite.CiConclusion_CI_CONCLUSION_CANCELLED}
	668	+	case j.cancelRequested() \|\| ctx.Err() != nil:
	669	+	// The executor did not report: it was stopped. A timeout wins over a
	670	+	// shutdown or a user cancel that came later, as on the server (the first
	671	+	// reason decides, and the timeout is the first when it fired first).
	672	+	switch {
	673	+	case j.firstCancelKind() == execpb.CancelKind_CANCEL_KIND_TIMEOUT:
	674	+	return outcome{conclusion: ermite.CiConclusion_CI_CONCLUSION_TIMED_OUT}
	675	+	case j.isStopping():
	676	+	return outcome{conclusion: ermite.CiConclusion_CI_CONCLUSION_CANCELLED, shutdown: true}
	677	+	}
	678	+	return outcome{conclusion: ermite.CiConclusion_CI_CONCLUSION_CANCELLED}
	679	+	}
	680	+	why := "the executor ended without a result"
	681	+	if driveErr != nil {
	682	+	why = driveErr.Error()
	683	+	} else if exit != nil && exit.Error != "" {
	684	+	why = exit.Error
	685	+	}
	686	+	j.sandboxError(fmt.Sprintf("The job's sandbox ended unexpectedly: %s", why))
	687	+	return failure()
	688	+	}
	689	+
	690	+	// timeoutMinutes is the job timeout: at least 1, 360 when the job has none
	691	+	// (runner.proto: Job.timeout_minutes).
	692	+	func (j *jobRun) timeoutMinutes() int {
	693	+	if m := int(j.job.TimeoutMinutes); m > 0 {
	694	+	return m
	695	+	}
	696	+	return 360
	697	+	}
	698	+
	699	+	// deadline is when the job times out, counted from its receipt.
	700	+	func (j *jobRun) deadline() time.Time {
	701	+	return j.receivedAt.Add(time.Duration(j.timeoutMinutes()) * time.Minute)
	702	+	}
	703	+
	704	+	func (j *jobRun) isStopping() bool {
	705	+	j.cancelMu.Lock()
	706	+	defer j.cancelMu.Unlock()
	707	+	return j.stopping
	708	+	}
	709	+
	710	+	// failEarly reports a job that could not start: step 1 fails with the reason
	711	+	// in its log.
	712	+	func (j *jobRun) failEarly(msg string) {
	713	+	now := time.Now()
	714	+	j.stepStarted(&execpb.StepStarted{Number: 1, Name: "Set up job", UnixNano: now.UnixNano()})
	715	+	j.outbox.Log(logLine{step: 1, unixNano: now.UnixNano(), text: "##[error]" + msg})
	716	+	j.stepCompleted(&execpb.StepCompleted{Number: 1, Name: "Set up job", Conclusion: execpb.StepConclusion_STEP_CONCLUSION_FAILURE, UnixNano: time.Now().UnixNano()})
	717	+	}
	718	+
	719	+	// sandboxError writes the reason into the log of the step that was running.
	720	+	func (j *jobRun) sandboxError(msg string) {
	721	+	j.mu.Lock()
	722	+	var last int32 = 1
	723	+	for n := range j.steps {
	724	+	last = max(last, n)
	725	+	}
	726	+	j.mu.Unlock()
	727	+	j.outbox.Log(logLine{step: last, unixNano: time.Now().UnixNano(), text: "##[error]" + msg})
	728	+	}
	729	+
	730	+	// complete reports the end of the job. CompleteJob is idempotent, so a lost
	731	+	// response is retried until it is answered (applied or not).
	732	+	func (j *jobRun) complete(ctx context.Context, conclusion ermite.CiConclusion, outputs map[string]string, shutdown bool) {
	733	+	req := &ermite.CompleteJobRequest{
	734	+	JobId: j.job.JobId, Conclusion: conclusion, Outputs: outputs,
	735	+	RunnerShutdown: shutdown && conclusion == ermite.CiConclusion_CI_CONCLUSION_CANCELLED,
	736	+	Steps: j.finalSteps(conclusion),
	737	+	}
	738	+	err := j.sendComplete(ctx, req, conclusion)
	739	+	if err == nil \|\| ctx.Err() != nil \|\| classify(err) != kindPermanent {
	740	+	return
	741	+	}
	742	+	// The server refused the result itself (invalid_argument, or
	743	+	// failed_precondition that is not the end of the job). CompleteJob is
	744	+	// "applied whole or not at all", so the job is still open: say why in the
	745	+	// log, then report a plain failure without outputs and with steps that are
	746	+	// valid by construction. Without this the job would stay running until the
	747	+	// heartbeat deadline of the server and then be reported as a lost runner.
	748	+	j.errorLog(ctx, fmt.Sprintf("The runner could not report the result of the job: %s. The job is reported as failed.", connect.CodeOf(err)))
	749	+	j.log.Error().Err(err).Msg("CompleteJob refused: retrying as a failure without outputs")
	750	+	req = &ermite.CompleteJobRequest{
	751	+	JobId: j.job.JobId, Conclusion: ermite.CiConclusion_CI_CONCLUSION_FAILURE,
	752	+	Steps: j.rebuiltSteps(),
	753	+	}
	754	+	if err := j.sendComplete(ctx, req, req.Conclusion); err != nil && ctx.Err() == nil {
	755	+	j.log.Error().Err(err).Msg("CompleteJob refused again, giving up")
	756	+	}
	757	+	}
	758	+
	759	+	// sendComplete sends the request until the server answers: success, the end of
	760	+	// the job, a lost assignment, or a refusal (returned).
	761	+	func (j jobRun) sendComplete(ctx context.Context, req ermite.CompleteJobRequest, conclusion ermite.CiConclusion) error {
	762	+	b := backoff{min: j.t.BackoffMin, max: j.t.BackoffMax}
	763	+	for {
	764	+	cctx, cancel := context.WithTimeout(ctx, j.t.CallTimeout)
	765	+	resp, err := j.client.CompleteJob(cctx, connect.NewRequest(req))
	766	+	cancel()
	767	+	if err == nil {
	768	+	if resp.Msg.Conclusion != conclusion {
	769	+	j.log.Warn().Str("reported", conclusion.String()).Str("stored", resp.Msg.Conclusion.String()).Msg("the server holds another conclusion")
	770	+	}
	771	+	return nil
	772	+	}
	773	+	if ctx.Err() != nil {
	774	+	j.log.Error().Err(ctx.Err()).Msg("could not report the result")
	775	+	return err
	776	+	}
	777	+	switch classify(err) {
	778	+	case kindEnded, kindAbandoned:
	779	+	return nil
	780	+	case kindPermanent:
	781	+	return err
	782	+	}
	783	+	j.log.Warn().Err(err).Msg("CompleteJob failed, retrying")
	784	+	if !sleep(ctx, b.next()) {
	785	+	return err
	786	+	}
	787	+	}
	788	+	}
	789	+
	790	+	// errorLog appends a line to the log after the outbox is closed.
	791	+	func (j *jobRun) errorLog(ctx context.Context, msg string) {
	792	+	j.mu.Lock()
	793	+	var last int32 = 1
	794	+	for n := range j.steps {
	795	+	last = max(last, n)
	796	+	}
	797	+	j.mu.Unlock()
	798	+	j.outbox.AppendNow(ctx, logLine{step: last, unixNano: time.Now().UnixNano(), text: "##[error]" + msg})
	799	+	}
	800	+
	801	+	// rebuiltSteps returns every known step as a valid, completed snapshot: number
	802	+	// in 1..maxSteps, a name of at most 1,024 bytes, a conclusion the server
	803	+	// accepts (success, failure, cancelled, skipped), both times set, and no step
	804	+	// that finished before it began. A step that was running is failed.
	805	+	func (j jobRun) rebuiltSteps() []ermite.StepUpdate {
	806	+	j.mu.Lock()
	807	+	defer j.mu.Unlock()
	808	+	now := timestamppb.Now()
	809	+	var out []*ermite.StepUpdate
	810	+	for n, u := range j.steps {
	811	+	if n < 1 \|\| n > maxSteps {
	812	+	continue
	813	+	}
	814	+	c := proto.Clone(u).(*ermite.StepUpdate)
	815	+	c.Name = cutName(c.Name)
	816	+	if c.StartedAt == nil {
	817	+	c.StartedAt = now
	818	+	}
	819	+	if c.State != ermite.StepState_STEP_STATE_COMPLETED {
	820	+	c.Conclusion = ermite.CiConclusion_CI_CONCLUSION_FAILURE
	821	+	c.CompletedAt = now
	822	+	}
	823	+	switch c.Conclusion {
	824	+	case ermite.CiConclusion_CI_CONCLUSION_SUCCESS, ermite.CiConclusion_CI_CONCLUSION_FAILURE,
	825	+	ermite.CiConclusion_CI_CONCLUSION_CANCELLED, ermite.CiConclusion_CI_CONCLUSION_SKIPPED:
	826	+	default:
	827	+	c.Conclusion = ermite.CiConclusion_CI_CONCLUSION_FAILURE
	828	+	}
	829	+	c.State = ermite.StepState_STEP_STATE_COMPLETED
	830	+	if c.CompletedAt == nil \|\| c.CompletedAt.AsTime().Before(c.StartedAt.AsTime()) {
	831	+	c.CompletedAt = c.StartedAt
	832	+	}
	833	+	out = append(out, c)
	834	+	}
	835	+	sort.Slice(out, func(a, b int) bool { return out[a].Number < out[b].Number })
	836	+	return out
	837	+	}
	838	+
	839	+	// Limits of CompleteJobRequest.outputs.
	840	+	const (
	841	+	maxOutputBytes = 1 << 20
	842	+	maxOutputName = 255
	843	+	)
	844	+
	845	+	// checkOutputs returns a message when the server would refuse the outputs: a
	846	+	// name of 1 to 255 letters, digits, "_" and "-" starting with a letter or "_",
	847	+	// no two names equal ignoring case, each value and the sum of names and values
	848	+	// at most 1 MiB.
	849	+	func checkOutputs(outputs map[string]string) string {
	850	+	total := 0
	851	+	seen := map[string]string{}
	852	+	names := make([]string, 0, len(outputs))
	853	+	for n := range outputs {
	854	+	names = append(names, n)
	855	+	}
	856	+	sort.Strings(names)
	857	+	for _, n := range names {
	858	+	v := outputs[n]
	859	+	if !validOutputName(n) {
	860	+	return fmt.Sprintf("Job output name %q is not valid: use 1 to 255 letters, digits, '_' and '-', starting with a letter or '_'.", n)
	861	+	}
	862	+	if prev, dup := seen[strings.ToLower(n)]; dup {
	863	+	return fmt.Sprintf("Job outputs %q and %q differ only by case.", prev, n)
	864	+	}
	865	+	seen[strings.ToLower(n)] = n
	866	+	if len(v) > maxOutputBytes {
	867	+	return fmt.Sprintf("Job output %q is larger than 1 MiB.", n)
	868	+	}
	869	+	total += len(n) + len(v)
	870	+	}
	871	+	if total > maxOutputBytes {
	872	+	return "The job outputs are larger than 1 MiB in total."
	873	+	}
	874	+	return ""
	875	+	}
	876	+
	877	+	func validOutputName(n string) bool {
	878	+	if n == "" \|\| len(n) > maxOutputName {
	879	+	return false
	880	+	}
	881	+	for i, r := range n {
	882	+	letter := r >= 'a' && r <= 'z' \|\| r >= 'A' && r <= 'Z' \|\| r == '_'
	883	+	if ok := letter \|\| (i > 0 && (r >= '0' && r <= '9' \|\| r == '-')); !ok {
	884	+	return false
	885	+	}
	886	+	}
	887	+	return true
	888	+	}

A internal/host/outbox.go +325 -0

@@ -0,0 +1,325 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package host
	4	+
	5	+	import (
	6	+	"context"
	7	+	"sync"
	8	+	"time"
	9	+
	10	+	"connectrpc.com/connect"
	11	+	"github.com/bnema/zerowrap"
	12	+	"google.golang.org/protobuf/types/known/timestamppb"
	13	+
	14	+	ermite "bnema.dev/ermite-api/gen/ermite"
	15	+	)
	16	+
	17	+	// Limits of AppendLog (runner.proto): at most 1,000 lines and 64 KiB of text
	18	+	// per request, all in the same step.
	19	+	const (
	20	+	maxBatchLines = 1000
	21	+	maxBatchBytes = 64 * 1024
	22	+	// maxQueuedBytes is the log kept in memory while the server cannot be
	23	+	// reached; beyond it the executor is slowed down (the stream is not read)
	24	+	// rather than the host's memory filled.
	25	+	maxQueuedBytes = 16 << 20
	26	+	)
	27	+
	28	+	type logLine struct {
	29	+	step int32
	30	+	unixNano int64
	31	+	text string
	32	+	continued bool
	33	+	}
	34	+
	35	+	// event is something to send, in the order it happened: a log line, or a
	36	+	// snapshot of a step (UpdateJob).
	37	+	type event struct {
	38	+	line *logLine
	39	+	step *ermite.StepUpdate
	40	+	}
	41	+
	42	+	// outbox sends logs and step updates of one job to ermite, in order, one call
	43	+	// at a time. Log lines are numbered from 0 across the whole job; the number of
	44	+	// the next line is the offset of the next AppendLog, so that a retry after a
	45	+	// lost response sends the same lines at the same offset (the server skips what
	46	+	// it already holds).
	47	+	type outbox struct {
	48	+	client callClient
	49	+	jobID string
	50	+	t Timing
	51	+	// onEnded is called once when the server says the job is over (JobEnded)
	52	+	// or the assignment is gone; the outbox then drops everything.
	53	+	onEnded func(error)
	54	+	log zerowrap.Logger
	55	+
	56	+	mu sync.Mutex
	57	+	cond *sync.Cond
	58	+	events []event
	59	+	queued int // bytes of log text in events
	60	+	closing bool
	61	+	dead bool
	62	+	done chan struct{}
	63	+
	64	+	offset int64 // lines the server holds, as far as we know
	65	+	// offsetMaybeStale is set when a send was cut short by its context while
	66	+	// the call may have been applied by the server with the response lost: the
	67	+	// offset is then possibly behind the server's count.
	68	+	offsetMaybeStale bool
	69	+	}
	70	+
	71	+	// callClient is the part of the RunnerService client the job uses.
	72	+	type callClient interface {
	73	+	UpdateJob(context.Context, connect.Request[ermite.UpdateJobRequest]) (connect.Response[ermite.UpdateJobResponse], error)
	74	+	AppendLog(context.Context, connect.Request[ermite.AppendLogRequest]) (connect.Response[ermite.AppendLogResponse], error)
	75	+	}
	76	+
	77	+	func newOutbox(c callClient, jobID string, t Timing, onEnded func(error), log zerowrap.Logger) *outbox {
	78	+	o := &outbox{client: c, jobID: jobID, t: t, onEnded: onEnded, log: log, done: make(chan struct{})}
	79	+	o.cond = sync.NewCond(&o.mu)
	80	+	return o
	81	+	}
	82	+
	83	+	// start runs the sender until Close or until ctx ends.
	84	+	func (o *outbox) start(ctx context.Context) {
	85	+	go func() {
	86	+	defer close(o.done)
	87	+	o.loop(ctx)
	88	+	}()
	89	+	// A cancelled context must wake the loop and the producers.
	90	+	go func() {
	91	+	select {
	92	+	case <-ctx.Done():
	93	+	o.kill()
	94	+	case <-o.done:
	95	+	}
	96	+	}()
	97	+	}
	98	+
	99	+	func (o *outbox) kill() {
	100	+	o.mu.Lock()
	101	+	o.dead = true
	102	+	o.events = nil
	103	+	o.queued = 0
	104	+	o.mu.Unlock()
	105	+	o.cond.Broadcast()
	106	+	}
	107	+
	108	+	// Log queues a line of the job log. It blocks while too much is queued.
	109	+	func (o *outbox) Log(l logLine) {
	110	+	o.mu.Lock()
	111	+	for o.queued > maxQueuedBytes && !o.dead && !o.closing {
	112	+	o.cond.Wait()
	113	+	}
	114	+	if o.dead {
	115	+	o.mu.Unlock()
	116	+	return
	117	+	}
	118	+	o.events = append(o.events, event{line: &l})
	119	+	o.queued += len(l.text) + 32
	120	+	o.mu.Unlock()
	121	+	o.cond.Broadcast()
	122	+	}
	123	+
	124	+	// Step queues a step snapshot.
	125	+	func (o outbox) Step(s ermite.StepUpdate) {
	126	+	o.mu.Lock()
	127	+	if o.dead {
	128	+	o.mu.Unlock()
	129	+	return
	130	+	}
	131	+	o.events = append(o.events, event{step: s})
	132	+	o.mu.Unlock()
	133	+	o.cond.Broadcast()
	134	+	}
	135	+
	136	+	// Close sends what is queued, then stops the sender. It waits at most until
	137	+	// ctx ends.
	138	+	func (o *outbox) Close(ctx context.Context) {
	139	+	o.mu.Lock()
	140	+	o.closing = true
	141	+	o.mu.Unlock()
	142	+	o.cond.Broadcast()
	143	+	select {
	144	+	case <-o.done:
	145	+	case <-ctx.Done():
	146	+	o.kill()
	147	+	<-o.done
	148	+	}
	149	+	}
	150	+
	151	+	// Lines returns how many lines the server holds (as far as we know).
	152	+	func (o *outbox) Lines() int64 {
	153	+	o.mu.Lock()
	154	+	defer o.mu.Unlock()
	155	+	return o.offset
	156	+	}
	157	+
	158	+	// next waits for a batch of events. It returns nil when the outbox is done.
	159	+	func (o *outbox) next(ctx context.Context) []event {
	160	+	o.mu.Lock()
	161	+	defer o.mu.Unlock()
	162	+	for len(o.events) == 0 {
	163	+	if o.dead \|\| o.closing {
	164	+	return nil
	165	+	}
	166	+	o.cond.Wait()
	167	+	}
	168	+	// Let a burst of lines fill the batch, unless it is full, a step event is
	169	+	// at the front, or we are closing.
	170	+	if o.events[0].line != nil && !o.closing && o.queued < maxBatchBytes && len(o.events) < maxBatchLines {
	171	+	o.mu.Unlock()
	172	+	sleep(ctx, o.t.LogFlush)
	173	+	o.mu.Lock()
	174	+	if o.dead {
	175	+	return nil
	176	+	}
	177	+	}
	178	+	if len(o.events) == 0 {
	179	+	return nil
	180	+	}
	181	+	first := o.events[0]
	182	+	if first.step != nil {
	183	+	o.events = o.events[1:]
	184	+	return []event{first}
	185	+	}
	186	+	// A run of lines of the same step, within the limits.
	187	+	n, bytes := 0, 0
	188	+	step := first.line.step
	189	+	for _, e := range o.events {
	190	+	if e.line == nil \|\| e.line.step != step \|\| n == maxBatchLines \|\| (n > 0 && bytes+len(e.line.text) > maxBatchBytes) {
	191	+	break
	192	+	}
	193	+	n++
	194	+	bytes += len(e.line.text)
	195	+	}
	196	+	batch := append([]event(nil), o.events[:n]...)
	197	+	o.events = o.events[n:]
	198	+	for _, e := range batch {
	199	+	o.queued -= len(e.line.text) + 32
	200	+	}
	201	+	o.cond.Broadcast() // producers waiting for room
	202	+	return batch
	203	+	}
	204	+
	205	+	func (o *outbox) loop(ctx context.Context) {
	206	+	for {
	207	+	batch := o.next(ctx)
	208	+	if batch == nil {
	209	+	return
	210	+	}
	211	+	if batch[0].step != nil {
	212	+	o.sendStep(ctx, batch[0].step)
	213	+	} else {
	214	+	o.sendLines(ctx, batch)
	215	+	}
	216	+	o.mu.Lock()
	217	+	dead := o.dead
	218	+	o.mu.Unlock()
	219	+	if dead {
	220	+	return
	221	+	}
	222	+	}
	223	+	}
	224	+
	225	+	// retry runs call until it succeeds, the server gives a verdict, or ctx ends.
	226	+	// It returns nil on success.
	227	+	func (o *outbox) retry(ctx context.Context, what string, call func(context.Context) error) error {
	228	+	b := backoff{min: o.t.BackoffMin, max: o.t.BackoffMax}
	229	+	for {
	230	+	cctx, cancel := context.WithTimeout(ctx, o.t.CallTimeout)
	231	+	err := call(cctx)
	232	+	cancel()
	233	+	if err == nil {
	234	+	return nil
	235	+	}
	236	+	if ctx.Err() != nil {
	237	+	return ctx.Err()
	238	+	}
	239	+	switch classify(err) {
	240	+	case kindEnded, kindAbandoned:
	241	+	o.onEnded(err)
	242	+	o.kill()
	243	+	return err
	244	+	case kindPermanent:
	245	+	return err
	246	+	}
	247	+	o.log.Warn().Err(err).Str("call", what).Msg("call failed, retrying")
	248	+	if !sleep(ctx, b.next()) {
	249	+	return ctx.Err()
	250	+	}
	251	+	}
	252	+	}
	253	+
	254	+	func (o outbox) sendStep(ctx context.Context, s ermite.StepUpdate) {
	255	+	err := o.retry(ctx, "UpdateJob", func(c context.Context) error {
	256	+	_, err := o.client.UpdateJob(c, connect.NewRequest(&ermite.UpdateJobRequest{JobId: o.jobID, Steps: []*ermite.StepUpdate{s}}))
	257	+	return err
	258	+	})
	259	+	if err != nil && ctx.Err() == nil && classify(err) == kindPermanent {
	260	+	o.log.Error().Err(err).Int32("step", s.Number).Msg("UpdateJob refused")
	261	+	}
	262	+	}
	263	+
	264	+	func (o *outbox) sendLines(ctx context.Context, batch []event) {
	265	+	req := &ermite.AppendLogRequest{JobId: o.jobID, Step: batch[0].line.step}
	266	+	for _, e := range batch {
	267	+	req.Lines = append(req.Lines, &ermite.LogLine{Time: timestamppb.New(time.Unix(0, e.line.unixNano)), Text: e.line.text, Continued: e.line.continued})
	268	+	}
	269	+	// The offset is fixed before the first attempt: every retry sends the same
	270	+	// lines at the same offset.
	271	+	req.Offset = o.offset
	272	+	var next int64
	273	+	err := o.retry(ctx, "AppendLog", func(c context.Context) error {
	274	+	resp, err := o.client.AppendLog(c, connect.NewRequest(req))
	275	+	if err != nil {
	276	+	return err
	277	+	}
	278	+	next = resp.Msg.NextOffset
	279	+	return nil
	280	+	})
	281	+	switch {
	282	+	case err == nil:
	283	+	o.mu.Lock()
	284	+	o.offsetMaybeStale = false
	285	+	// The server says how many lines it holds: that is the truth. A
	286	+	// difference means lines were stored that we did not count (a retry
	287	+	// whose answer we lost) or dropped; continue from the server's count,
	288	+	// so that the next offset is never a gap (out_of_range) and never a
	289	+	// repeat of lines that would be skipped.
	290	+	if want := req.Offset + int64(len(req.Lines)); next != want {
	291	+	o.log.Warn().Int64("server_lines", next).Int64("expected", want).Msg("AppendLog: the server holds another number of lines, continuing from its count")
	292	+	}
	293	+	o.offset = next
	294	+	o.mu.Unlock()
	295	+	case ctx.Err() != nil:
	296	+	// Cut short in the middle of a call or of its retries: the server may
	297	+	// hold these lines without our having seen the answer.
	298	+	o.mu.Lock()
	299	+	o.offsetMaybeStale = true
	300	+	o.mu.Unlock()
	301	+	case classify(err) == kindPermanent:
	302	+	// The server refused these lines; they are dropped and the offset stays
	303	+	// where the server is.
	304	+	o.log.Error().Err(err).Int("lines", len(req.Lines)).Int32("step", req.Step).Msg("AppendLog refused, the lines are dropped")
	305	+	}
	306	+	}
	307	+
	308	+	// AppendNow sends one line synchronously, after Close: the sender has stopped,
	309	+	// so nothing else uses the offset. It is for the last words of a job (an error
	310	+	// that explains why the result could not be reported).
	311	+	//
	312	+	// If Close gave up in the middle of a send, the offset may be behind the
	313	+	// server's: the line is sent anyway (the server answers with its own count, and
	314	+	// an offset that is too low makes it skip the line as a repeat or refuse it),
	315	+	// and the warning says why it may be missing.
	316	+	func (o *outbox) AppendNow(ctx context.Context, l logLine) {
	317	+	o.mu.Lock()
	318	+	stale := o.offsetMaybeStale
	319	+	offset := o.offset
	320	+	o.mu.Unlock()
	321	+	if stale {
	322	+	o.log.Warn().Int64("offset", offset).Msg("the log offset may be stale: Close was cut short in the middle of a send, so the last line may be skipped or refused by the server")
	323	+	}
	324	+	o.sendLines(ctx, []event{{line: &l}})
	325	+	}

A internal/host/outbox_test.go +45 -0

@@ -0,0 +1,45 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package host
	4	+
	5	+	import (
	6	+	"context"
	7	+	"strings"
	8	+	"testing"
	9	+	"time"
	10	+
	11	+	"github.com/bnema/zerowrap"
	12	+	)
	13	+
	14	+	// AppendNow says so when an earlier send was cut short and the offset may be
	15	+	// behind the server's (GO-005), and stays silent otherwise.
	16	+	func TestAppendNowWarnsOfAStaleOffset(t *testing.T) {
	17	+	const warning = "log offset may be stale"
	18	+	logs := &logBuf{}
	19	+	log := zerowrap.New(zerowrap.Config{Level: "debug", Format: "json", Output: logs})
	20	+	tm := Timing{CallTimeout: time.Hour, BackoffMin: time.Millisecond, BackoffMax: time.Millisecond}
	21	+	cl := &stubClient{hang: true}
	22	+	o := newOutbox(cl, "j1", tm, func(error) {}, log)
	23	+
	24	+	// A send that Close cut short: the server may have applied it.
	25	+	ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
	26	+	defer cancel()
	27	+	o.sendLines(ctx, []event{{line: &logLine{step: 1, text: "a"}}})
	28	+
	29	+	cl.mu.Lock()
	30	+	cl.hang = false
	31	+	cl.mu.Unlock()
	32	+	o.AppendNow(context.Background(), logLine{step: 1, text: "b"})
	33	+	if !strings.Contains(logs.String(), warning) {
	34	+	t.Fatalf("no warning about the offset: %q", logs.String())
	35	+	}
	36	+
	37	+	// The send went through: the offset is the server's again.
	38	+	logs.mu.Lock()
	39	+	logs.b.Reset()
	40	+	logs.mu.Unlock()
	41	+	o.AppendNow(context.Background(), logLine{step: 1, text: "c"})
	42	+	if strings.Contains(logs.String(), warning) {
	43	+	t.Fatalf("a warning with a fresh offset: %q", logs.String())
	44	+	}
	45	+	}

A internal/host/rpc.go +258 -0

@@ -0,0 +1,258 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	// Package host is the host side of ermite-runner: it registers the runner with
	4	+	// ermite, fetches jobs, runs each one in a sandbox through the executor, and
	5	+	// reports logs, steps and the result with the job token. The sandbox never sees
	6	+	// a token.
	7	+	package host
	8	+
	9	+	import (
	10	+	"context"
	11	+	"errors"
	12	+	"fmt"
	13	+	"net/http"
	14	+	"net/netip"
	15	+	"net/url"
	16	+	"strings"
	17	+	"time"
	18	+
	19	+	"connectrpc.com/connect"
	20	+
	21	+	ermite "bnema.dev/ermite-api/gen/ermite"
	22	+	"bnema.dev/ermite-api/gen/ermite/ermiteconnect"
	23	+	)
	24	+
	25	+	// bearer adds "Authorization: Bearer <token>" to every request. The token is
	26	+	// never logged. It refuses to send the token over a connection that is not
	27	+	// protected: plain http is only for loopback, unless AllowInsecure is set.
	28	+	type bearer struct {
	29	+	token string
	30	+	base http.RoundTripper
	31	+	allowInsecure bool
	32	+	}
	33	+
	34	+	// ErrInsecureURL is returned for a server URL that is not https and not
	35	+	// loopback: the runner token and the job tokens would travel in clear.
	36	+	var ErrInsecureURL = errors.New("the server URL must be https (plain http is only accepted for localhost; ERMITE_RUNNER_INSECURE_HTTP=1 allows it, for tests, and sends every token in clear)")
	37	+
	38	+	// CheckURL validates the URL of the server: http or https, a host, no
	39	+	// credentials, and https unless the host is loopback or allowInsecure is set.
	40	+	func CheckURL(raw string, allowInsecure bool) error {
	41	+	u, err := url.Parse(raw)
	42	+	if err != nil \|\| u.Host == "" {
	43	+	return fmt.Errorf("%q is not a valid URL", raw)
	44	+	}
	45	+	if u.User != nil {
	46	+	return errors.New("the server URL must not contain credentials")
	47	+	}
	48	+	switch u.Scheme {
	49	+	case "https":
	50	+	return nil
	51	+	case "http":
	52	+	if allowInsecure \|\| isLoopback(u.Hostname()) {
	53	+	return nil
	54	+	}
	55	+	return ErrInsecureURL
	56	+	}
	57	+	return fmt.Errorf("%q: the scheme must be https", raw)
	58	+	}
	59	+
	60	+	func isLoopback(host string) bool {
	61	+	// Host names are not case sensitive (LOCALHOST, Localhost).
	62	+	if strings.EqualFold(host, "localhost") {
	63	+	return true
	64	+	}
	65	+	ip, err := netip.ParseAddr(host)
	66	+	return err == nil && ip.IsLoopback()
	67	+	}
	68	+
	69	+	func (b bearer) RoundTrip(r http.Request) (http.Response, error) {
	70	+	if err := CheckURL(r.URL.String(), b.allowInsecure); err != nil {
	71	+	return nil, err
	72	+	}
	73	+	r = r.Clone(r.Context())
	74	+	if b.token != "" {
	75	+	r.Header.Set("Authorization", "Bearer "+b.token)
	76	+	}
	77	+	base := b.base
	78	+	if base == nil {
	79	+	base = http.DefaultTransport
	80	+	}
	81	+	return base.RoundTrip(r)
	82	+	}
	83	+
	84	+	// NewClient returns a RunnerService client that authenticates with token. The
	85	+	// HTTP client has no overall timeout: calls carry their own context deadline
	86	+	// (WatchJob is a long stream). It never follows a redirect: a response from the
	87	+	// server that points elsewhere is returned as is, so that the token is not
	88	+	// carried to another host (Go would keep the Authorization header on a redirect
	89	+	// to the same host and drop it for others, but the client has no business
	90	+	// following any).
	91	+	func NewClient(baseURL, token string, base http.RoundTripper) ermiteconnect.RunnerServiceClient {
	92	+	return newClient(baseURL, token, base, false)
	93	+	}
	94	+
	95	+	// NewInsecureClient is NewClient for a server reached over plain http that is
	96	+	// not loopback. Only for tests and trusted networks.
	97	+	func NewInsecureClient(baseURL, token string, base http.RoundTripper) ermiteconnect.RunnerServiceClient {
	98	+	return newClient(baseURL, token, base, true)
	99	+	}
	100	+
	101	+	func newClient(baseURL, token string, base http.RoundTripper, insecure bool) ermiteconnect.RunnerServiceClient {
	102	+	hc := &http.Client{
	103	+	Transport: bearer{token: token, base: base, allowInsecure: insecure},
	104	+	CheckRedirect: func(http.Request, []http.Request) error { return http.ErrUseLastResponse },
	105	+	}
	106	+	return ermiteconnect.NewRunnerServiceClient(hc, strings.TrimRight(baseURL, "/"))
	107	+	}
	108	+
	109	+	// ClientFactory builds a client for a token; tests replace it.
	110	+	type ClientFactory func(token string) ermiteconnect.RunnerServiceClient
	111	+
	112	+	// jobEnded extracts the JobEnded detail of a failed_precondition error.
	113	+	func jobEnded(err error) (*ermite.JobEnded, bool) {
	114	+	var ce *connect.Error
	115	+	if !errors.As(err, &ce) \|\| ce.Code() != connect.CodeFailedPrecondition {
	116	+	return nil, false
	117	+	}
	118	+	for _, d := range ce.Details() {
	119	+	m, derr := d.Value()
	120	+	if derr != nil {
	121	+	continue
	122	+	}
	123	+	if je, ok := m.(*ermite.JobEnded); ok {
	124	+	return je, true
	125	+	}
	126	+	}
	127	+	return nil, false
	128	+	}
	129	+
	130	+	// kind classifies the error of a job call.
	131	+	type kind int
	132	+
	133	+	const (
	134	+	kindRetry kind = iota // network error, unavailable, internal...: try again
	135	+	kindEnded // failed_precondition with JobEnded: stop, send nothing more
	136	+	kindAbandoned // unauthenticated: the assignment is gone, stop at once
	137	+	kindPermanent // the request itself is wrong: do not repeat it
	138	+	)
	139	+
	140	+	func classify(err error) kind {
	141	+	if err == nil {
	142	+	return kindRetry
	143	+	}
	144	+	if _, ok := jobEnded(err); ok {
	145	+	return kindEnded
	146	+	}
	147	+	switch connect.CodeOf(err) {
	148	+	case connect.CodeUnauthenticated:
	149	+	return kindAbandoned
	150	+	case connect.CodeInvalidArgument, connect.CodeFailedPrecondition, connect.CodeOutOfRange,
	151	+	connect.CodePermissionDenied, connect.CodeNotFound, connect.CodeUnimplemented, connect.CodeAlreadyExists:
	152	+	return kindPermanent
	153	+	}
	154	+	// Canceled by our own context is handled by the callers through ctx.Err().
	155	+	return kindRetry
	156	+	}
	157	+
	158	+	// Timing holds the delays of the host loop. Zero fields take the defaults,
	159	+	// which are the protocol's.
	160	+	type Timing struct {
	161	+	// PollTimeout bounds one FetchJob call (the server holds it up to 50 s).
	162	+	PollTimeout time.Duration
	163	+	// HeartbeatEvery is the heartbeat period (30 s); HeartbeatRetry the delay
	164	+	// after a failed heartbeat (5 s).
	165	+	HeartbeatEvery, HeartbeatRetry time.Duration
	166	+	// CallTimeout bounds one unary job call.
	167	+	CallTimeout time.Duration
	168	+	// BackoffMin and BackoffMax bound the retry delay of calls that fail
	169	+	// without a verdict.
	170	+	BackoffMin, BackoffMax time.Duration
	171	+	// LogFlush is how long log lines are held to fill a batch.
	172	+	LogFlush time.Duration
	173	+	// WatchRetry is the delay before WatchJob is reopened.
	174	+	WatchRetry time.Duration
	175	+	// TimeoutGrace is how long the executor gets after the job timeout to stop
	176	+	// its steps (SIGINT, then SIGTERM after 7.5 s, then SIGKILL 2.5 s later) and
	177	+	// report, before the sandbox is destroyed.
	178	+	TimeoutGrace time.Duration
	179	+	// ShutdownGrace is how long jobs get to stop after a shutdown signal before
	180	+	// their sandbox is destroyed. The executor needs 10 s for its escalation
	181	+	// (SIGINT, SIGTERM after 7.5 s, SIGKILL 2.5 s later) and some for the
	182	+	// post-steps and the report to the host.
	183	+	ShutdownGrace time.Duration
	184	+	// StopDeadline bounds the destruction of a sandbox.
	185	+	StopDeadline time.Duration
	186	+	// ShutdownReport bounds the last flush of the log and CompleteJob of a job
	187	+	// that a shutdown cut short.
	188	+	ShutdownReport time.Duration
	189	+	// CompleteDeadline bounds the flush of logs and CompleteJob at the end of
	190	+	// a job.
	191	+	CompleteDeadline time.Duration
	192	+	// NoJobPause is the pause after a poll without a job and without
	193	+	// retry_after.
	194	+	NoJobPause time.Duration
	195	+	// ReapRetry is the delay between attempts to reap after a leaked sandbox.
	196	+	ReapRetry time.Duration
	197	+	}
	198	+
	199	+	func (t Timing) withDefaults() Timing {
	200	+	def := func(d *time.Duration, v time.Duration) {
	201	+	if *d == 0 {
	202	+	*d = v
	203	+	}
	204	+	}
	205	+	def(&t.PollTimeout, 60*time.Second)
	206	+	def(&t.HeartbeatEvery, 30*time.Second)
	207	+	def(&t.HeartbeatRetry, 5*time.Second)
	208	+	def(&t.CallTimeout, 30*time.Second)
	209	+	def(&t.BackoffMin, time.Second)
	210	+	def(&t.BackoffMax, 30*time.Second)
	211	+	def(&t.LogFlush, 250*time.Millisecond)
	212	+	def(&t.WatchRetry, time.Second)
	213	+	def(&t.TimeoutGrace, 30*time.Second)
	214	+	def(&t.ShutdownGrace, 30*time.Second)
	215	+	def(&t.StopDeadline, 30*time.Second)
	216	+	def(&t.ShutdownReport, 30*time.Second)
	217	+	def(&t.CompleteDeadline, 5*time.Minute)
	218	+	def(&t.NoJobPause, 0)
	219	+	def(&t.ReapRetry, 5*time.Second)
	220	+	return t
	221	+	}
	222	+
	223	+	// ShutdownBudget is the longest a graceful shutdown takes with these timings:
	224	+	// the grace for the executor, the destruction of the sandbox, and the report.
	225	+	// The jobs shut down in parallel, so it does not depend on their number. The
	226	+	// service manager must wait at least this long (contrib/ermite-runner.service
	227	+	// has TimeoutStopSec=120 for the defaults, 90 s).
	228	+	func (t Timing) ShutdownBudget() time.Duration {
	229	+	t = t.withDefaults()
	230	+	return t.ShutdownGrace + t.StopDeadline + t.ShutdownReport
	231	+	}
	232	+
	233	+	// sleep waits d or until ctx ends; it reports whether the full delay passed.
	234	+	func sleep(ctx context.Context, d time.Duration) bool {
	235	+	if d <= 0 {
	236	+	return ctx.Err() == nil
	237	+	}
	238	+	t := time.NewTimer(d)
	239	+	defer t.Stop()
	240	+	select {
	241	+	case <-t.C:
	242	+	return true
	243	+	case <-ctx.Done():
	244	+	return false
	245	+	}
	246	+	}
	247	+
	248	+	// backoff doubles from min to max.
	249	+	type backoff struct{ min, max, cur time.Duration }
	250	+
	251	+	func (b *backoff) next() time.Duration {
	252	+	if b.cur == 0 {
	253	+	b.cur = b.min
	254	+	} else if b.cur *= 2; b.cur > b.max {
	255	+	b.cur = b.max
	256	+	}
	257	+	return b.cur
	258	+	}

A internal/host/rpc_test.go +136 -0

@@ -0,0 +1,136 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package host
	4	+
	5	+	import (
	6	+	"context"
	7	+	"net/http"
	8	+	"net/http/httptest"
	9	+	"strings"
	10	+	"sync/atomic"
	11	+	"testing"
	12	+
	13	+	"connectrpc.com/connect"
	14	+
	15	+	ermite "bnema.dev/ermite-api/gen/ermite"
	16	+	)
	17	+
	18	+	func TestCheckURL(t *testing.T) {
	19	+	for _, tc := range []struct {
	20	+	url string
	21	+	insecure bool
	22	+	want string // "" = accepted
	23	+	}{
	24	+	{"https://ermite.example.org", false, ""},
	25	+	{"https://ermite.example.org:8443/base", false, ""},
	26	+	{"http://localhost:8080", false, ""},
	27	+	{"http://LOCALHOST:8080", false, ""},
	28	+	{"http://127.0.0.1:8080", false, ""},
	29	+	{"http://[::1]:8080", false, ""},
	30	+	{"http://ermite.example.org", false, "must be https"},
	31	+	{"http://10.0.0.5:8080", false, "must be https"},
	32	+	{"http://localhost.evil.example", false, "must be https"},
	33	+	{"http://127.0.0.1.evil.example", false, "must be https"},
	34	+	{"http://ermite.example.org", true, ""},
	35	+	{"ftp://ermite.example.org", false, "scheme must be https"},
	36	+	{"ermite.example.org", false, "not a valid URL"},
	37	+	{"https://user:pw@ermite.example.org", false, "credentials"},
	38	+	{"", false, "not a valid URL"},
	39	+	} {
	40	+	err := CheckURL(tc.url, tc.insecure)
	41	+	if tc.want == "" {
	42	+	if err != nil {
	43	+	t.Errorf("%q: %v", tc.url, err)
	44	+	}
	45	+	continue
	46	+	}
	47	+	if err == nil \|\| !strings.Contains(err.Error(), tc.want) {
	48	+	t.Errorf("%q (insecure=%v): %v, want %q", tc.url, tc.insecure, err, tc.want)
	49	+	}
	50	+	}
	51	+	}
	52	+
	53	+	// The client does not follow a redirect: the token must not reach the target,
	54	+	// whoever it is.
	55	+	func TestClientDoesNotFollowRedirects(t *testing.T) {
	56	+	var hits atomic.Int32
	57	+	var leaked atomic.Value
	58	+	target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	59	+	hits.Add(1)
	60	+	leaked.Store(r.Header.Get("Authorization"))
	61	+	w.WriteHeader(200)
	62	+	}))
	63	+	defer target.Close()
	64	+	redirector := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	65	+	http.Redirect(w, r, target.URL+"/stolen", http.StatusTemporaryRedirect)
	66	+	}))
	67	+	defer redirector.Close()
	68	+
	69	+	c := NewClient(redirector.URL, "ermiterun_secret", nil)
	70	+	_, err := c.FetchJob(context.Background(), connect.NewRequest(&ermite.FetchJobRequest{}))
	71	+	if err == nil {
	72	+	t.Fatal("a redirect answered a call")
	73	+	}
	74	+	if hits.Load() != 0 {
	75	+	t.Fatalf("the redirect was followed (%d hits), the token was sent to the target: %v", hits.Load(), leaked.Load())
	76	+	}
	77	+	}
	78	+
	79	+	// The token is not sent over plain http to a host that is not loopback.
	80	+	func TestClientRefusesToSendTheTokenInClear(t *testing.T) {
	81	+	var hits atomic.Int32
	82	+	rt := roundTripFunc(func(http.Request) (http.Response, error) { hits.Add(1); return nil, nil })
	83	+	c := NewClient("http://ermite.example.org", "ermiterun_secret", rt)
	84	+	_, err := c.FetchJob(context.Background(), connect.NewRequest(&ermite.FetchJobRequest{}))
	85	+	if err == nil \|\| !strings.Contains(err.Error(), "must be https") \|\| hits.Load() != 0 {
	86	+	t.Fatalf("hits=%d err=%v", hits.Load(), err)
	87	+	}
	88	+	// The explicit flag allows it.
	89	+	c = NewInsecureClient("http://ermite.example.org", "ermiterun_secret", rt)
	90	+	_, _ = c.FetchJob(context.Background(), connect.NewRequest(&ermite.FetchJobRequest{}))
	91	+	if hits.Load() != 1 {
	92	+	t.Fatal("the insecure client did not send the request")
	93	+	}
	94	+	}
	95	+
	96	+	type roundTripFunc func(http.Request) (http.Response, error)
	97	+
	98	+	func (f roundTripFunc) RoundTrip(r http.Request) (http.Response, error) { return f(r) }
	99	+
	100	+	func TestLoadConfigChecksTheURL(t *testing.T) {
	101	+	env := map[string]string{EnvURL: "http://ermite.example.org"}
	102	+	get := func(k string) string { return env[k] }
	103	+	if _, err := LoadConfig(get, "h", "/s", 1, ""); err == nil \|\| !strings.Contains(err.Error(), EnvURL) {
	104	+	t.Fatalf("%v", err)
	105	+	}
	106	+	env[EnvInsecureHTTP] = "1"
	107	+	c, err := LoadConfig(get, "h", "/s", 1, "")
	108	+	if err != nil \|\| !c.InsecureHTTP {
	109	+	t.Fatalf("%+v %v", c, err)
	110	+	}
	111	+	env[EnvInsecureHTTP] = "maybe"
	112	+	if _, err := LoadConfig(get, "h", "/s", 1, ""); err == nil {
	113	+	t.Fatal("a bad value was accepted")
	114	+	}
	115	+	}
	116	+
	117	+	func TestResolveRegisterToken(t *testing.T) {
	118	+	env := map[string]string{}
	119	+	get := func(k string) string { return env[k] }
	120	+	if _, err := ResolveRegisterToken("", false, strings.NewReader(""), get); err == nil \|\| !strings.Contains(err.Error(), EnvRegisterToken) {
	121	+	t.Fatalf("%v", err)
	122	+	}
	123	+	if tok, _ := ResolveRegisterToken("from-flag", false, nil, get); tok != "from-flag" {
	124	+	t.Fatal(tok)
	125	+	}
	126	+	env[EnvRegisterToken] = " from-env \n"
	127	+	if tok, _ := ResolveRegisterToken("from-flag", false, nil, get); tok != "from-env" {
	128	+	t.Fatalf("the environment must win over the flag: %q", tok)
	129	+	}
	130	+	if tok, _ := ResolveRegisterToken("from-flag", true, strings.NewReader("from-stdin\n"), get); tok != "from-stdin" {
	131	+	t.Fatalf("standard input must win: %q", tok)
	132	+	}
	133	+	if _, err := ResolveRegisterToken("x", true, strings.NewReader(" \n"), get); err == nil {
	134	+	t.Fatal("an empty standard input was accepted")
	135	+	}
	136	+	}

A internal/host/runner.go +240 -0

@@ -0,0 +1,240 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package host
	4	+
	5	+	import (
	6	+	"context"
	7	+	"crypto/rand"
	8	+	"encoding/base64"
	9	+	"errors"
	10	+	"fmt"
	11	+	"runtime"
	12	+	"sync"
	13	+	"sync/atomic"
	14	+	"time"
	15	+
	16	+	"connectrpc.com/connect"
	17	+	"github.com/bnema/zerowrap"
	18	+
	19	+	ermite "bnema.dev/ermite-api/gen/ermite"
	20	+	"bnema.dev/ermite-api/gen/ermite/ermiteconnect"
	21	+
	22	+	"bnema.dev/ermite-runner/internal/execstream/execpb"
	23	+	"bnema.dev/ermite-runner/internal/sandbox"
	24	+	)
	25	+
	26	+	// Runner fetches jobs from ermite and runs each one in its own sandbox.
	27	+	type Runner struct {
	28	+	Cfg Config
	29	+	Driver sandbox.Driver
	30	+	// Poll is the client that carries the runner token (FetchJob).
	31	+	Poll ermiteconnect.RunnerServiceClient
	32	+	// Job builds the client of a job from its job token.
	33	+	Job func(jobToken string) ermiteconnect.RunnerServiceClient
	34	+	Timing Timing
	35	+
	36	+	// now returns the current time; tests replace it.
	37	+	now func() time.Time
	38	+
	39	+	leaked atomic.Bool
	40	+	active sync.WaitGroup
	41	+	}
	42	+
	43	+	func (r *Runner) clock() time.Time {
	44	+	if r.now != nil {
	45	+	return r.now()
	46	+	}
	47	+	return time.Now()
	48	+	}
	49	+
	50	+	// spec is the sandbox request of a job. It names the job and leaves CPUs,
	51	+	// memory and disk at zero: the driver fills them from its configuration
	52	+	// (sandbox.Config.Resolve in Start), that is from ERMITE_RUNNER_JOB_CPUS,
	53	+	// _JOB_MEMORY and _JOB_DISK, so that the limits are one setting of the runner
	54	+	// and not something a job or this package can change.
	55	+	func (r Runner) spec(j ermite.Job) sandbox.JobSpec {
	56	+	return sandbox.JobSpec{JobID: j.JobId}
	57	+	}
	58	+
	59	+	// markLeaked records that a sandbox could not be cleaned up: no new job is
	60	+	// taken until the running ones are over and Driver.Reap has succeeded.
	61	+	func (r *Runner) markLeaked() { r.leaked.Store(true) }
	62	+
	63	+	// ErrRemoved is returned by Run when ermite no longer knows the runner token:
	64	+	// the runner was removed or replaced.
	65	+	var ErrRemoved = errors.New("ermite rejected the runner token: the runner was removed or replaced; register it again")
	66	+
	67	+	// newPollID is 16 random bytes in base64url: 22 characters of the set the
	68	+	// protocol allows.
	69	+	func newPollID() string {
	70	+	var b [16]byte
	71	+	_, _ = rand.Read(b[:])
	72	+	return base64.RawURLEncoding.EncodeToString(b[:])
	73	+	}
	74	+
	75	+	// Run is the main loop. It returns nil after a graceful shutdown (ctx ended and
	76	+	// every running job reported), or an error when the runner cannot go on.
	77	+	//
	78	+	// Capacity: a poll is made only when a slot is free. A lost response is retried
	79	+	// with the same poll_id, so that the server hands back the same assignment.
	80	+	func (r *Runner) Run(ctx context.Context) error {
	81	+	log := zerowrap.FromCtx(ctx)
	82	+	t := r.Timing.withDefaults()
	83	+	shutdown := make(chan struct{})
	84	+	slots := make(chan struct{}, max(1, r.Cfg.Capacity))
	85	+	defer func() {
	86	+	// Graceful shutdown: running jobs are cancelled with runner_shutdown and
	87	+	// get ShutdownGrace to stop; Run returns once they have reported.
	88	+	close(shutdown)
	89	+	r.active.Wait()
	90	+	}()
	91	+
	92	+	if err := r.Driver.Reap(ctx); err != nil && !errors.Is(err, sandbox.ErrBusy) {
	93	+	log.Error().Err(err).Msg("reaping what a previous run left")
	94	+	}
	95	+
	96	+	var pollID string
	97	+	bo := backoff{min: t.BackoffMin, max: t.BackoffMax}
	98	+	for ctx.Err() == nil {
	99	+	// Wait for a free slot, and for the end of a leak.
	100	+	select {
	101	+	case slots <- struct{}{}:
	102	+	case <-ctx.Done():
	103	+	return nil
	104	+	}
	105	+	if r.leaked.Load() {
	106	+	<-slots
	107	+	if !r.recover(ctx, log, slots, t.ReapRetry) {
	108	+	return nil
	109	+	}
	110	+	continue
	111	+	}
	112	+	if pollID == "" {
	113	+	pollID = newPollID()
	114	+	}
	115	+	pctx, cancel := context.WithTimeout(ctx, t.PollTimeout)
	116	+	resp, err := r.Poll.FetchJob(pctx, connect.NewRequest(&ermite.FetchJobRequest{PollId: pollID}))
	117	+	cancel()
	118	+	if err != nil {
	119	+	<-slots
	120	+	if ctx.Err() != nil {
	121	+	return nil
	122	+	}
	123	+	if connect.CodeOf(err) == connect.CodeUnauthenticated {
	124	+	return ErrRemoved
	125	+	}
	126	+	if connect.CodeOf(err) == connect.CodeInvalidArgument {
	127	+	pollID = "" // our id was refused: a new one
	128	+	}
	129	+	d := bo.next()
	130	+	log.Warn().Err(err).Dur("retry_in", d).Msg("FetchJob failed")
	131	+	if !sleep(ctx, d) {
	132	+	return nil
	133	+	}
	134	+	continue
	135	+	}
	136	+	bo.cur = 0
	137	+	pollID = "" // answered: the next poll is a new one
	138	+	if resp.Msg.Job == nil {
	139	+	<-slots
	140	+	d := t.NoJobPause
	141	+	if ra := resp.Msg.RetryAfter; ra != nil && ra.AsDuration() > d {
	142	+	d = ra.AsDuration()
	143	+	}
	144	+	if !sleep(ctx, d) {
	145	+	return nil
	146	+	}
	147	+	continue
	148	+	}
	149	+	r.startJob(ctx, log, resp.Msg.Job, t, shutdown, slots)
	150	+	if ra := resp.Msg.RetryAfter; ra != nil && ra.AsDuration() > 0 {
	151	+	if !sleep(ctx, ra.AsDuration()) {
	152	+	return nil
	153	+	}
	154	+	}
	155	+	}
	156	+	return nil
	157	+	}
	158	+
	159	+	// recover waits until no job runs, then reaps. It returns false when ctx ends.
	160	+	func (r *Runner) recover(ctx context.Context, log zerowrap.Logger, slots chan struct{}, retry time.Duration) bool {
	161	+	log.Error().Msg("a sandbox could not be cleaned up: no new job until the running ones end and the leftovers are reaped")
	162	+	for ctx.Err() == nil {
	163	+	if len(slots) == 0 {
	164	+	if err := r.Driver.Reap(ctx); err == nil {
	165	+	r.leaked.Store(false)
	166	+	log.Info().Msg("reaped: taking jobs again")
	167	+	return true
	168	+	} else if !errors.Is(err, sandbox.ErrBusy) {
	169	+	log.Error().Err(err).Msg("reaping failed, retrying")
	170	+	}
	171	+	}
	172	+	if !sleep(ctx, retry) {
	173	+	return false
	174	+	}
	175	+	}
	176	+	return false
	177	+	}
	178	+
	179	+	func (r Runner) startJob(ctx context.Context, log zerowrap.Logger, job ermite.Job, t Timing, shutdown <-chan struct{}, slots chan struct{}) {
	180	+	jctx, cancel := context.WithCancelCause(context.WithoutCancel(ctx))
	181	+	jr := &jobRun{
	182	+	r: r, job: job, client: r.Job(job.JobToken), t: t, ctx: jctx, cancel: cancel,
	183	+	log: zerowrap.Logger{Logger: log.With().Str("job_id", job.JobId).Int64("run_id", job.RunId).Int32("attempt", job.Attempt).Logger()},
	184	+	receivedAt: r.clock(), cancelCh: make(chan execpb.Cancel, 1), steps: map[int32]ermite.StepUpdate{},
	185	+	}
	186	+	jr.log.Info().Str("job_key", job.JobKey).Msg("job assigned")
	187	+	r.active.Add(1)
	188	+	go func() {
	189	+	defer r.active.Done()
	190	+	defer func() { <-slots }()
	191	+	jr.run(r, shutdown)
	192	+	jr.log.Info().Msg("job finished")
	193	+	}()
	194	+	}
	195	+
	196	+	// ---- registration
	197	+
	198	+	// Isolation maps the driver's isolation to the protocol's.
	199	+	func Isolation(i sandbox.Isolation) ermite.RunnerIsolation {
	200	+	switch i {
	201	+	case sandbox.IsolationVM:
	202	+	return ermite.RunnerIsolation_RUNNER_ISOLATION_VM
	203	+	case sandbox.IsolationContainerSysbox:
	204	+	return ermite.RunnerIsolation_RUNNER_ISOLATION_CONTAINER_SYSBOX
	205	+	case sandbox.IsolationContainerPrivileged:
	206	+	return ermite.RunnerIsolation_RUNNER_ISOLATION_CONTAINER_PRIVILEGED
	207	+	}
	208	+	return ermite.RunnerIsolation_RUNNER_ISOLATION_UNSPECIFIED
	209	+	}
	210	+
	211	+	// GoArch is the architecture in GitHub's names.
	212	+	func GoArch() string {
	213	+	if runtime.GOARCH == "arm64" {
	214	+	return "arm64"
	215	+	}
	216	+	return "x64"
	217	+	}
	218	+
	219	+	// Register registers the runner with ermite using a token that has the
	220	+	// runner:register scope, and stores the runner token (mode 0600) in the state
	221	+	// directory. replace removes a runner of the same name first.
	222	+	func Register(ctx context.Context, cfg Config, client ermiteconnect.RunnerServiceClient, isolation sandbox.Isolation, replace bool) (State, error) {
	223	+	iso := Isolation(isolation)
	224	+	if iso == ermite.RunnerIsolation_RUNNER_ISOLATION_UNSPECIFIED {
	225	+	return State{}, fmt.Errorf("unknown isolation %q", isolation)
	226	+	}
	227	+	rctx, cancel := context.WithTimeout(ctx, 30*time.Second)
	228	+	defer cancel()
	229	+	resp, err := client.RegisterRunner(rctx, connect.NewRequest(&ermite.RegisterRunnerRequest{
	230	+	Name: cfg.Name, Labels: cfg.Labels, Version: cfg.Version, Isolation: iso, Arch: GoArch(), Os: "linux", Replace: replace,
	231	+	}))
	232	+	if err != nil {
	233	+	return State{}, fmt.Errorf("register: %w", err)
	234	+	}
	235	+	st := State{URL: cfg.URL, RunnerID: resp.Msg.RunnerId, Name: cfg.Name, RunnerToken: resp.Msg.RunnerToken}
	236	+	if err := SaveState(cfg.StateDir, st); err != nil {
	237	+	return State{}, fmt.Errorf("the runner is registered but its token could not be stored: %w", err)
	238	+	}
	239	+	return st, nil
	240	+	}

A internal/host/runner_test.go +1316 -0

@@ -0,0 +1,1316 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package host
	4	+
	5	+	import (
	6	+	"context"
	7	+	"encoding/json"
	8	+	"errors"
	9	+	"io"
	10	+	"net/http"
	11	+	"os"
	12	+	"path/filepath"
	13	+	"regexp"
	14	+	"strconv"
	15	+	"strings"
	16	+	"testing"
	17	+	"time"
	18	+	"unicode/utf8"
	19	+
	20	+	"google.golang.org/protobuf/types/known/timestamppb"
	21	+
	22	+	"connectrpc.com/connect"
	23	+
	24	+	ermite "bnema.dev/ermite-api/gen/ermite"
	25	+	"bnema.dev/ermite-api/gen/ermite/ermiteconnect"
	26	+
	27	+	"bnema.dev/ermite-runner/internal/execstream"
	28	+	"bnema.dev/ermite-runner/internal/execstream/execpb"
	29	+	"bnema.dev/ermite-runner/internal/sandbox"
	30	+	)
	31	+
	32	+	func TestJobRunsAndReports(t *testing.T) {
	33	+	h := newHarness(t, simpleScript("hello", "world"))
	34	+	h.srv.jobs = []*ermite.Job{testJob("j1")}
	35	+	stop := h.run(t)
	36	+
	37	+	eventually(t, "CompleteJob", func() bool { return h.srv.completeCount() == 1 })
	38	+	if err := stop(); err != nil {
	39	+	t.Fatal(err)
	40	+	}
	41	+	s := h.srv
	42	+	s.mu.Lock()
	43	+	defer s.mu.Unlock()
	44	+	c := s.completes[0]
	45	+	if c.Conclusion != ermite.CiConclusion_CI_CONCLUSION_SUCCESS \|\| c.Outputs["out"] != "v" \|\| c.RunnerShutdown {
	46	+	t.Fatalf("%+v", c)
	47	+	}
	48	+	if len(s.beats) == 0 {
	49	+	t.Error("no heartbeat: the first one is the acknowledgement")
	50	+	}
	51	+	if strings.Join(s.lines, "\|") != "hello\|world" {
	52	+	t.Errorf("lines %q", s.lines)
	53	+	}
	54	+	for n, name := range map[int32]string{1: "Set up job", 2: "Run it", 3: "Complete job"} {
	55	+	u := s.steps[n]
	56	+	if u == nil \|\| u.Name != name \|\| u.State != ermite.StepState_STEP_STATE_COMPLETED \|\| u.Conclusion != ermite.CiConclusion_CI_CONCLUSION_SUCCESS \|\| u.StartedAt == nil \|\| u.CompletedAt == nil {
	57	+	t.Errorf("step %d: %v", n, u)
	58	+	}
	59	+	}
	60	+	if len(c.Steps) != 3 {
	61	+	t.Errorf("CompleteJob carries %d final steps", len(c.Steps))
	62	+	}
	63	+	if h.driver.stops != 1 \|\| h.driver.reaps < 1 {
	64	+	t.Errorf("stops=%d reaps=%d: the sandbox must be stopped and the start must reap", h.driver.stops, h.driver.reaps)
	65	+	}
	66	+	if strings.Contains(h.logs.String(), "ermitejob_") \|\| strings.Contains(h.logs.String(), "ermiterun_") {
	67	+	t.Errorf("a token reached the log: %s", h.logs.String())
	68	+	}
	69	+	}
	70	+
	71	+	func TestTheJobTokenAndTheRunnerTokenStayApart(t *testing.T) {
	72	+	h := newHarness(t, simpleScript("x"))
	73	+	h.srv.jobs = []*ermite.Job{testJob("j1")}
	74	+	var sawJobSpec *execpb.JobSpec
	75	+	inner := h.driver.script
	76	+	h.driver.script = func(t testing.T, c execstream.Conn, run execpb.Run, cn <-chan execpb.Cancel) {
	77	+	sawJobSpec = run.Job
	78	+	inner(t, c, run, cn)
	79	+	}
	80	+	h.run(t)
	81	+	eventually(t, "CompleteJob", func() bool { return h.srv.completeCount() == 1 })
	82	+	// The sandbox gets the workflow and contexts, never a token.
	83	+	if sawJobSpec == nil \|\| sawJobSpec.JobKey != "j" \|\| sawJobSpec.RunnerName != "r1" \|\| sawJobSpec.RunnerEnvironment != "self-hosted" \|\| sawJobSpec.TimeoutMinutes != 5 {
	84	+	t.Fatalf("%v", sawJobSpec)
	85	+	}
	86	+	if strings.Contains(sawJobSpec.String(), "ermitejob_") \|\| strings.Contains(sawJobSpec.String(), "ermiterun_") {
	87	+	t.Fatal("a token reached the sandbox")
	88	+	}
	89	+	}
	90	+
	91	+	func TestPollRetryUsesTheSamePollIDAfterAnError(t *testing.T) {
	92	+	h := newHarness(t, simpleScript("x"))
	93	+	h.srv.pollHook = func(n int, req ermite.FetchJobRequest) (ermite.FetchJobResponse, error) {
	94	+	if n <= 2 { // the response is lost twice
	95	+	return nil, connect.NewError(connect.CodeUnavailable, io.EOF)
	96	+	}
	97	+	if n == 3 {
	98	+	return &ermite.FetchJobResponse{Job: testJob("j1")}, nil
	99	+	}
	100	+	return nil, nil
	101	+	}
	102	+	h.run(t)
	103	+	eventually(t, "CompleteJob and the next poll", func() bool {
	104	+	h.srv.mu.Lock()
	105	+	defer h.srv.mu.Unlock()
	106	+	return len(h.srv.completes) == 1 && len(h.srv.polls) >= 4
	107	+	})
	108	+	h.srv.mu.Lock()
	109	+	defer h.srv.mu.Unlock()
	110	+	if len(h.srv.polls) < 4 {
	111	+	t.Fatalf("polls %v", h.srv.polls)
	112	+	}
	113	+	if h.srv.polls[0] == "" \|\| h.srv.polls[0] != h.srv.polls[1] \|\| h.srv.polls[1] != h.srv.polls[2] {
	114	+	t.Errorf("a retried poll must keep its poll_id: %v", h.srv.polls)
	115	+	}
	116	+	if h.srv.polls[3] == h.srv.polls[2] {
	117	+	t.Errorf("an answered poll is followed by a new id: %v", h.srv.polls)
	118	+	}
	119	+	}
	120	+
	121	+	func TestPollRespectsRetryAfter(t *testing.T) {
	122	+	h := newHarness(t, simpleScript("x"))
	123	+	h.srv.retryAfter = 300 * time.Millisecond
	124	+	h.run(t)
	125	+	time.Sleep(1000 * time.Millisecond)
	126	+	h.srv.mu.Lock()
	127	+	n := h.srv.pollCalls
	128	+	h.srv.mu.Unlock()
	129	+	if n < 2 \|\| n > 5 {
	130	+	t.Fatalf("%d polls in 1 s with retry_after=300ms", n)
	131	+	}
	132	+	}
	133	+
	134	+	func TestPollRemovedRunnerStops(t *testing.T) {
	135	+	h := newHarness(t, simpleScript("x"))
	136	+	h.runner.Poll = NewClient(h.url, "ermiterun_wrong", http.DefaultTransport)
	137	+	err := h.runner.Run(h.logs.ctx())
	138	+	if !errors.Is(err, ErrRemoved) {
	139	+	t.Fatalf("%v", err)
	140	+	}
	141	+	}
	142	+
	143	+	func TestNoPollWithoutFreeCapacity(t *testing.T) {
	144	+	release := make(chan struct{})
	145	+	h := newHarness(t, func(_ testing.T, c execstream.Conn, _ execpb.Run, _ <-chan execpb.Cancel) {
	146	+	stepStart(c, 1, "Set up job")
	147	+	<-release
	148	+	stepDone(c, 1, "Set up job", cOK)
	149	+	jobDone(c, cOK, nil, false)
	150	+	})
	151	+	h.srv.jobs = []*ermite.Job{testJob("j1"), testJob("j2")}
	152	+	h.run(t)
	153	+	eventually(t, "first job started", func() bool { h.driver.mu.Lock(); defer h.driver.mu.Unlock(); return h.driver.started == 1 })
	154	+	time.Sleep(300 * time.Millisecond)
	155	+	h.srv.mu.Lock()
	156	+	calls := h.srv.pollCalls
	157	+	h.srv.mu.Unlock()
	158	+	if calls != 1 {
	159	+	t.Fatalf("polled %d times with the only slot busy", calls)
	160	+	}
	161	+	h.driver.mu.Lock()
	162	+	if h.driver.started != 1 {
	163	+	t.Errorf("started=%d", h.driver.started)
	164	+	}
	165	+	h.driver.mu.Unlock()
	166	+	close(release)
	167	+	eventually(t, "both jobs done", func() bool { return h.srv.completeCount() == 2 })
	168	+	}
	169	+
	170	+	func TestCapacityTwoRunsTwoJobsAtOnce(t *testing.T) {
	171	+	release := make(chan struct{})
	172	+	h := newHarness(t, func(_ testing.T, c execstream.Conn, _ execpb.Run, _ <-chan execpb.Cancel) {
	173	+	stepStart(c, 1, "Set up job")
	174	+	<-release
	175	+	stepDone(c, 1, "Set up job", cOK)
	176	+	jobDone(c, cOK, nil, false)
	177	+	})
	178	+	h.runner.Cfg.Capacity = 2
	179	+	h.srv.jobs = []*ermite.Job{testJob("j1"), testJob("j2"), testJob("j3")}
	180	+	h.run(t)
	181	+	eventually(t, "two sandboxes", func() bool { h.driver.mu.Lock(); defer h.driver.mu.Unlock(); return h.driver.started == 2 })
	182	+	time.Sleep(200 * time.Millisecond)
	183	+	h.driver.mu.Lock()
	184	+	if h.driver.started != 2 {
	185	+	t.Errorf("started=%d with capacity 2", h.driver.started)
	186	+	}
	187	+	h.driver.mu.Unlock()
	188	+	close(release)
	189	+	eventually(t, "three jobs done", func() bool { return h.srv.completeCount() == 3 })
	190	+	}
	191	+
	192	+	func TestFirstHeartbeatRetriesEvery5sStyle(t *testing.T) {
	193	+	h := newHarness(t, simpleScript("x"))
	194	+	h.srv.jobs = []*ermite.Job{testJob("j1")}
	195	+	h.srv.beatHook = func(n int) error {
	196	+	if n <= 2 {
	197	+	return connect.NewError(connect.CodeUnavailable, io.EOF)
	198	+	}
	199	+	return nil
	200	+	}
	201	+	h.run(t)
	202	+	eventually(t, "CompleteJob", func() bool { return h.srv.completeCount() == 1 })
	203	+	h.srv.mu.Lock()
	204	+	defer h.srv.mu.Unlock()
	205	+	if len(h.srv.beats) < 3 {
	206	+	t.Fatalf("beats %d", len(h.srv.beats))
	207	+	}
	208	+	// The job starts only after the acknowledgement: no log before the third beat.
	209	+	if gap := h.srv.beats[1].Sub(h.srv.beats[0]); gap > 90*time.Millisecond {
	210	+	t.Errorf("retry after %v, want about HeartbeatRetry (30ms), not the 100ms period", gap)
	211	+	}
	212	+	}
	213	+
	214	+	func TestHeartbeatTicksAndRetriesQuickly(t *testing.T) {
	215	+	release := make(chan struct{})
	216	+	h := newHarness(t, func(_ testing.T, c execstream.Conn, _ execpb.Run, _ <-chan execpb.Cancel) {
	217	+	stepStart(c, 1, "Set up job")
	218	+	<-release
	219	+	stepDone(c, 1, "Set up job", cOK)
	220	+	jobDone(c, cOK, nil, false)
	221	+	})
	222	+	h.srv.jobs = []*ermite.Job{testJob("j1")}
	223	+	h.srv.beatHook = func(n int) error {
	224	+	if n == 3 { // one failure in the middle of the job
	225	+	return connect.NewError(connect.CodeUnavailable, io.EOF)
	226	+	}
	227	+	return nil
	228	+	}
	229	+	h.run(t)
	230	+	eventually(t, "five heartbeats", func() bool { h.srv.mu.Lock(); defer h.srv.mu.Unlock(); return len(h.srv.beats) >= 5 })
	231	+	h.srv.mu.Lock()
	232	+	b := append([]time.Time(nil), h.srv.beats...)
	233	+	h.srv.mu.Unlock()
	234	+	if d := b[2].Sub(b[1]); d < 80*time.Millisecond {
	235	+	t.Errorf("period %v, want about 100ms", d)
	236	+	}
	237	+	if d := b[3].Sub(b[2]); d > 80*time.Millisecond {
	238	+	t.Errorf("the retry after a failure came after %v, want about 30ms", d)
	239	+	}
	240	+	close(release)
	241	+	eventually(t, "CompleteJob", func() bool { return h.srv.completeCount() == 1 })
	242	+	}
	243	+
	244	+	func TestCancelFromWatchJobReachesTheExecutor(t *testing.T) {
	245	+	got := make(chan *execpb.Cancel, 1)
	246	+	h := newHarness(t, func(_ testing.T, c execstream.Conn, _ execpb.Run, cancels <-chan execpb.Cancel) {
	247	+	stepStart(c, 1, "Set up job")
	248	+	stepStart(c, 2, "Run sleep")
	249	+	logLineFrame(c, 2, "working", false)
	250	+	cn := <-cancels
	251	+	got <- cn
	252	+	stepDone(c, 2, "Run sleep", cCancel)
	253	+	jobDone(c, cCancel, nil, false)
	254	+	})
	255	+	h.srv.jobs = []*ermite.Job{testJob("j1")}
	256	+	h.run(t)
	257	+	eventually(t, "the job to run", func() bool { return len(h.srv.snapshotLines()) == 1 })
	258	+	h.srv.watchMsgs <- &ermite.WatchJobResponse{Msg: &ermite.WatchJobResponse_Keepalive{Keepalive: &ermite.Keepalive{}}}
	259	+	h.srv.watchMsgs <- &ermite.WatchJobResponse{Msg: &ermite.WatchJobResponse_Cancel{Cancel: &ermite.CancelRequest{Reason: ermite.CancelReason_CANCEL_REASON_USER, Message: "Canceled by the owner"}}}
	260	+	select {
	261	+	case cn := <-got:
	262	+	if cn.Reason != "Canceled by the owner" \|\| cn.Kind != execpb.CancelKind_CANCEL_KIND_USER {
	263	+	t.Fatalf("%v", cn)
	264	+	}
	265	+	case <-time.After(5 * time.Second):
	266	+	t.Fatal("the cancel did not reach the executor")
	267	+	}
	268	+	eventually(t, "CompleteJob", func() bool { return h.srv.completeCount() == 1 })
	269	+	c := h.srv.completes[0]
	270	+	if c.Conclusion != ermite.CiConclusion_CI_CONCLUSION_CANCELLED \|\| c.RunnerShutdown {
	271	+	t.Fatalf("%v", c)
	272	+	}
	273	+	}
	274	+
	275	+	func TestTimeoutCancelKind(t *testing.T) {
	276	+	got := make(chan *execpb.Cancel, 1)
	277	+	h := newHarness(t, func(_ testing.T, c execstream.Conn, _ execpb.Run, cancels <-chan execpb.Cancel) {
	278	+	stepStart(c, 1, "Set up job")
	279	+	logLineFrame(c, 1, "x", false)
	280	+	got <- <-cancels
	281	+	jobDone(c, execpb.StepConclusion_STEP_CONCLUSION_TIMED_OUT, nil, false)
	282	+	})
	283	+	h.srv.jobs = []*ermite.Job{testJob("j1")}
	284	+	h.run(t)
	285	+	eventually(t, "log", func() bool { return len(h.srv.snapshotLines()) == 1 })
	286	+	h.srv.watchMsgs <- &ermite.WatchJobResponse{Msg: &ermite.WatchJobResponse_Cancel{Cancel: &ermite.CancelRequest{Reason: ermite.CancelReason_CANCEL_REASON_TIMEOUT}}}
	287	+	if cn := <-got; cn.Kind != execpb.CancelKind_CANCEL_KIND_TIMEOUT {
	288	+	t.Fatalf("%v", cn)
	289	+	}
	290	+	eventually(t, "CompleteJob", func() bool { return h.srv.completeCount() == 1 })
	291	+	if c := h.srv.completes[0]; c.Conclusion != ermite.CiConclusion_CI_CONCLUSION_TIMED_OUT {
	292	+	t.Fatalf("%v", c.Conclusion)
	293	+	}
	294	+	}
	295	+
	296	+	func TestWatchJobIsReopenedAfterAnError(t *testing.T) {
	297	+	release := make(chan struct{})
	298	+	h := newHarness(t, func(_ testing.T, c execstream.Conn, _ execpb.Run, cancels <-chan execpb.Cancel) {
	299	+	stepStart(c, 1, "Set up job")
	300	+	<-release
	301	+	jobDone(c, cOK, nil, false)
	302	+	})
	303	+	h.srv.jobs = []*ermite.Job{testJob("j1")}
	304	+	h.run(t)
	305	+	eventually(t, "the first watch", func() bool { h.srv.mu.Lock(); defer h.srv.mu.Unlock(); return h.srv.watches >= 1 })
	306	+	// A message that ends the stream cleanly without Ended: the runner reopens.
	307	+	// The fake closes the stream only on Ended or context end, so cut the
	308	+	// transport instead: close idle connections of the test server.
	309	+	http.DefaultTransport.(*http.Transport).CloseIdleConnections()
	310	+	close(release)
	311	+	eventually(t, "CompleteJob", func() bool { return h.srv.completeCount() == 1 })
	312	+	}
	313	+
	314	+	func TestLogBatchesRespectTheLimitsAndKeepOrder(t *testing.T) {
	315	+	var lines []string
	316	+	big := strings.Repeat("x", 1000)
	317	+	for i := 0; i < 2500; i++ {
	318	+	lines = append(lines, big[:1+i%1000])
	319	+	}
	320	+	h := newHarness(t, simpleScript(lines...))
	321	+	h.runner.Timing.LogFlush = 50 * time.Millisecond
	322	+	h.srv.jobs = []*ermite.Job{testJob("j1")}
	323	+	h.run(t)
	324	+	eventually(t, "CompleteJob", func() bool { return h.srv.completeCount() == 1 })
	325	+	h.srv.mu.Lock()
	326	+	defer h.srv.mu.Unlock()
	327	+	if len(h.srv.lines) != 2500 {
	328	+	t.Fatalf("the server holds %d lines", len(h.srv.lines))
	329	+	}
	330	+	for i, l := range h.srv.lines {
	331	+	if len(l) != 1+(i)%1000 {
	332	+	t.Fatalf("line %d out of order or wrong", i)
	333	+	}
	334	+	}
	335	+	var next int64
	336	+	for _, r := range h.srv.appendReq {
	337	+	if len(r.Lines) > 1000 {
	338	+	t.Fatalf("batch of %d lines", len(r.Lines))
	339	+	}
	340	+	n := 0
	341	+	for _, l := range r.Lines {
	342	+	n += len(l.Text)
	343	+	}
	344	+	if n > 64*1024 {
	345	+	t.Fatalf("batch of %d bytes", n)
	346	+	}
	347	+	if r.Offset != next {
	348	+	t.Fatalf("offset %d, want %d", r.Offset, next)
	349	+	}
	350	+	next += int64(len(r.Lines))
	351	+	}
	352	+	if next != 2500 {
	353	+	t.Fatalf("offsets end at %d", next)
	354	+	}
	355	+	// Batches never mix steps.
	356	+	for _, r := range h.srv.appendReq {
	357	+	if r.Step < 1 {
	358	+	t.Fatalf("step %d", r.Step)
	359	+	}
	360	+	}
	361	+	}
	362	+
	363	+	func TestLogStepsAreSeparateRequests(t *testing.T) {
	364	+	h := newHarness(t, func(_ testing.T, c execstream.Conn, _ execpb.Run, _ <-chan execpb.Cancel) {
	365	+	for n := 1; n <= 3; n++ {
	366	+	stepStart(c, n, "s")
	367	+	logLineFrame(c, n, "a", false)
	368	+	logLineFrame(c, n, "b", false)
	369	+	stepDone(c, n, "s", cOK)
	370	+	}
	371	+	jobDone(c, cOK, nil, false)
	372	+	})
	373	+	h.runner.Timing.LogFlush = 50 * time.Millisecond
	374	+	h.srv.jobs = []*ermite.Job{testJob("j1")}
	375	+	h.run(t)
	376	+	eventually(t, "CompleteJob", func() bool { return h.srv.completeCount() == 1 })
	377	+	h.srv.mu.Lock()
	378	+	defer h.srv.mu.Unlock()
	379	+	if len(h.srv.appendReq) != 3 {
	380	+	t.Fatalf("%d requests, want one per step", len(h.srv.appendReq))
	381	+	}
	382	+	for i, r := range h.srv.appendReq {
	383	+	if r.Step != int32(i+1) \|\| r.Offset != int64(2*i) \|\| len(r.Lines) != 2 {
	384	+	t.Errorf("request %d: step %d offset %d lines %d", i, r.Step, r.Offset, len(r.Lines))
	385	+	}
	386	+	}
	387	+	}
	388	+
	389	+	func TestContinuedLinesAreForwarded(t *testing.T) {
	390	+	h := newHarness(t, func(_ testing.T, c execstream.Conn, _ execpb.Run, _ <-chan execpb.Cancel) {
	391	+	stepStart(c, 1, "s")
	392	+	logLineFrame(c, 1, "first", false)
	393	+	logLineFrame(c, 1, "piece1", false)
	394	+	logLineFrame(c, 1, "piece2", true)
	395	+	stepDone(c, 1, "s", cOK)
	396	+	jobDone(c, cOK, nil, false)
	397	+	})
	398	+	h.srv.jobs = []*ermite.Job{testJob("j1")}
	399	+	h.run(t)
	400	+	eventually(t, "CompleteJob", func() bool { return h.srv.completeCount() == 1 })
	401	+	h.srv.mu.Lock()
	402	+	defer h.srv.mu.Unlock()
	403	+	if len(h.srv.cont) != 3 \|\| h.srv.cont[0] \|\| h.srv.cont[1] \|\| !h.srv.cont[2] {
	404	+	t.Fatalf("%v", h.srv.cont)
	405	+	}
	406	+	}
	407	+
	408	+	func TestAppendLogRetriesAreIdempotent(t *testing.T) {
	409	+	h := newHarness(t, simpleScript("l1", "l2", "l3"))
	410	+	h.srv.jobs = []*ermite.Job{testJob("j1")}
	411	+	// The first AppendLog is applied but its response is lost; the second fails
	412	+	// before being applied. The runner resends the same lines at the same offset.
	413	+	h.srv.appendHook = func(n int, _ *ermite.AppendLogRequest) (bool, error) {
	414	+	switch n {
	415	+	case 1:
	416	+	return true, connect.NewError(connect.CodeUnavailable, io.EOF)
	417	+	case 2:
	418	+	return false, connect.NewError(connect.CodeInternal, io.EOF)
	419	+	}
	420	+	return false, nil
	421	+	}
	422	+	h.run(t)
	423	+	eventually(t, "CompleteJob", func() bool { return h.srv.completeCount() == 1 })
	424	+	h.srv.mu.Lock()
	425	+	defer h.srv.mu.Unlock()
	426	+	if strings.Join(h.srv.lines, "\|") != "l1\|l2\|l3" {
	427	+	t.Fatalf("lines %q: a retry must neither lose nor duplicate", h.srv.lines)
	428	+	}
	429	+	if h.srv.appendN < 3 {
	430	+	t.Fatalf("only %d calls", h.srv.appendN)
	431	+	}
	432	+	first := h.srv.appendReq[0]
	433	+	for _, r := range h.srv.appendReq[1:3] {
	434	+	if r.Offset != first.Offset \|\| len(r.Lines) != len(first.Lines) {
	435	+	t.Errorf("a retry changed the request: offset %d/%d lines %d/%d", r.Offset, first.Offset, len(r.Lines), len(first.Lines))
	436	+	}
	437	+	}
	438	+	}
	439	+
	440	+	func TestStepUpdatesAreRetried(t *testing.T) {
	441	+	h := newHarness(t, simpleScript("x"))
	442	+	h.srv.jobs = []*ermite.Job{testJob("j1")}
	443	+	h.run(t)
	444	+	eventually(t, "CompleteJob", func() bool { return h.srv.completeCount() == 1 })
	445	+	// UpdateJob snapshots carry the state at each step boundary.
	446	+	h.srv.mu.Lock()
	447	+	defer h.srv.mu.Unlock()
	448	+	if u := h.srv.steps[2]; u == nil \|\| u.State != ermite.StepState_STEP_STATE_COMPLETED {
	449	+	t.Fatalf("%v", u)
	450	+	}
	451	+	}
	452	+
	453	+	func TestCompleteJobIsRetriedAndIdempotent(t *testing.T) {
	454	+	h := newHarness(t, simpleScript("x"))
	455	+	h.srv.jobs = []*ermite.Job{testJob("j1")}
	456	+	h.srv.completeHook = func(n int) (bool, error) {
	457	+	switch n {
	458	+	case 1: // applied, response lost
	459	+	return true, connect.NewError(connect.CodeUnavailable, io.EOF)
	460	+	case 2: // not even reached
	461	+	return false, connect.NewError(connect.CodeInternal, io.EOF)
	462	+	}
	463	+	return false, nil
	464	+	}
	465	+	h.run(t)
	466	+	eventually(t, "three CompleteJob calls", func() bool { return h.srv.completeCount() == 3 })
	467	+	time.Sleep(100 * time.Millisecond)
	468	+	if n := h.srv.completeCount(); n != 3 {
	469	+	t.Fatalf("%d calls", n)
	470	+	}
	471	+	h.srv.mu.Lock()
	472	+	defer h.srv.mu.Unlock()
	473	+	a, b := h.srv.completes[0], h.srv.completes[2]
	474	+	if a.Conclusion != b.Conclusion \|\| a.Outputs["out"] != b.Outputs["out"] \|\| len(a.Steps) != len(b.Steps) {
	475	+	t.Error("a retry must repeat the same result")
	476	+	}
	477	+	}
	478	+
	479	+	func TestJobEndedStopsEverything(t *testing.T) {
	480	+	release := make(chan struct{})
	481	+	h := newHarness(t, func(_ testing.T, c execstream.Conn, _ execpb.Run, cancels <-chan execpb.Cancel) {
	482	+	stepStart(c, 1, "Set up job")
	483	+	logLineFrame(c, 1, "before", false)
	484	+	<-release
	485	+	logLineFrame(c, 1, "after", false)
	486	+	stepDone(c, 1, "Set up job", cOK)
	487	+	jobDone(c, cOK, nil, false)
	488	+	})
	489	+	h.srv.jobs = []*ermite.Job{testJob("j1")}
	490	+	h.run(t)
	491	+	eventually(t, "first line", func() bool { return len(h.srv.snapshotLines()) == 1 })
	492	+	// The server ends the job (lost runner, log limit, ...): every call now says so.
	493	+	h.srv.mu.Lock()
	494	+	h.srv.ended = &ermite.JobEnded{Conclusion: ermite.CiConclusion_CI_CONCLUSION_CANCELLED, Error: "ended by the server"}
	495	+	h.srv.mu.Unlock()
	496	+	h.srv.watchMsgs <- &ermite.WatchJobResponse{Msg: &ermite.WatchJobResponse_Ended{Ended: &ermite.JobEnded{Conclusion: ermite.CiConclusion_CI_CONCLUSION_CANCELLED}}}
	497	+	eventually(t, "the sandbox to be stopped", func() bool { h.driver.mu.Lock(); defer h.driver.mu.Unlock(); return h.driver.stops == 1 })
	498	+	close(release)
	499	+	time.Sleep(300 * time.Millisecond)
	500	+	if n := h.srv.completeCount(); n != 0 {
	501	+	t.Fatalf("CompleteJob was called %d times for a job the server ended", n)
	502	+	}
	503	+	if got := h.srv.snapshotLines(); len(got) != 1 {
	504	+	t.Fatalf("lines after the end: %q", got)
	505	+	}
	506	+	}
	507	+
	508	+	func TestJobEndedFromAppendLogAbandonsTheJob(t *testing.T) {
	509	+	release := make(chan struct{})
	510	+	h := newHarness(t, func(_ testing.T, c execstream.Conn, _ execpb.Run, _ <-chan execpb.Cancel) {
	511	+	stepStart(c, 1, "Set up job")
	512	+	logLineFrame(c, 1, "a", false)
	513	+	<-release
	514	+	jobDone(c, cOK, nil, false)
	515	+	})
	516	+	h.srv.jobs = []*ermite.Job{testJob("j1")}
	517	+	h.srv.appendHook = func(n int, _ *ermite.AppendLogRequest) (bool, error) {
	518	+	return false, endedErr(&ermite.JobEnded{Conclusion: ermite.CiConclusion_CI_CONCLUSION_FAILURE, Error: "log limit reached"})
	519	+	}
	520	+	h.run(t)
	521	+	eventually(t, "the sandbox to be stopped", func() bool { h.driver.mu.Lock(); defer h.driver.mu.Unlock(); return h.driver.stops == 1 })
	522	+	close(release)
	523	+	time.Sleep(200 * time.Millisecond)
	524	+	if n := h.srv.completeCount(); n != 0 {
	525	+	t.Fatalf("CompleteJob was called %d times", n)
	526	+	}
	527	+	}
	528	+
	529	+	func TestUnauthenticatedJobCallAbandonsAtOnce(t *testing.T) {
	530	+	release := make(chan struct{})
	531	+	h := newHarness(t, func(_ testing.T, c execstream.Conn, _ execpb.Run, _ <-chan execpb.Cancel) {
	532	+	stepStart(c, 1, "Set up job")
	533	+	<-release
	534	+	jobDone(c, cOK, nil, false)
	535	+	})
	536	+	h.srv.jobs = []*ermite.Job{testJob("j1")}
	537	+	h.srv.beatHook = func(n int) error {
	538	+	if n >= 2 { // the assignment was replaced
	539	+	return connect.NewError(connect.CodeUnauthenticated, io.EOF)
	540	+	}
	541	+	return nil
	542	+	}
	543	+	h.run(t)
	544	+	eventually(t, "the sandbox to be stopped", func() bool { h.driver.mu.Lock(); defer h.driver.mu.Unlock(); return h.driver.stops == 1 })
	545	+	close(release)
	546	+	time.Sleep(200 * time.Millisecond)
	547	+	if n := h.srv.completeCount(); n != 0 {
	548	+	t.Fatalf("CompleteJob was called %d times", n)
	549	+	}
	550	+	}
	551	+
	552	+	func TestGracefulShutdownReportsRunnerShutdown(t *testing.T) {
	553	+	h := newHarness(t, func(_ testing.T, c execstream.Conn, _ execpb.Run, cancels <-chan execpb.Cancel) {
	554	+	stepStart(c, 1, "Set up job")
	555	+	stepStart(c, 2, "Run sleep")
	556	+	logLineFrame(c, 2, "working", false)
	557	+	cn := <-cancels
	558	+	if cn.Kind != execpb.CancelKind_CANCEL_KIND_SHUTDOWN {
	559	+	t.Errorf("kind %v", cn.Kind)
	560	+	}
	561	+	stepDone(c, 2, "Run sleep", cCancel)
	562	+	jobDone(c, cCancel, nil, true)
	563	+	})
	564	+	h.srv.jobs = []*ermite.Job{testJob("j1")}
	565	+	stop := h.run(t)
	566	+	eventually(t, "the job to run", func() bool { return len(h.srv.snapshotLines()) == 1 })
	567	+	if err := stop(); err != nil {
	568	+	t.Fatal(err)
	569	+	}
	570	+	h.srv.mu.Lock()
	571	+	defer h.srv.mu.Unlock()
	572	+	if len(h.srv.completes) != 1 {
	573	+	t.Fatalf("%d CompleteJob calls", len(h.srv.completes))
	574	+	}
	575	+	if c := h.srv.completes[0]; c.Conclusion != ermite.CiConclusion_CI_CONCLUSION_CANCELLED \|\| !c.RunnerShutdown {
	576	+	t.Fatalf("%+v", c)
	577	+	}
	578	+	if h.driver.stops != 1 {
	579	+	t.Errorf("stops=%d", h.driver.stops)
	580	+	}
	581	+	}
	582	+
	583	+	func TestShutdownHardStopsAnExecutorThatIgnoresIt(t *testing.T) {
	584	+	h := newHarness(t, func(_ testing.T, c execstream.Conn, _ execpb.Run, cancels <-chan execpb.Cancel) {
	585	+	stepStart(c, 1, "Set up job")
	586	+	stepStart(c, 2, "Run hang")
	587	+	logLineFrame(c, 2, "hung", false)
	588	+	<-cancels // received, ignored: keep running until the stream is cut
	589	+	select {}
	590	+	})
	591	+	h.runner.Timing.ShutdownGrace = 300 * time.Millisecond
	592	+	h.srv.jobs = []*ermite.Job{testJob("j1")}
	593	+	stop := h.run(t)
	594	+	eventually(t, "the job to run", func() bool { return len(h.srv.snapshotLines()) == 1 })
	595	+	start := time.Now()
	596	+	_ = stop()
	597	+	if time.Since(start) > 5*time.Second {
	598	+	t.Fatal("shutdown waited for an executor that does not answer")
	599	+	}
	600	+	h.srv.mu.Lock()
	601	+	defer h.srv.mu.Unlock()
	602	+	if len(h.srv.completes) != 1 \|\| h.srv.completes[0].Conclusion != ermite.CiConclusion_CI_CONCLUSION_CANCELLED \|\| !h.srv.completes[0].RunnerShutdown {
	603	+	t.Fatalf("%+v", h.srv.completes)
	604	+	}
	605	+	// Steps that never completed are closed by the runner.
	606	+	for _, u := range h.srv.completes[0].Steps {
	607	+	if u.State != ermite.StepState_STEP_STATE_COMPLETED \|\| u.Conclusion != ermite.CiConclusion_CI_CONCLUSION_CANCELLED {
	608	+	t.Errorf("step %d: %v/%v", u.Number, u.State, u.Conclusion)
	609	+	}
	610	+	}
	611	+	}
	612	+
	613	+	func TestExecutorDyingFailsTheJob(t *testing.T) {
	614	+	h := newHarness(t, func(_ testing.T, c execstream.Conn, _ execpb.Run, _ <-chan execpb.Cancel) {
	615	+	stepStart(c, 1, "Set up job")
	616	+	logLineFrame(c, 1, "partial", false)
	617	+	// the stream closes without a result
	618	+	})
	619	+	h.srv.jobs = []*ermite.Job{testJob("j1")}
	620	+	h.run(t)
	621	+	eventually(t, "CompleteJob", func() bool { return h.srv.completeCount() == 1 })
	622	+	h.srv.mu.Lock()
	623	+	defer h.srv.mu.Unlock()
	624	+	c := h.srv.completes[0]
	625	+	if c.Conclusion != ermite.CiConclusion_CI_CONCLUSION_FAILURE \|\| c.RunnerShutdown {
	626	+	t.Fatalf("%+v", c)
	627	+	}
	628	+	if len(h.srv.lines) != 2 \|\| !strings.HasPrefix(h.srv.lines[1], "##[error]The job's sandbox ended unexpectedly") {
	629	+	t.Fatalf("lines %q", h.srv.lines)
	630	+	}
	631	+	if len(c.Steps) != 1 \|\| c.Steps[0].State != ermite.StepState_STEP_STATE_COMPLETED \|\| c.Steps[0].Conclusion != ermite.CiConclusion_CI_CONCLUSION_FAILURE {
	632	+	t.Fatalf("%v", c.Steps)
	633	+	}
	634	+	}
	635	+
	636	+	func TestSandboxStartFailureFailsTheJobInStepOne(t *testing.T) {
	637	+	h := newHarness(t, simpleScript("x"))
	638	+	h.driver.startErr = errors.New("no kvm today")
	639	+	h.srv.jobs = []*ermite.Job{testJob("j1")}
	640	+	h.run(t)
	641	+	eventually(t, "CompleteJob", func() bool { return h.srv.completeCount() == 1 })
	642	+	h.srv.mu.Lock()
	643	+	defer h.srv.mu.Unlock()
	644	+	if c := h.srv.completes[0]; c.Conclusion != ermite.CiConclusion_CI_CONCLUSION_FAILURE {
	645	+	t.Fatalf("%v", c.Conclusion)
	646	+	}
	647	+	if len(h.srv.lines) != 1 \|\| !strings.Contains(h.srv.lines[0], "The sandbox did not start: no kvm today") {
	648	+	t.Fatalf("%q", h.srv.lines)
	649	+	}
	650	+	if h.driver.stops != 1 {
	651	+	t.Errorf("a sandbox that failed to start is still stopped: stops=%d", h.driver.stops)
	652	+	}
	653	+	}
	654	+
	655	+	func TestOutputsOverTheLimitFailTheJob(t *testing.T) {
	656	+	h := newHarness(t, func(_ testing.T, c execstream.Conn, _ execpb.Run, _ <-chan execpb.Cancel) {
	657	+	stepStart(c, 1, "Set up job")
	658	+	stepDone(c, 1, "Set up job", cOK)
	659	+	// Names that ermite would refuse: the job fails without outputs.
	660	+	jobDone(c, cOK, map[string]string{"1bad": "x", "fine": "y"}, false)
	661	+	})
	662	+	h.srv.jobs = []*ermite.Job{testJob("j1")}
	663	+	h.run(t)
	664	+	eventually(t, "CompleteJob", func() bool { return h.srv.completeCount() == 1 })
	665	+	h.srv.mu.Lock()
	666	+	defer h.srv.mu.Unlock()
	667	+	c := h.srv.completes[0]
	668	+	if c.Conclusion != ermite.CiConclusion_CI_CONCLUSION_FAILURE \|\| len(c.Outputs) != 0 {
	669	+	t.Fatalf("%v %v", c.Conclusion, len(c.Outputs))
	670	+	}
	671	+	if len(h.srv.lines) != 1 \|\| !strings.Contains(h.srv.lines[0], "is not valid") {
	672	+	t.Fatalf("%q", h.srv.lines)
	673	+	}
	674	+	}
	675	+
	676	+	func TestCheckOutputs(t *testing.T) {
	677	+	for _, tc := range []struct {
	678	+	in map[string]string
	679	+	want string
	680	+	}{
	681	+	{nil, ""},
	682	+	{map[string]string{"a-b_c1": "v", "_x": ""}, ""},
	683	+	{map[string]string{"1a": "v"}, "not valid"},
	684	+	{map[string]string{"a b": "v"}, "not valid"},
	685	+	{map[string]string{"": "v"}, "not valid"},
	686	+	{map[string]string{strings.Repeat("a", 256): "v"}, "not valid"},
	687	+	{map[string]string{"Foo": "1", "foo": "2"}, "differ only by case"},
	688	+	{map[string]string{"a": strings.Repeat("x", 1<<20+1)}, "larger than 1 MiB"},
	689	+	{map[string]string{"a": strings.Repeat("x", 600000), "b": strings.Repeat("x", 600000)}, "in total"},
	690	+	} {
	691	+	got := checkOutputs(tc.in)
	692	+	if (tc.want == "") != (got == "") \|\| !strings.Contains(got, tc.want) {
	693	+	t.Errorf("%v: %q, want %q", tc.in, got, tc.want)
	694	+	}
	695	+	}
	696	+	}
	697	+
	698	+	func TestLeakedSandboxStopsTakingJobsUntilReaped(t *testing.T) {
	699	+	h := newHarness(t, simpleScript("x"))
	700	+	h.driver.stopErr = sandbox.ErrLeaked
	701	+	h.driver.reapErr = errors.New("still leaked")
	702	+	h.runner.Timing.ReapRetry = 50 * time.Millisecond
	703	+	h.srv.jobs = []*ermite.Job{testJob("j1"), testJob("j2")}
	704	+	h.run(t)
	705	+	eventually(t, "first job done", func() bool { return h.srv.completeCount() == 1 })
	706	+	time.Sleep(400 * time.Millisecond)
	707	+	if h.srv.completeCount() != 1 {
	708	+	t.Fatal("a second job started while a sandbox is leaked and cannot be reaped")
	709	+	}
	710	+	// Once the reap works, jobs are taken again.
	711	+	h.driver.mu.Lock()
	712	+	h.driver.reapErr = nil
	713	+	h.driver.stopErr = nil
	714	+	h.driver.mu.Unlock()
	715	+	eventually(t, "the second job", func() bool { return h.srv.completeCount() == 2 })
	716	+	}
	717	+
	718	+	func TestReapAtStartAndIsolation(t *testing.T) {
	719	+	h := newHarness(t, simpleScript("x"))
	720	+	h.run(t)
	721	+	eventually(t, "reap", func() bool { h.driver.mu.Lock(); defer h.driver.mu.Unlock(); return h.driver.reaps >= 1 })
	722	+	for in, want := range map[sandbox.Isolation]ermite.RunnerIsolation{
	723	+	sandbox.IsolationVM: ermite.RunnerIsolation_RUNNER_ISOLATION_VM,
	724	+	sandbox.IsolationContainerSysbox: ermite.RunnerIsolation_RUNNER_ISOLATION_CONTAINER_SYSBOX,
	725	+	sandbox.IsolationContainerPrivileged: ermite.RunnerIsolation_RUNNER_ISOLATION_CONTAINER_PRIVILEGED,
	726	+	"other": ermite.RunnerIsolation_RUNNER_ISOLATION_UNSPECIFIED,
	727	+	} {
	728	+	if got := Isolation(in); got != want {
	729	+	t.Errorf("%s: %v", in, got)
	730	+	}
	731	+	}
	732	+	}
	733	+
	734	+	func TestRegisterStoresTheToken(t *testing.T) {
	735	+	srv := newFakeServer()
	736	+	url := srv.start(t)
	737	+	dir := t.TempDir() + "/state"
	738	+	cfg := Config{URL: url, Name: "r1", Labels: []string{"ubuntu-latest"}, StateDir: dir, Version: "v1"}
	739	+	st, err := Register(context.Background(), cfg, NewClient(url, "reg-token", http.DefaultTransport), sandbox.IsolationContainerSysbox, true)
	740	+	if err != nil {
	741	+	t.Fatal(err)
	742	+	}
	743	+	if st.RunnerToken != "ermiterun_new" \|\| st.RunnerID != "rid1" {
	744	+	t.Fatalf("%+v", st)
	745	+	}
	746	+	req := srv.registers[0]
	747	+	if req.Name != "r1" \|\| req.Isolation != ermite.RunnerIsolation_RUNNER_ISOLATION_CONTAINER_SYSBOX \|\| req.Os != "linux" \|\| (req.Arch != "x64" && req.Arch != "arm64") \|\| !req.Replace \|\| req.Version != "v1" \|\| len(req.Labels) != 1 {
	748	+	t.Fatalf("%v", req)
	749	+	}
	750	+	fi, err := os.Stat(StateFile(dir))
	751	+	if err != nil \|\| fi.Mode().Perm() != 0o600 {
	752	+	t.Fatalf("%v %v", fi, err)
	753	+	}
	754	+	got, err := LoadState(dir)
	755	+	if err != nil \|\| got != st {
	756	+	t.Fatalf("%+v %v", got, err)
	757	+	}
	758	+	// Anyone can read a state file with a loose mode: refuse it.
	759	+	if err := os.Chmod(StateFile(dir), 0o644); err != nil {
	760	+	t.Fatal(err)
	761	+	}
	762	+	if _, err := LoadState(dir); err == nil \|\| !strings.Contains(err.Error(), "accessible by other users") {
	763	+	t.Fatalf("%v", err)
	764	+	}
	765	+	if _, err := LoadState(t.TempDir()); !errors.Is(err, ErrNotRegistered) {
	766	+	t.Fatalf("%v", err)
	767	+	}
	768	+	// A refused registration stores nothing.
	769	+	dir2 := t.TempDir()
	770	+	cfg.StateDir = dir2
	771	+	if _, err := Register(context.Background(), cfg, NewClient(url, "bad", http.DefaultTransport), sandbox.IsolationVM, false); err == nil {
	772	+	t.Fatal("registration with a bad token succeeded")
	773	+	}
	774	+	if _, err := os.Stat(StateFile(dir2)); err == nil {
	775	+	t.Fatal("a state file was written")
	776	+	}
	777	+	}
	778	+
	779	+	func TestLoadConfig(t *testing.T) {
	780	+	env := map[string]string{EnvURL: " https://ermite.example ", EnvLabels: "ubuntu-latest, Ubuntu-24.04 ,ubuntu-latest,"}
	781	+	c, err := LoadConfig(func(k string) string { return env[k] }, "my host.local", "/state", 2, "v1")
	782	+	if err != nil {
	783	+	t.Fatal(err)
	784	+	}
	785	+	if c.URL != "https://ermite.example" \|\| c.Name != "my-host.local" \|\| strings.Join(c.Labels, ",") != "ubuntu-latest,Ubuntu-24.04" \|\| c.Capacity != 2 {
	786	+	t.Fatalf("%+v", c)
	787	+	}
	788	+	env[EnvName] = "bad name"
	789	+	if _, err := LoadConfig(func(k string) string { return env[k] }, "h", "/s", 1, ""); err == nil {
	790	+	t.Error("an invalid name was accepted")
	791	+	}
	792	+	env[EnvName] = "ok"
	793	+	env[EnvLabels] = "good,bad label"
	794	+	if _, err := LoadConfig(func(k string) string { return env[k] }, "h", "/s", 1, ""); err == nil {
	795	+	t.Error("an invalid label was accepted")
	796	+	}
	797	+	}
	798	+
	799	+	var _ = ermiteconnect.RunnerServiceClient(nil)
	800	+
	801	+	// The job timeout counts from the receipt of the job, and the host enforces it
	802	+	// even when the executor does not: the sandbox is stopped and the job ends
	803	+	// timed_out.
	804	+	func TestHostEnforcesTheJobTimeout(t *testing.T) {
	805	+	h := newHarness(t, func(_ testing.T, c execstream.Conn, _ execpb.Run, cancels <-chan execpb.Cancel) {
	806	+	stepStart(c, 1, "Set up job")
	807	+	logLineFrame(c, 1, "stuck", false)
	808	+	cn := <-cancels
	809	+	if cn.Kind != execpb.CancelKind_CANCEL_KIND_TIMEOUT {
	810	+	t.Errorf("kind %v", cn.Kind)
	811	+	}
	812	+	select {} // the executor ignores the cancel
	813	+	})
	814	+	h.runner.Timing.TimeoutGrace = 200 * time.Millisecond
	815	+	job := testJob("j1")
	816	+	job.TimeoutMinutes = 1
	817	+	h.runner.Job = func(tok string) ermiteconnect.RunnerServiceClient {
	818	+	return NewClient(h.url, tok, http.DefaultTransport)
	819	+	}
	820	+	h.runner.now = func() time.Time { return time.Now().Add(-59time.Second - 700time.Millisecond) } // received 59.7 s ago
	821	+	h.srv.jobs = []*ermite.Job{job}
	822	+	start := time.Now()
	823	+	h.run(t)
	824	+	eventually(t, "CompleteJob", func() bool { return h.srv.completeCount() == 1 })
	825	+	if d := time.Since(start); d > 5*time.Second {
	826	+	t.Fatalf("took %v", d)
	827	+	}
	828	+	h.srv.mu.Lock()
	829	+	defer h.srv.mu.Unlock()
	830	+	c := h.srv.completes[0]
	831	+	if c.Conclusion != ermite.CiConclusion_CI_CONCLUSION_TIMED_OUT \|\| c.RunnerShutdown {
	832	+	t.Fatalf("%v", c.Conclusion)
	833	+	}
	834	+	for _, u := range c.Steps {
	835	+	if u.State != ermite.StepState_STEP_STATE_COMPLETED {
	836	+	t.Errorf("step %d not completed", u.Number)
	837	+	}
	838	+	}
	839	+	}
	840	+
	841	+	// The remaining time, not the whole timeout, goes to the executor.
	842	+	func TestRunCarriesTheRemainingDeadline(t *testing.T) {
	843	+	var spec *execpb.JobSpec
	844	+	inner := simpleScript("x")
	845	+	h := newHarness(t, func(t testing.T, c execstream.Conn, run execpb.Run, cn <-chan execpb.Cancel) {
	846	+	spec = run.Job
	847	+	inner(t, c, run, cn)
	848	+	})
	849	+	h.runner.now = func() time.Time { return time.Now().Add(-3 * time.Minute) }
	850	+	h.srv.jobs = []*ermite.Job{testJob("j1")} // 5 minutes
	851	+	h.run(t)
	852	+	eventually(t, "CompleteJob", func() bool { return h.srv.completeCount() == 1 })
	853	+	if spec == nil \|\| spec.TimeoutMinutes != 5 \|\| spec.DeadlineMs < 110_000 \|\| spec.DeadlineMs > 120_000 {
	854	+	t.Fatalf("%v", spec)
	855	+	}
	856	+	}
	857	+
	858	+	// ---- the guest is untrusted: step numbers and lines are checked (M3)
	859	+
	860	+	// hostile runs a job whose executor sends the frames of body, and checks that
	861	+	// the job fails, the sandbox is destroyed, the server never saw a violation of
	862	+	// its contract, and a line in the log names the reason.
	863	+	func hostile(t testing.T, wantReason string, body script) harness {
	864	+	t.Helper()
	865	+	h := newHarness(t, func(t testing.T, c execstream.Conn, run execpb.Run, cn <-chan execpb.Cancel) {
	866	+	body(t, c, run, cn)
	867	+	// A real executor would go on; this one waits for the host to cut it.
	868	+	<-cn
	869	+	})
	870	+	h.srv.jobs = []*ermite.Job{testJob("j1")}
	871	+	h.run(t)
	872	+	eventually(t, "CompleteJob", func() bool { return h.srv.completeCount() == 1 })
	873	+	h.srv.mu.Lock()
	874	+	defer h.srv.mu.Unlock()
	875	+	c := h.srv.completes[0]
	876	+	if c.Conclusion != ermite.CiConclusion_CI_CONCLUSION_FAILURE \|\| len(c.Outputs) != 0 {
	877	+	t.Fatalf("conclusion %v outputs %v", c.Conclusion, c.Outputs)
	878	+	}
	879	+	found := false
	880	+	for _, l := range h.srv.lines {
	881	+	if strings.Contains(l, "broke the protocol") && strings.Contains(l, wantReason) {
	882	+	found = true
	883	+	}
	884	+	}
	885	+	if !found {
	886	+	t.Fatalf("no error line mentioning %q in %q", wantReason, h.srv.lines)
	887	+	}
	888	+	for _, u := range c.Steps {
	889	+	if u.State != ermite.StepState_STEP_STATE_COMPLETED \|\| u.Number < 1 \|\| u.Number > 1000 {
	890	+	t.Errorf("invalid final step %v", u)
	891	+	}
	892	+	}
	893	+	if h.driver.stops != 1 {
	894	+	t.Errorf("the sandbox was not destroyed: stops=%d", h.driver.stops)
	895	+	}
	896	+	return h
	897	+	}
	898	+
	899	+	func TestGuestStepNumberZeroFailsTheJob(t *testing.T) {
	900	+	hostile(t, "outside 1..1000", func(_ testing.T, c execstream.Conn, _ execpb.Run, _ <-chan execpb.Cancel) {
	901	+	stepStart(c, 1, "Set up job")
	902	+	stepStart(c, 0, "zero")
	903	+	})
	904	+	}
	905	+
	906	+	func TestGuestStepNumberOverTheLimitFailsTheJob(t *testing.T) {
	907	+	hostile(t, "outside 1..1000", func(_ testing.T, c execstream.Conn, _ execpb.Run, _ <-chan execpb.Cancel) {
	908	+	stepStart(c, 1, "Set up job")
	909	+	stepStart(c, 1001, "far")
	910	+	})
	911	+	}
	912	+
	913	+	func TestGuestStepGoingBackFailsTheJob(t *testing.T) {
	914	+	hostile(t, "started after step 3", func(_ testing.T, c execstream.Conn, _ execpb.Run, _ <-chan execpb.Cancel) {
	915	+	stepStart(c, 1, "a")
	916	+	stepStart(c, 3, "c")
	917	+	stepStart(c, 2, "b")
	918	+	})
	919	+	}
	920	+
	921	+	func TestGuestStepRestartedFailsTheJob(t *testing.T) {
	922	+	hostile(t, "started after step 1", func(_ testing.T, c execstream.Conn, _ execpb.Run, _ <-chan execpb.Cancel) {
	923	+	stepStart(c, 1, "a")
	924	+	stepDone(c, 1, "a", cOK)
	925	+	stepStart(c, 1, "again")
	926	+	})
	927	+	}
	928	+
	929	+	func TestGuestCompletedStepIsNeverTouchedAgain(t *testing.T) {
	930	+	h := hostile(t, "completed twice", func(_ testing.T, c execstream.Conn, _ execpb.Run, _ <-chan execpb.Cancel) {
	931	+	stepStart(c, 1, "a")
	932	+	stepDone(c, 1, "a", cOK)
	933	+	stepDone(c, 1, "a", cFail)
	934	+	})
	935	+	if u := h.srv.Result("j1").Steps[1]; u == nil \|\| u.Conclusion != ermite.CiConclusion_CI_CONCLUSION_SUCCESS {
	936	+	t.Errorf("a completed step was changed: %v", u.Conclusion)
	937	+	}
	938	+	}
	939	+
	940	+	func TestGuestCompletingAnUnknownStepFailsTheJob(t *testing.T) {
	941	+	hostile(t, "without having started", func(_ testing.T, c execstream.Conn, _ execpb.Run, _ <-chan execpb.Cancel) {
	942	+	stepStart(c, 1, "a")
	943	+	stepDone(c, 7, "ghost", cOK)
	944	+	})
	945	+	}
	946	+
	947	+	func TestGuestLogForAnUnstartedOrCompletedStepFailsTheJob(t *testing.T) {
	948	+	hostile(t, "has not started", func(_ testing.T, c execstream.Conn, _ execpb.Run, _ <-chan execpb.Cancel) {
	949	+	stepStart(c, 1, "a")
	950	+	logLineFrame(c, 5, "x", false)
	951	+	})
	952	+	hostile(t, "which is completed", func(_ testing.T, c execstream.Conn, _ execpb.Run, _ <-chan execpb.Cancel) {
	953	+	stepStart(c, 1, "a")
	954	+	stepDone(c, 1, "a", cOK)
	955	+	logLineFrame(c, 1, "late", false)
	956	+	})
	957	+	}
	958	+
	959	+	func TestGuestLogGoingBackAStepFailsTheJob(t *testing.T) {
	960	+	hostile(t, "after one for step 2", func(_ testing.T, c execstream.Conn, _ execpb.Run, _ <-chan execpb.Cancel) {
	961	+	stepStart(c, 1, "a")
	962	+	stepStart(c, 2, "b")
	963	+	logLineFrame(c, 2, "x", false)
	964	+	logLineFrame(c, 1, "y", false)
	965	+	})
	966	+	}
	967	+
	968	+	func TestGuestOversizedLineFailsTheJob(t *testing.T) {
	969	+	hostile(t, "bytes (the limit is 65536)", func(_ testing.T, c execstream.Conn, _ execpb.Run, _ <-chan execpb.Cancel) {
	970	+	stepStart(c, 1, "a")
	971	+	logLineFrame(c, 1, strings.Repeat("x", 64*1024+1), false)
	972	+	})
	973	+	}
	974	+
	975	+	// Text that the server would refuse is repaired, not forwarded: a line feed
	976	+	// makes more lines and long names are cut. (Invalid UTF-8 cannot reach the host:
	977	+	// proto3 strings are validated when the frame is decoded, which fails the
	978	+	// session; the executor replaces invalid bytes before it sends.)
	979	+	func TestGuestTextIsMadeValidForTheServer(t *testing.T) {
	980	+	h := newHarness(t, func(_ testing.T, c execstream.Conn, _ execpb.Run, _ <-chan execpb.Cancel) {
	981	+	stepStart(c, 1, strings.Repeat("é", 800)) // 1,600 bytes
	982	+	logLineFrame(c, 1, "two\nlines", false)
	983	+	stepDone(c, 1, "", cOK)
	984	+	jobDone(c, cOK, nil, false)
	985	+	})
	986	+	h.srv.jobs = []*ermite.Job{testJob("j1")}
	987	+	h.run(t)
	988	+	eventually(t, "CompleteJob", func() bool { return h.srv.completeCount() == 1 })
	989	+	h.srv.mu.Lock()
	990	+	defer h.srv.mu.Unlock()
	991	+	if strings.Join(h.srv.lines, "\|") != "two\|lines" {
	992	+	t.Fatalf("%q", h.srv.lines)
	993	+	}
	994	+	if n := h.srv.steps[1].Name; len(n) > 1024 \|\| !utf8.ValidString(n) {
	995	+	t.Fatalf("name of %d bytes", len(n))
	996	+	}
	997	+	}
	998	+
	999	+	// A guest that sends steps without end cannot grow the memory of the host.
	1000	+	func TestGuestFloodIsBounded(t *testing.T) {
	1001	+	h := hostile(t, "outside 1..1000", func(_ testing.T, c execstream.Conn, _ execpb.Run, _ <-chan execpb.Cancel) {
	1002	+	for n := 1; n <= 1100; n++ {
	1003	+	stepStart(c, n, "s")
	1004	+	stepDone(c, n, "s", cOK)
	1005	+	}
	1006	+	})
	1007	+	_ = h
	1008	+	}
	1009	+
	1010	+	// The server's line count is the truth (m6): when the answer to an AppendLog
	1011	+	// says the server holds more lines than the request accounts for, the next
	1012	+	// offset starts from the server's count and no gap or repeat is created.
	1013	+	func TestAppendLogResyncsToTheServersCount(t *testing.T) {
	1014	+	h := newHarness(t, func(_ testing.T, c execstream.Conn, _ execpb.Run, _ <-chan execpb.Cancel) {
	1015	+	stepStart(c, 1, "a")
	1016	+	logLineFrame(c, 1, "l1", false)
	1017	+	time.Sleep(150 * time.Millisecond) // a first request of its own
	1018	+	logLineFrame(c, 1, "l2", false)
	1019	+	logLineFrame(c, 1, "l3", false)
	1020	+	stepDone(c, 1, "a", cOK)
	1021	+	jobDone(c, cOK, nil, false)
	1022	+	})
	1023	+	h.runner.Timing.LogFlush = 30 * time.Millisecond
	1024	+	h.srv.jobs = []*ermite.Job{testJob("j1")}
	1025	+	// A line the runner never sent appears in the server's log (a retry whose
	1026	+	// answer was lost, in a real server): the runner must continue after it.
	1027	+	h.srv.mu.Lock()
	1028	+	h.srv.appendHook = func(n int, r *ermite.AppendLogRequest) (bool, error) {
	1029	+	if n == 1 {
	1030	+	_, _ = h.srv.store.Apply(&ermite.AppendLogRequest{JobId: r.JobId, Step: 1, Offset: 0, Lines: []*ermite.LogLine{
	1031	+	{Time: timestamppb.Now(), Text: "planted"}, {Time: timestamppb.Now(), Text: "planted2"}}})
	1032	+	return false, nil
	1033	+	}
	1034	+	return false, nil
	1035	+	}
	1036	+	h.srv.mu.Unlock()
	1037	+	h.run(t)
	1038	+	eventually(t, "CompleteJob", func() bool { return h.srv.completeCount() == 1 })
	1039	+	h.srv.mu.Lock()
	1040	+	defer h.srv.mu.Unlock()
	1041	+	// The first request (offset 0, "l1") was skipped by the server because the
	1042	+	// planted lines already took offsets 0 and 1; the runner learns it holds 2
	1043	+	// lines (it expected 1), and goes on at offset 2: no gap, no repeat.
	1044	+	if len(h.srv.lines) < 3 \|\| h.srv.lines[0] != "planted" \|\| h.srv.lines[len(h.srv.lines)-1] != "l3" {
	1045	+	t.Fatalf("%q", h.srv.lines)
	1046	+	}
	1047	+	if !strings.Contains(h.logs.String(), "continuing from its count") {
	1048	+	t.Fatalf("no warning about the resync:\n%s", h.logs.String())
	1049	+	}
	1050	+	}
	1051	+
	1052	+	// ---- M4: the server refuses the result itself
	1053	+
	1054	+	func TestCompleteJobRefusedIsRetriedAsAFailureWithoutOutputs(t *testing.T) {
	1055	+	h := newHarness(t, simpleScript("hello"))
	1056	+	h.srv.jobs = []*ermite.Job{testJob("j1")}
	1057	+	h.srv.completeHook = func(n int) (bool, error) {
	1058	+	if n == 1 { // e.g. an output the server's rules refuse
	1059	+	return false, connect.NewError(connect.CodeInvalidArgument, io.EOF)
	1060	+	}
	1061	+	return false, nil
	1062	+	}
	1063	+	h.run(t)
	1064	+	eventually(t, "two CompleteJob calls", func() bool { return h.srv.completeCount() == 2 })
	1065	+	time.Sleep(100 * time.Millisecond)
	1066	+	h.srv.mu.Lock()
	1067	+	defer h.srv.mu.Unlock()
	1068	+	if len(h.srv.completes) != 2 {
	1069	+	t.Fatalf("%d calls", len(h.srv.completes))
	1070	+	}
	1071	+	first, second := h.srv.completes[0], h.srv.completes[1]
	1072	+	if first.Conclusion != ermite.CiConclusion_CI_CONCLUSION_SUCCESS \|\| first.Outputs["out"] != "v" {
	1073	+	t.Fatalf("first: %v %v", first.Conclusion, first.Outputs)
	1074	+	}
	1075	+	if second.Conclusion != ermite.CiConclusion_CI_CONCLUSION_FAILURE \|\| len(second.Outputs) != 0 \|\| second.RunnerShutdown {
	1076	+	t.Fatalf("second: %v %v", second.Conclusion, second.Outputs)
	1077	+	}
	1078	+	if len(second.Steps) != 3 {
	1079	+	t.Fatalf("steps %v", second.Steps)
	1080	+	}
	1081	+	for _, u := range second.Steps {
	1082	+	if u.State != ermite.StepState_STEP_STATE_COMPLETED \|\| u.StartedAt == nil \|\| u.CompletedAt == nil \|\| u.CompletedAt.AsTime().Before(u.StartedAt.AsTime()) {
	1083	+	t.Errorf("invalid step %v", u)
	1084	+	}
	1085	+	}
	1086	+	last := h.srv.lines[len(h.srv.lines)-1]
	1087	+	if !strings.HasPrefix(last, "##[error]The runner could not report the result of the job: invalid_argument") {
	1088	+	t.Fatalf("last line %q", last)
	1089	+	}
	1090	+	}
	1091	+
	1092	+	func TestCompleteJobRefusedTwiceGivesUp(t *testing.T) {
	1093	+	h := newHarness(t, simpleScript("x"))
	1094	+	h.srv.jobs = []*ermite.Job{testJob("j1")}
	1095	+	h.srv.completeHook = func(int) (bool, error) {
	1096	+	return false, connect.NewError(connect.CodeFailedPrecondition, io.EOF) // not a JobEnded detail
	1097	+	}
	1098	+	h.run(t)
	1099	+	eventually(t, "two calls", func() bool { return h.srv.completeCount() == 2 })
	1100	+	time.Sleep(200 * time.Millisecond)
	1101	+	if n := h.srv.completeCount(); n != 2 {
	1102	+	t.Fatalf("%d calls: one retry only", n)
	1103	+	}
	1104	+	if !strings.Contains(h.logs.String(), `"level":"error"`) \|\| !strings.Contains(h.logs.String(), "refused again") {
	1105	+	t.Fatalf("no error log:\n%s", h.logs.String())
	1106	+	}
	1107	+	}
	1108	+
	1109	+	// ---- G1: structured logs with the job's fields, warn for retries, error for refusals
	1110	+
	1111	+	func TestLogsCarryTheJobFieldsAndLevels(t *testing.T) {
	1112	+	h := newHarness(t, simpleScript("x"))
	1113	+	h.srv.jobs = []*ermite.Job{testJob("j1")}
	1114	+	h.srv.appendHook = func(n int, _ *ermite.AppendLogRequest) (bool, error) {
	1115	+	if n == 1 {
	1116	+	return false, connect.NewError(connect.CodeUnavailable, io.EOF)
	1117	+	}
	1118	+	return false, nil
	1119	+	}
	1120	+	h.srv.beatHook = func(n int) error { return nil }
	1121	+	h.run(t)
	1122	+	eventually(t, "CompleteJob", func() bool { return h.srv.completeCount() == 1 })
	1123	+	var retry, assigned map[string]any
	1124	+	for _, l := range strings.Split(h.logs.String(), "\n") {
	1125	+	if l == "" {
	1126	+	continue
	1127	+	}
	1128	+	var m map[string]any
	1129	+	if err := json.Unmarshal([]byte(l), &m); err != nil {
	1130	+	t.Fatalf("not JSON: %q", l)
	1131	+	}
	1132	+	switch m["message"] {
	1133	+	case "call failed, retrying":
	1134	+	retry = m
	1135	+	case "job assigned":
	1136	+	assigned = m
	1137	+	}
	1138	+	}
	1139	+	for name, m := range map[string]map[string]any{"retry": retry, "assigned": assigned} {
	1140	+	if m == nil {
	1141	+	t.Fatalf("no %s event in\n%s", name, h.logs.String())
	1142	+	}
	1143	+	if m["job_id"] != "j1" \|\| m["run_id"] != float64(7) \|\| m["attempt"] != float64(1) {
	1144	+	t.Errorf("%s event lacks the job fields: %v", name, m)
	1145	+	}
	1146	+	}
	1147	+	if retry["level"] != "warn" \|\| retry["call"] != "AppendLog" {
	1148	+	t.Errorf("a retry is a warning: %v", retry)
	1149	+	}
	1150	+	}
	1151	+
	1152	+	// The sizes of a sandbox come from ERMITE_RUNNER_JOB_CPUS, _JOB_MEMORY and
	1153	+	// _JOB_DISK (sandbox.Config.Resolve, applied by the driver at Start): the host
	1154	+	// asks for none, so that nothing in a job can change them.
	1155	+	func TestSandboxSizesAreLeftToTheDriversConfiguration(t *testing.T) {
	1156	+	h := newHarness(t, simpleScript("x"))
	1157	+	h.srv.jobs = []*ermite.Job{testJob("j1")}
	1158	+	h.run(t)
	1159	+	eventually(t, "CompleteJob", func() bool { return h.srv.completeCount() == 1 })
	1160	+	h.driver.mu.Lock()
	1161	+	defer h.driver.mu.Unlock()
	1162	+	if len(h.driver.specs) != 1 {
	1163	+	t.Fatalf("%v", h.driver.specs)
	1164	+	}
	1165	+	if s := h.driver.specs[0]; s.JobID != "j1" \|\| s.CPUs != 0 \|\| s.MemoryMiB != 0 \|\| s.DiskMiB != 0 {
	1166	+	t.Fatalf("%+v", s)
	1167	+	}
	1168	+	// And the configuration fills them (the real rule, in package sandbox).
	1169	+	cfg, err := sandbox.LoadConfig(func(k string) string {
	1170	+	return map[string]string{sandbox.EnvJobCPUs: "3", sandbox.EnvJobMemory: "2G", sandbox.EnvJobDisk: "10G"}[k]
	1171	+	})
	1172	+	if err != nil {
	1173	+	t.Fatal(err)
	1174	+	}
	1175	+	if got := cfg.Resolve(h.driver.specs[0]); got.CPUs != 3 \|\| got.MemoryMiB != 2048 \|\| got.DiskMiB != 10240 {
	1176	+	t.Fatalf("%+v", got)
	1177	+	}
	1178	+	}
	1179	+
	1180	+	// ---- M5: shutdown
	1181	+
	1182	+	// A job whose acknowledgement is pending when the runner shuts down is left to
	1183	+	// the server (it requeues it after 30 s): the sandbox is never started, no
	1184	+	// result is reported, and Run returns promptly.
	1185	+	func TestShutdownBeforeTheAcknowledgementLeavesTheJobToTheServer(t *testing.T) {
	1186	+	h := newHarness(t, simpleScript("x"))
	1187	+	h.srv.jobs = []*ermite.Job{testJob("j1")}
	1188	+	h.srv.beatHook = func(int) error { return connect.NewError(connect.CodeUnavailable, io.EOF) } // never acknowledged
	1189	+	stop := h.run(t)
	1190	+	eventually(t, "a failed acknowledgement", func() bool { h.srv.mu.Lock(); defer h.srv.mu.Unlock(); return len(h.srv.beats) >= 2 })
	1191	+	start := time.Now()
	1192	+	if err := stop(); err != nil {
	1193	+	t.Fatal(err)
	1194	+	}
	1195	+	if time.Since(start) > 3*time.Second {
	1196	+	t.Fatalf("Run took %v to return", time.Since(start))
	1197	+	}
	1198	+	h.driver.mu.Lock()
	1199	+	defer h.driver.mu.Unlock()
	1200	+	if h.driver.news != 0 \|\| h.driver.started != 0 {
	1201	+	t.Fatalf("a sandbox was created for a job that was never acknowledged: news=%d started=%d", h.driver.news, h.driver.started)
	1202	+	}
	1203	+	if n := h.srv.completeCount(); n != 0 {
	1204	+	t.Fatalf("CompleteJob was called %d times", n)
	1205	+	}
	1206	+	if !strings.Contains(h.logs.String(), "before the job was acknowledged") {
	1207	+	t.Fatalf("no explanation in the log:\n%s", h.logs.String())
	1208	+	}
	1209	+	}
	1210	+
	1211	+	// The graceful shutdown of the defaults fits the time the unit gives.
	1212	+	func TestShutdownBudgetFitsTheUnit(t *testing.T) {
	1213	+	budget := Timing{}.ShutdownBudget()
	1214	+	if budget != 90*time.Second {
	1215	+	t.Fatalf("the budget is %v, README and contrib/ermite-runner.service say 90 s", budget)
	1216	+	}
	1217	+	unit, err := os.ReadFile("../../contrib/ermite-runner.service")
	1218	+	if err != nil {
	1219	+	t.Skip(err)
	1220	+	}
	1221	+	m := regexp.MustCompile(`(?m)^TimeoutStopSec=(\d+)$`).FindStringSubmatch(string(unit))
	1222	+	if m == nil {
	1223	+	t.Fatal("no TimeoutStopSec in the unit")
	1224	+	}
	1225	+	secs, _ := strconv.Atoi(m[1])
	1226	+	if time.Duration(secs)time.Second < budget+15time.Second {
	1227	+	t.Fatalf("TimeoutStopSec=%d does not leave 15 s of margin over the %v budget", secs, budget)
	1228	+	}
	1229	+	}
	1230	+
	1231	+	// ---- m7: the state directory is private
	1232	+
	1233	+	func TestSaveStateMakesAnExistingDirectoryPrivate(t *testing.T) {
	1234	+	dir := filepath.Join(t.TempDir(), "state")
	1235	+	if err := os.Mkdir(dir, 0o755); err != nil {
	1236	+	t.Fatal(err)
	1237	+	}
	1238	+	if err := SaveState(dir, State{URL: "https://x", RunnerID: "r", Name: "n", RunnerToken: "ermiterun_t"}); err != nil {
	1239	+	t.Fatal(err)
	1240	+	}
	1241	+	fi, err := os.Stat(dir)
	1242	+	if err != nil \|\| fi.Mode().Perm() != 0o700 {
	1243	+	t.Fatalf("%v %v", fi, err)
	1244	+	}
	1245	+	if fi, _ := os.Stat(StateFile(dir)); fi.Mode().Perm() != 0o600 {
	1246	+	t.Fatalf("%v", fi.Mode())
	1247	+	}
	1248	+	}
	1249	+
	1250	+	// mountNamespaceOptions are the [Service] options that give a unit a mount
	1251	+	// namespace of its own. The unit must set none of them: in container mode the
	1252	+	// runner loop-mounts the scratch disk of a job; the Docker daemon, in the host's
	1253	+	// namespace, binds it into the container, and a mount made inside a private mount
	1254	+	// namespace of the unit is invisible to it (GO-001). Besides the options that say
	1255	+	// so (PrivateTmp, ProtectHome, ProtectSystem...), systemd builds a mount
	1256	+	// namespace for ProtectKernelModules and ProtectKernelLogs too.
	1257	+	var mountNamespaceOptions = []string{
	1258	+	"PrivateTmp", "PrivateDevices", "PrivateMounts", "PrivateUsers", "ProtectHome", "ProtectSystem",
	1259	+	"ProtectKernelModules", "ProtectKernelLogs", "ProtectKernelTunables", "ProtectControlGroups",
	1260	+	"ProtectProc", "ProcSubset", "ProtectHostname", "ReadOnlyPaths", "ReadWritePaths", "InaccessiblePaths",
	1261	+	"ExecPaths", "NoExecPaths", "TemporaryFileSystem", "BindPaths", "BindReadOnlyPaths", "MountFlags",
	1262	+	"MountAPIVFS", "RootDirectory", "RootImage", "RootEphemeral", "BindLogSockets", "LogNamespace",
	1263	+	"PrivateIPC", "PrivatePIDs", "MountImages", "ExtensionImages", "ExtensionDirectories",
	1264	+	}
	1265	+
	1266	+	// unitMountNamespaceOptions returns the lines of the [Service] section of a unit
	1267	+	// that set one of mountNamespaceOptions. Comments, which name the options on
	1268	+	// purpose, and other sections are ignored.
	1269	+	func unitMountNamespaceOptions(unit string) []string {
	1270	+	var found []string
	1271	+	inService := false
	1272	+	for _, line := range strings.Split(unit, "\n") {
	1273	+	line = strings.TrimSpace(line)
	1274	+	if strings.HasPrefix(line, "[") {
	1275	+	inService = line == "[Service]"
	1276	+	continue
	1277	+	}
	1278	+	if !inService \|\| line == "" \|\| strings.HasPrefix(line, "#") {
	1279	+	continue
	1280	+	}
	1281	+	key, _, _ := strings.Cut(line, "=")
	1282	+	for _, f := range mountNamespaceOptions {
	1283	+	if strings.EqualFold(strings.TrimSpace(key), f) {
	1284	+	found = append(found, line)
	1285	+	}
	1286	+	}
	1287	+	}
	1288	+	return found
	1289	+	}
	1290	+
	1291	+	func TestUnitHasNoMountNamespaceOptions(t *testing.T) {
	1292	+	unit, err := os.ReadFile("../../contrib/ermite-runner.service")
	1293	+	if err != nil {
	1294	+	t.Skip(err)
	1295	+	}
	1296	+	if found := unitMountNamespaceOptions(string(unit)); len(found) > 0 {
	1297	+	t.Fatalf("the unit sets options that give it a mount namespace of its own, which hides the loop mounts from Docker: %q", found)
	1298	+	}
	1299	+	// The hardening that stays must still be there.
	1300	+	for _, want := range []string{"NoNewPrivileges=yes", "RestrictSUIDSGID=yes", "ProtectClock=yes", "RestrictAddressFamilies="} {
	1301	+	if !strings.Contains(string(unit), "\n"+want) {
	1302	+	t.Errorf("the unit lost %s", want)
	1303	+	}
	1304	+	}
	1305	+	}
	1306	+
	1307	+	// The check sees the options in [Service] (also with spaces and another case),
	1308	+	// and ignores comments and other sections.
	1309	+	func TestUnitMountNamespaceCheckSeesOptions(t *testing.T) {
	1310	+	unit := "[Unit]\nPrivateTmp=yes\n[Service]\nPrivateTmp=yes\n protecthome = read-only\nProtectKernelModules=yes\n# ProtectSystem=strict\nNoNewPrivileges=yes\n[Install]\nProtectSystem=yes\n"
	1311	+	got := unitMountNamespaceOptions(unit)
	1312	+	want := []string{"PrivateTmp=yes", "protecthome = read-only", "ProtectKernelModules=yes"}
	1313	+	if strings.Join(got, "\|") != strings.Join(want, "\|") {
	1314	+	t.Fatalf("%q", got)
	1315	+	}
	1316	+	}

A internal/host/shutdown_test.go +99 -0

@@ -0,0 +1,99 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package host
	4	+
	5	+	import (
	6	+	"context"
	7	+	"sync"
	8	+	"testing"
	9	+	"time"
	10	+
	11	+	"connectrpc.com/connect"
	12	+	"github.com/bnema/zerowrap"
	13	+
	14	+	ermite "bnema.dev/ermite-api/gen/ermite"
	15	+	"bnema.dev/ermite-runner/internal/execstream/execpb"
	16	+	)
	17	+
	18	+	// stubClient answers the calls of the final report. CompleteJob records the
	19	+	// deadline of its context, then waits for the context to end.
	20	+	type stubClient struct {
	21	+	mu sync.Mutex
	22	+	deadline time.Time
	23	+	hang bool // AppendLog waits for its context to end
	24	+	}
	25	+
	26	+	func (c stubClient) UpdateJob(context.Context, connect.Request[ermite.UpdateJobRequest]) (*connect.Response[ermite.UpdateJobResponse], error) {
	27	+	return connect.NewResponse(&ermite.UpdateJobResponse{}), nil
	28	+	}
	29	+
	30	+	func (c stubClient) AppendLog(ctx context.Context, r connect.Request[ermite.AppendLogRequest]) (*connect.Response[ermite.AppendLogResponse], error) {
	31	+	c.mu.Lock()
	32	+	hang := c.hang
	33	+	c.mu.Unlock()
	34	+	if hang {
	35	+	<-ctx.Done()
	36	+	return nil, ctx.Err()
	37	+	}
	38	+	return connect.NewResponse(&ermite.AppendLogResponse{NextOffset: r.Msg.Offset + int64(len(r.Msg.Lines))}), nil
	39	+	}
	40	+
	41	+	func (c stubClient) Heartbeat(context.Context, connect.Request[ermite.HeartbeatRequest]) (*connect.Response[ermite.HeartbeatResponse], error) {
	42	+	return connect.NewResponse(&ermite.HeartbeatResponse{}), nil
	43	+	}
	44	+
	45	+	func (c stubClient) WatchJob(context.Context, connect.Request[ermite.WatchJobRequest]) (*connect.ServerStreamForClient[ermite.WatchJobResponse], error) {
	46	+	return nil, connect.NewError(connect.CodeUnimplemented, nil)
	47	+	}
	48	+
	49	+	func (c stubClient) CompleteJob(ctx context.Context, _ connect.Request[ermite.CompleteJobRequest]) (*connect.Response[ermite.CompleteJobResponse], error) {
	50	+	c.mu.Lock()
	51	+	c.deadline, _ = ctx.Deadline()
	52	+	c.mu.Unlock()
	53	+	return connect.NewResponse(&ermite.CompleteJobResponse{}), nil
	54	+	}
	55	+
	56	+	// The report of a job gets ShutdownReport once the runner's shutdown channel is
	57	+	// closed, even when the stop timers did not mark the job as stopping (GO-004),
	58	+	// and CompleteDeadline otherwise.
	59	+	func TestReportBudgetFollowsTheShutdownChannel(t *testing.T) {
	60	+	for _, tc := range []struct {
	61	+	name string
	62	+	shutdown bool
	63	+	want time.Duration
	64	+	}{
	65	+	{"running", false, time.Hour},
	66	+	{"shutdown", true, 30 * time.Second},
	67	+	} {
	68	+	t.Run(tc.name, func(t *testing.T) {
	69	+	tm := Timing{
	70	+	CallTimeout: 24 * time.Hour, CompleteDeadline: time.Hour, ShutdownReport: 30 * time.Second,
	71	+	BackoffMin: time.Millisecond, BackoffMax: time.Millisecond,
	72	+	}
	73	+	cl := &stubClient{}
	74	+	ctx, cancel := context.WithCancel(context.Background())
	75	+	defer cancel()
	76	+	j := &jobRun{client: cl, t: tm, ctx: ctx, job: &ermite.Job{JobId: "j1"}, steps: map[int32]*ermite.StepUpdate{},
	77	+	cancelCh: make(chan *execpb.Cancel, 1), log: zerowrap.New(zerowrap.Config{Level: "debug", Format: "json", Output: &logBuf{}})}
	78	+	j.outbox = newOutbox(cl, "j1", tm, func(error) {}, j.log)
	79	+	j.outbox.start(ctx)
	80	+	shutdown := make(chan struct{})
	81	+	if tc.shutdown {
	82	+	close(shutdown)
	83	+	}
	84	+	if j.isStopping() {
	85	+	t.Fatal("the job must not be marked as stopping: only the channel is closed")
	86	+	}
	87	+	start := time.Now()
	88	+	j.report(ctx, outcome{conclusion: ermite.CiConclusion_CI_CONCLUSION_SUCCESS}, shutdown)
	89	+	cl.mu.Lock()
	90	+	defer cl.mu.Unlock()
	91	+	if cl.deadline.IsZero() {
	92	+	t.Fatal("CompleteJob was not called")
	93	+	}
	94	+	if got := cl.deadline.Sub(start); got > tc.want+time.Second \|\| got < tc.want-time.Second {
	95	+	t.Fatalf("budget %v, want about %v", got, tc.want)
	96	+	}
	97	+	})
	98	+	}
	99	+	}

A internal/host/util_test.go +11 -0

@@ -0,0 +1,11 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package host
	4	+
	5	+	import (
	6	+	"time"
	7	+
	8	+	"google.golang.org/protobuf/types/known/durationpb"
	9	+	)
	10	+
	11	+	func durationpbOf(d time.Duration) *durationpb.Duration { return durationpb.New(d) }

M internal/sandbox/container/docker_api_test.go +1 -1


@@ -62,7 +62,7 @@ func TestDockerAPIWiring(t *testing.T) {
62	62		run := noMountRunner{}
63	63		empty := filepath.Join(dir, "mountinfo")
64	64		_ = os.WriteFile(empty, nil, 0o644)
65		-	d, err := New(ctx, cfg, Options{Net: plumb, allowRootless: true, ConnectTimeout: 60 * time.Second, Run: run,
	65	+	d, err := New(ctx, cfg, Options{Net: plumb, UnsafeAllowRootlessForTests: true, ConnectTimeout: 60 * time.Second, Run: run,
66	66		Mounter: &disk.Mounter{Run: run, Mountinfo: empty}})
67	67		if err != nil {
68	68		t.Fatal(err)

M internal/sandbox/container/driver.go +6 -5


@@ -53,10 +53,11 @@ type Options struct {
53	53		Net netsetup.Plumbing
54	54		// engine overrides the Docker client (tests).
55	55		engine engine
56		-	// allowRootless lets tests exercise the Engine API against the rootless
57		-	// daemon of a workstation. Never set outside tests: a nested dockerd cannot
58		-	// work in a rootless daemon.
59		-	allowRootless bool
	56	+	// UnsafeAllowRootlessForTests lets tests exercise the Engine API and the executor against
	57	+	// the rootless daemon of a workstation. Never set outside tests: a nested
	58	+	// dockerd cannot work in a rootless daemon, so the jobs would run without
	59	+	// Docker. The factories of package drivers never set it.
	60	+	UnsafeAllowRootlessForTests bool
60	61		// ConnectTimeout bounds container start + init + executor connection
61	62		// (default 90 s).
62	63		ConnectTimeout time.Duration

@@ -133,7 +134,7 @@ func New(ctx context.Context, cfg sandbox.Config, opt Options) (*Driver, error)
133	134		if err != nil {
134	135		return fail(fmt.Errorf("cannot reach the Docker daemon (DOCKER_HOST): %w", err))
135	136		}
136		-	if info.Rootless && !opt.allowRootless {
	137	+	if info.Rootless && !opt.UnsafeAllowRootlessForTests {
137	138		return fail(errRootless)
138	139		}
139	140		if info.OSType != "" && info.OSType != "linux" {

A internal/sandbox/drivers/drivers_test.go +43 -0

@@ -0,0 +1,43 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package drivers
	4	+
	5	+	import (
	6	+	"go/ast"
	7	+	"go/parser"
	8	+	"go/token"
	9	+	"testing"
	10	+	)
	11	+
	12	+	// The container option that lets a driver run against a rootless Docker daemon
	13	+	// exists for tests: with it the nested dockerd of the job cannot start, so jobs
	14	+	// would run without Docker and with weaker isolation than the runner promises.
	15	+	// Production code (this package builds the drivers of `ermite-runner run`) must
	16	+	// never mention it, neither as a field of Options nor in a value.
	17	+	func TestProductionFactoriesNeverSetTheTestOnlyRootlessOption(t *testing.T) {
	18	+	fset := token.NewFileSet()
	19	+	files, err := parser.ParseFile(fset, "drivers.go", nil, 0)
	20	+	if err != nil {
	21	+	t.Fatal(err)
	22	+	}
	23	+	found := false
	24	+	ast.Inspect(files, func(n ast.Node) bool {
	25	+	switch x := n.(type) {
	26	+	case *ast.Ident:
	27	+	if x.Name == "UnsafeAllowRootlessForTests" \|\| x.Name == "AllowRootless" {
	28	+	t.Errorf("%s: %s is a test-only option", fset.Position(x.Pos()), x.Name)
	29	+	}
	30	+	case *ast.CompositeLit:
	31	+	if sel, ok := x.Type.(*ast.SelectorExpr); ok && sel.Sel.Name == "Options" {
	32	+	found = true
	33	+	if len(x.Elts) != 0 {
	34	+	t.Errorf("%s: %s{...} sets options in production code", fset.Position(x.Pos()), sel.Sel.Name)
	35	+	}
	36	+	}
	37	+	}
	38	+	return true
	39	+	})
	40	+	if !found {
	41	+	t.Fatal("drivers.go no longer builds container.Options{}: update this test")
	42	+	}
	43	+	}

M internal/sandbox/integration/vm_test.go +107 -0


@@ -8,7 +8,10 @@ import (
8	8		"strings"
9	9		"syscall"
10	10		"testing"
	11	+	"time"
11	12
	13	+	"bnema.dev/ermite-runner/internal/execstream"
	14	+	"bnema.dev/ermite-runner/internal/execstream/execpb"
12	15		"bnema.dev/ermite-runner/internal/sandbox"
13	16		"bnema.dev/ermite-runner/internal/sandbox/drivers"
14	17		)

@@ -101,3 +104,107 @@ ip -6 addr show scope global \| grep -c inet6 \|\| true
101	104		t.Errorf("the DNS forwarder on the gateway must answer:\n%s", r.out)
102	105		}
103	106		}
	107	+
	108	+	// TestVMRunsAJob drives a job (not the stub script) through the executor in a
	109	+	// micro-VM: the runner user, the work layout, file commands, outputs, a failing
	110	+	// step and cancellation by SIGINT. The VM is rootful-free like the other VM
	111	+	// tests (user + network namespace, jailer off).
	112	+	func TestVMRunsAJob(t *testing.T) {
	113	+	if os.Getenv("ERMITE_RUNNER_IT_VM") != "1" {
	114	+	t.Skip("set ERMITE_RUNNER_IT_VM=1 (needs KVM, firecracker, a kernel and a base image; see README)")
	115	+	}
	116	+	need(t, "mkfs.ext4")
	117	+	needNetAdmin(t)
	118	+	ctx := ctxWithLog(t)
	119	+	cfg := loadConfig(t, sandbox.VM)
	120	+	if why := sandbox.VMUnavailable(cfg, sandbox.Prober{}); len(why) > 0 {
	121	+	t.Fatalf("vm isolation is not available here: %s", strings.Join(why, "; "))
	122	+	}
	123	+	drv, err := sandbox.Select(ctx, cfg, drivers.Factories(), sandbox.Prober{})
	124	+	if err != nil {
	125	+	t.Fatal(err)
	126	+	}
	127	+	t.Cleanup(func() {
	128	+	_ = drv.Close()
	129	+	if d2, err := sandbox.Select(context.Background(), cfg, drivers.Factories(), sandbox.Prober{}); err == nil {
	130	+	_ = d2.Reap(context.Background())
	131	+	_ = d2.Close()
	132	+	}
	133	+	})
	134	+	if err := drv.Reap(ctx); err != nil {
	135	+	t.Fatal(err)
	136	+	}
	137	+
	138	+	run := func(yaml string, cancelAfter time.Duration) (steps []string, logs []string, res execpb.JobResult, exit execpb.Exit) {
	139	+	t.Helper()
	140	+	sb, err := drv.New()
	141	+	if err != nil {
	142	+	t.Fatal(err)
	143	+	}
	144	+	defer func() {
	145	+	if err := sb.Stop(ctx); err != nil {
	146	+	t.Errorf("stop: %v", err)
	147	+	}
	148	+	}()
	149	+	conn, err := sb.Start(ctx, sandbox.JobSpec{JobID: "vm-job"})
	150	+	if err != nil {
	151	+	t.Fatal(err)
	152	+	}
	153	+	var cancel chan *execpb.Cancel
	154	+	if cancelAfter > 0 {
	155	+	cancel = make(chan *execpb.Cancel, 1)
	156	+	time.AfterFunc(cancelAfter, func() { cancel <- &execpb.Cancel{Reason: "test"} })
	157	+	}
	158	+	_, exit, err = execstream.DriveJob(conn, &execpb.Run{Job: &execpb.JobSpec{
	159	+	WorkflowPath: ".github/workflows/ci.yml", WorkflowYaml: yaml, JobKey: "j", Repo: "hello", RunnerName: "it", RunnerVersion: "test",
	160	+	RunnerEnvironment: "self-hosted", Matrix: "{}",
	161	+	Contexts: `{"github":{"repository":"octo/hello","event_name":"push","job":"j","ref":"refs/heads/main","sha":"0123456789abcdef0123456789abcdef01234567"}}`,
	162	+	}}, cancel, execstream.JobHandler{
	163	+	StepStarted: func(s *execpb.StepStarted) { steps = append(steps, s.Name) },
	164	+	Log: func(l *execpb.Log) { logs = append(logs, l.Text) },
	165	+	JobResult: func(r *execpb.JobResult) { res = r },
	166	+	})
	167	+	if err != nil {
	168	+	t.Fatalf("driving the job: %v", err)
	169	+	}
	170	+	return
	171	+	}
	172	+
	173	+	_, logs, res, exit := run(`on: push
	174	+	jobs:
	175	+	j:
	176	+	runs-on: x
	177	+	outputs:
	178	+	who: ${{ steps.a.outputs.who }}
	179	+	steps:
	180	+	- id: a
	181	+	run: \|
	182	+	echo "who=$(id -un):$(id -u)" >> "$GITHUB_OUTPUT"
	183	+	echo "pwd=$PWD home=$HOME"
	184	+	echo "FOO=bar" >> "$GITHUB_ENV"
	185	+	- run: echo "FOO=$FOO CI=$CI"
	186	+	- run: exit 3
	187	+	- if: always()
	188	+	run: echo still-here
	189	+	`, 0)
	190	+	all := strings.Join(logs, "\n")
	191	+	if res == nil \|\| res.Conclusion != execpb.StepConclusion_STEP_CONCLUSION_FAILURE \|\| exit.Code == 0 {
	192	+	t.Fatalf("result %v exit %v\n%s", res, exit, all)
	193	+	}
	194	+	if res.Outputs["who"] != "runner:1001" {
	195	+	t.Errorf("steps ran as %q\n%s", res.Outputs["who"], all)
	196	+	}
	197	+	for _, w := range []string{"pwd=/home/runner/work/hello/hello home=/home/runner", "FOO=bar CI=true", "##[error]Process completed with exit code 3.", "still-here"} {
	198	+	if !strings.Contains(all, w) {
	199	+	t.Errorf("log lacks %q\n%s", w, all)
	200	+	}
	201	+	}
	202	+
	203	+	start := time.Now()
	204	+	_, logs, res, _ = run("on: push\njobs:\n j:\n runs-on: x\n steps:\n - run: \|\n trap 'echo got-int; exit 130' INT\n echo ready\n while :; do sleep 0.2; done\n", 8*time.Second)
	205	+	all = strings.Join(logs, "\n")
	206	+	if res == nil \|\| res.Conclusion != execpb.StepConclusion_STEP_CONCLUSION_CANCELLED \|\| !strings.Contains(all, "got-int") {
	207	+	t.Fatalf("cancel: %v\n%s", res, all)
	208	+	}
	209	+	t.Logf("cancelled job ended %v after the start", time.Since(start))
	210	+	}

M internal/sandbox/sandboxtest/sandboxtest.go +15 -0


@@ -6,14 +6,29 @@ package sandboxtest
6	6		import (
7	7		"context"
8	8		"encoding/binary"
	9	+	"io"
9	10		"net/netip"
10	11		"os"
11	12		"sync"
12	13		"testing"
13	14
	15	+	"bnema.dev/ermite-runner/internal/cmdrun"
14	16		"bnema.dev/ermite-runner/internal/netpolicy"
15	17		)
16	18
	19	+	// NoMountRunner runs real commands except mount and umount, which succeed
	20	+	// without doing anything: loop mounts need root, and the tests that use it
	21	+	// replace them by plain directories (mkfs still runs).
	22	+	type NoMountRunner struct{}
	23	+
	24	+	// Run implements cmdrun.Runner.
	25	+	func (NoMountRunner) Run(ctx context.Context, in io.Reader, name string, args ...string) ([]byte, error) {
	26	+	if name == "mount" \|\| name == "umount" {
	27	+	return nil, nil
	28	+	}
	29	+	return cmdrun.Exec{}.Run(ctx, in, name, args...)
	30	+	}
	31	+
17	32		// WriteStaticELF writes a minimal statically linked ELF64 file (one PT_LOAD, no
18	33		// PT_INTERP): enough for sandbox.CheckStatic, never executed.
19	34		func WriteStaticELF(t testing.TB, path string) {

M main.go +13 -6


@@ -4,10 +4,10 @@
4	4		// its own Firecracker micro-VM, or in its own container when the host has no
5	5		// KVM, and executes workflows compatible with GitHub Actions.
6	6		//
7		-	// Subcommands: version; init and exec run inside a sandbox (guest PID 1, and
8		-	// the executor); sandbox-check runs one stub job through the selected isolation
9		-	// mode and prints its output (phase D of the CI-1 plan: the host loop that
10		-	// fetches jobs from ermite comes later).
	7	+	// Subcommands: version; register stores a runner token obtained from ermite;
	8	+	// run fetches jobs from ermite and runs each in a sandbox; init and exec run
	9	+	// inside a sandbox (guest PID 1, and the executor); sandbox-check runs one stub
	10	+	// script through the selected isolation mode and prints its output.
11	11		package main
12	12
13	13		import (

@@ -49,13 +49,17 @@ func run(args []string) int {
49	49		return execCmd(ctx)
50	50		case "sandbox-check":
51	51		return sandboxCheck(ctx, args[1:])
	52	+	case "register":
	53	+	return registerCmd(ctx, args[1:])
	54	+	case "run":
	55	+	return runCmd(ctx)
52	56		}
53	57		usage()
54	58		return 2
55	59		}
56	60
57	61		func usage() {
58		-	fmt.Fprintln(os.Stderr, "usage: ermite-runner version \| sandbox-check [script]\n (inside a sandbox: ermite-runner init \| exec)")
	62	+	fmt.Fprintln(os.Stderr, "usage: ermite-runner version \| register [--token T] [--replace] \| run \| sandbox-check [script]\n (inside a sandbox: ermite-runner init \| exec)")
59	63		}
60	64
61	65		func envOr(key, def string) string {

@@ -65,7 +69,7 @@ func envOr(key, def string) string {
65	69		return def
66	70		}
67	71
68		-	// execCmd is `ermite-runner exec`, the stub executor.
	72	+	// execCmd is `ermite-runner exec`, the executor of a sandbox.
69	73		func execCmd(ctx context.Context) int {
70	74		log := zerowrap.FromCtx(ctx)
71	75		target, err := guest.ParseTarget(os.Getenv("ERMITE_CONNECT"))

@@ -77,6 +81,9 @@ func execCmd(ctx context.Context) int {
77	81		Target: target,
78	82		Sandbox: envOr("ERMITE_SANDBOX", "container"),
79	83		Version: envOr("ERMITE_VERSION", version),
	84	+	// A test image without the user "runner" (ERMITE_RUNNER_ALLOW_ROOT_STEPS=1);
	85	+	// a runner never sets it: the job refuses to start instead.
	86	+	AllowRootSteps: os.Getenv("ERMITE_RUNNER_ALLOW_ROOT_STEPS") == "1",
80	87		})
81	88		if err != nil {
82	89		log.Error().Err(err).Msg("exec")