docs: systemd unit with hardening and the list of what is absent, register with systemd-run and stdin, logs under journald, stop budget (G5, G6)

commit: 01928f461b00f112930f52c9529eaeaac5304bda
author: brice · 9d ago · 2026-10-12 19:47 UTC
parent: 198b1d9
signature: GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email

2 files · +147 -28

M README.md +77 -16


@@ -51,16 +51,28 @@ Everything a job creates (process, TAP or bridge, nftables rules, files, loop mo
51	51		## Running jobs
52	52
53	53		```
54		-	ermite-runner register --token <token with runner:register> # once
55		-	ermite-runner run # as a service
	54	+	printf %s "$TOKEN" \| ermite-runner register --token-stdin # once
	55	+	ermite-runner run # as a service
56	56		```
57	57
58	58		`register` picks the isolation mode like `run` will, registers the runner with
59	59		ermite (name, labels, version, isolation, architecture) and stores the runner
60		-	token in `$ERMITE_RUNNER_STATE_DIR/runner.json` (mode 0600; a file readable by
61		-	others is refused). The register token can also come from
62		-	`ERMITE_RUNNER_REGISTER_TOKEN`; `--replace` removes a runner of the same name
63		-	first.
	60	+	token in `$ERMITE_RUNNER_STATE_DIR/runner.json` (mode 0600, in a directory made
	61	+	0700; a file readable by others is refused). `--replace` removes a runner of the
	62	+	same name first.
	63	+
	64	+	The token of `register` (scope `runner:register`: it reads the code and the
	65	+	secrets of every repository, see `runner.proto`) is looked up in this order:
	66	+	standard input with `--token-stdin`, then `ERMITE_RUNNER_REGISTER_TOKEN`, then
	67	+	`--token`. Prefer the first two: **the arguments of a process are visible to
	68	+	every user of the host** (`ps`, `/proc/<pid>/cmdline`), the environment and
	69	+	standard input are not.
	70	+
	71	+	The server URL must be `https://`. Plain `http://` is accepted for
	72	+	`localhost`, `127.0.0.0/8` and `::1` only; `ERMITE_RUNNER_INSECURE_HTTP=1` allows
	73	+	any other `http://` URL (tests, a trusted network) and sends every token in
	74	+	clear. The client never follows a redirect, so a token cannot be carried to
	75	+	another host.
64	76
65	77		`run` reaps what a previous run left, then polls `FetchJob` whenever a slot is
66	78		free (`ERMITE_RUNNER_CAPACITY`), with a `poll_id` that is kept when a response

@@ -81,7 +93,10 @@ the host process; the sandbox receives the workflow and its contexts only.
81	93		\| `ERMITE_RUNNER_URL` \| base URL of ermite \|
82	94		\| `ERMITE_RUNNER_NAME` \| runner name (default: the host name) \|
83	95		\| `ERMITE_RUNNER_LABELS` \| extra labels, comma separated (`ubuntu-latest,ubuntu-24.04`) \|
84		-	\| `ERMITE_RUNNER_REGISTER_TOKEN` \| token for `register` \|
	96	+	\| `ERMITE_RUNNER_REGISTER_TOKEN` \| token for `register` (see above) \|
	97	+	\| `ERMITE_RUNNER_INSECURE_HTTP` \| `1` allows a non-loopback `http://` URL (tests only) \|
	98	+	\| `ERMITE_RUNNER_LOG_LEVEL` \| `debug`, `info` (default), `warn`, `error` \|
	99	+	\| `ERMITE_RUNNER_LOG_FORMAT` \| `console` (default, colours) or `json` \|
85	100
86	101		## Installing the service
87	102

@@ -93,7 +108,7 @@ jobs never do: they run as user `runner` in a micro-VM or a container.
93	108		sudo install -m 0755 ermite-runner /usr/local/bin/ermite-runner # a static build: CGO_ENABLED=0
94	109		sudo install -m 0644 contrib/ermite-runner.service /etc/systemd/system/
95	110		sudo install -d -m 0700 /etc/ermite-runner
96		-	sudo tee /etc/ermite-runner/env >/dev/null <<'ENV'
	111	+	sudo install -m 0600 /dev/stdin /etc/ermite-runner/env <<'ENV'
97	112		ERMITE_RUNNER_URL=https://ermite.example.org
98	113		ERMITE_RUNNER_NAME=igor
99	114		ERMITE_RUNNER_LABELS=ubuntu-latest,ubuntu-24.04

@@ -102,19 +117,65 @@ ERMITE_RUNNER_CAPACITY=2
102	117		ERMITE_RUNNER_JOB_CPUS=4
103	118		ERMITE_RUNNER_JOB_MEMORY=8G
104	119		ERMITE_RUNNER_JOB_DISK=64G
	120	+	ERMITE_RUNNER_LOG_FORMAT=json
105	121		ENV
106		-	sudo chmod 0600 /etc/ermite-runner/env
107		-	ERMITE_RUNNER_STATE_DIR=/var/lib/ermite-runner ERMITE_RUNNER_URL=https://ermite.example.org \
108		-	sudo -E ermite-runner register --token <token with runner:register>
	122	+	# Register once, in the same environment as the service (the settings, the state
	123	+	# directory), with the token on standard input so that it is not in any
	124	+	# command line:
	125	+	printf %s "$TOKEN" \| sudo systemd-run --pipe --wait -p EnvironmentFile=/etc/ermite-runner/env \
	126	+	-p StateDirectory=ermite-runner -p StateDirectoryMode=0700 \
	127	+	/usr/local/bin/ermite-runner register --token-stdin
109	128		sudo systemctl enable --now ermite-runner
	129	+	journalctl -u ermite-runner -f
110	130		```
111	131
112		-	`systemctl stop` sends SIGTERM: running jobs are cancelled with
	132	+	`systemd-run` reads the same `EnvironmentFile` as the unit, so `register` and
	133	+	`run` agree on the URL, the name, the labels, the isolation mode and the state
	134	+	directory (`/var/lib/ermite-runner`, the default of `ERMITE_RUNNER_STATE_DIR`).
	135	+
	136	+	Logs. The runner writes to standard error, which journald collects. Set
	137	+	`ERMITE_RUNNER_LOG_FORMAT=json` under journald: one JSON event per line, with
	138	+	`job_id`, `run_id` and `attempt` on every line about a job, `warn` for what is
	139	+	retried and `error` for what is refused or leaked; the colours of the `console`
	140	+	format are noise in a journal. `ERMITE_RUNNER_LOG_LEVEL=debug` adds detail.
	141	+	Tokens are never logged.
	142	+
	143	+	Stopping. `systemctl stop` sends SIGTERM: running jobs are cancelled with
113	144		`runner_shutdown`, reported to ermite, and their sandboxes removed before the
114		-	service exits (the unit allows 120 s). A crash or a power cut leaves sandboxes
115		-	behind; the next start reaps them. Only one runner may live on a host (a
116		-	host-wide lock enforces it), because reaping removes everything labelled as the
117		-	runner's own.
	145	+	service exits. The budget, with the defaults, is 90 s: 30 s for the executor
	146	+	to stop its steps (SIGINT, SIGTERM after 7.5 s, SIGKILL 2.5 s later, then the
	147	+	post-steps), 30 s to destroy the sandbox, 30 s to flush the log and report; jobs
	148	+	do this in parallel, so it does not grow with `ERMITE_RUNNER_CAPACITY`. The unit
	149	+	sets `TimeoutStopSec=120`, 30 s of margin (a test in `internal/host` fails if the
	150	+	two drift apart). A job whose first heartbeat is still pending when the signal
	151	+	arrives is not started: ermite gives it to another runner after 30 s. A crash or
	152	+	a power cut leaves sandboxes behind; the next start reaps them. Only one runner
	153	+	may live on a host (a host-wide lock enforces it), because reaping removes
	154	+	everything labelled as the runner's own.
	155	+
	156	+	Sandboxing of the unit. The runner has to be root (TAP devices, nftables,
	157	+	loop mounts, the jailer, Docker), so the unit only takes away what it does not
	158	+	use: `NoNewPrivileges`, `ProtectHome`, `PrivateTmp`, `ProtectClock`,
	159	+	`ProtectKernelModules`, `ProtectKernelLogs`, `RestrictSUIDSGID`,
	160	+	`LockPersonality`, `RestrictRealtime`, `SystemCallArchitectures=native`,
	161	+	`RestrictAddressFamilies` (UNIX, INET, INET6, NETLINK, VSOCK) and `UMask=0077`.
	162	+	Checked on a workstation (a user manager, no root): the micro-VM job test
	163	+	(`TestVMRunsAJob`: user and network namespaces, Firecracker, TAP, vsock) passes
	164	+	under every option above except `ProtectHome`, which hides the home directory
	165	+	where that test's tree lives; with it the binary itself was run (`version`) and
	166	+	`/home` is denied as intended. Not checked, because they need root on a real
	167	+	host: the jailer, loop mounts, nftables in the host namespace, Docker. Deliberately absent, and why:
	168	+	`Delegate=` (the runner manages no cgroup of its own: the jailer writes the
	169	+	VMs' cgroups as root, Docker has its own), `ProtectSystem=`,
	170	+	`ProtectControlGroups=`, `ProtectKernelTunables=` (the jailer writes cgroups, the
	171	+	runner turns on `net.ipv4.ip_forward`), `PrivateDevices=` (`/dev/kvm`,
	172	+	`/dev/net/tun`, `/dev/loop*`), `RestrictNamespaces=`, `PrivateNetwork=`,
	173	+	`CapabilityBoundingSet=` and `SystemCallFilter=` (network namespaces and
	174	+	devices, mounts, capabilities handed to the jailer and Docker),
	175	+	`MemoryDenyWriteExecute=` (Firecracker maps guest memory writable and
	176	+	executable). The boundary of a job is its micro-VM or container, not this unit;
	177	+	the unit cannot be tested end to end without a rootful host, so the options that
	178	+	would only break it silently are left out.
118	179
119	180		## The image
120	181


@@ -51,16 +51,28 @@ Everything a job creates (process, TAP or bridge, nftables rules, files, loop mo
51		## Running jobs	51		## Running jobs
52			52
53		```	53		```
54	-	ermite-runner register --token <token with runner:register> # once	54	+	printf %s "$TOKEN" \| ermite-runner register --token-stdin # once
55	-	ermite-runner run # as a service	55	+	ermite-runner run # as a service
56		```	56		```
57			57
58		`register` picks the isolation mode like `run` will, registers the runner with	58		`register` picks the isolation mode like `run` will, registers the runner with
59		ermite (name, labels, version, isolation, architecture) and stores the runner	59		ermite (name, labels, version, isolation, architecture) and stores the runner
60	-	token in `$ERMITE_RUNNER_STATE_DIR/runner.json` (mode 0600; a file readable by	60	+	token in `$ERMITE_RUNNER_STATE_DIR/runner.json` (mode 0600, in a directory made
61	-	others is refused). The register token can also come from	61	+	0700; a file readable by others is refused). `--replace` removes a runner of the
62	-	`ERMITE_RUNNER_REGISTER_TOKEN`; `--replace` removes a runner of the same name	62	+	same name first.
63	-	first.	63	+
			64	+	The token of `register` (scope `runner:register`: it reads the code and the
			65	+	secrets of every repository, see `runner.proto`) is looked up in this order:
			66	+	standard input with `--token-stdin`, then `ERMITE_RUNNER_REGISTER_TOKEN`, then
			67	+	`--token`. Prefer the first two: **the arguments of a process are visible to
			68	+	every user of the host** (`ps`, `/proc/<pid>/cmdline`), the environment and
			69	+	standard input are not.
			70	+
			71	+	The server URL must be `https://`. Plain `http://` is accepted for
			72	+	`localhost`, `127.0.0.0/8` and `::1` only; `ERMITE_RUNNER_INSECURE_HTTP=1` allows
			73	+	any other `http://` URL (tests, a trusted network) and sends every token in
			74	+	clear. The client never follows a redirect, so a token cannot be carried to
			75	+	another host.
64			76
65		`run` reaps what a previous run left, then polls `FetchJob` whenever a slot is	77		`run` reaps what a previous run left, then polls `FetchJob` whenever a slot is
66		free (`ERMITE_RUNNER_CAPACITY`), with a `poll_id` that is kept when a response	78		free (`ERMITE_RUNNER_CAPACITY`), with a `poll_id` that is kept when a response

@@ -81,7 +93,10 @@ the host process; the sandbox receives the workflow and its contexts only.
81		\| `ERMITE_RUNNER_URL` \| base URL of ermite \|	93		\| `ERMITE_RUNNER_URL` \| base URL of ermite \|
82		\| `ERMITE_RUNNER_NAME` \| runner name (default: the host name) \|	94		\| `ERMITE_RUNNER_NAME` \| runner name (default: the host name) \|
83		\| `ERMITE_RUNNER_LABELS` \| extra labels, comma separated (`ubuntu-latest,ubuntu-24.04`) \|	95		\| `ERMITE_RUNNER_LABELS` \| extra labels, comma separated (`ubuntu-latest,ubuntu-24.04`) \|
84	-	\| `ERMITE_RUNNER_REGISTER_TOKEN` \| token for `register` \|	96	+	\| `ERMITE_RUNNER_REGISTER_TOKEN` \| token for `register` (see above) \|
			97	+	\| `ERMITE_RUNNER_INSECURE_HTTP` \| `1` allows a non-loopback `http://` URL (tests only) \|
			98	+	\| `ERMITE_RUNNER_LOG_LEVEL` \| `debug`, `info` (default), `warn`, `error` \|
			99	+	\| `ERMITE_RUNNER_LOG_FORMAT` \| `console` (default, colours) or `json` \|
85			100
86		## Installing the service	101		## Installing the service
87			102

@@ -93,7 +108,7 @@ jobs never do: they run as user `runner` in a micro-VM or a container.
93		sudo install -m 0755 ermite-runner /usr/local/bin/ermite-runner # a static build: CGO_ENABLED=0	108		sudo install -m 0755 ermite-runner /usr/local/bin/ermite-runner # a static build: CGO_ENABLED=0
94		sudo install -m 0644 contrib/ermite-runner.service /etc/systemd/system/	109		sudo install -m 0644 contrib/ermite-runner.service /etc/systemd/system/
95		sudo install -d -m 0700 /etc/ermite-runner	110		sudo install -d -m 0700 /etc/ermite-runner
96	-	sudo tee /etc/ermite-runner/env >/dev/null <<'ENV'	111	+	sudo install -m 0600 /dev/stdin /etc/ermite-runner/env <<'ENV'
97		ERMITE_RUNNER_URL=https://ermite.example.org	112		ERMITE_RUNNER_URL=https://ermite.example.org
98		ERMITE_RUNNER_NAME=igor	113		ERMITE_RUNNER_NAME=igor
99		ERMITE_RUNNER_LABELS=ubuntu-latest,ubuntu-24.04	114		ERMITE_RUNNER_LABELS=ubuntu-latest,ubuntu-24.04

@@ -102,19 +117,65 @@ ERMITE_RUNNER_CAPACITY=2
102		ERMITE_RUNNER_JOB_CPUS=4	117		ERMITE_RUNNER_JOB_CPUS=4
103		ERMITE_RUNNER_JOB_MEMORY=8G	118		ERMITE_RUNNER_JOB_MEMORY=8G
104		ERMITE_RUNNER_JOB_DISK=64G	119		ERMITE_RUNNER_JOB_DISK=64G
			120	+	ERMITE_RUNNER_LOG_FORMAT=json
105		ENV	121		ENV
106	-	sudo chmod 0600 /etc/ermite-runner/env	122	+	# Register once, in the same environment as the service (the settings, the state
107	-	ERMITE_RUNNER_STATE_DIR=/var/lib/ermite-runner ERMITE_RUNNER_URL=https://ermite.example.org \	123	+	# directory), with the token on standard input so that it is not in any
108	-	sudo -E ermite-runner register --token <token with runner:register>	124	+	# command line:
			125	+	printf %s "$TOKEN" \| sudo systemd-run --pipe --wait -p EnvironmentFile=/etc/ermite-runner/env \
			126	+	-p StateDirectory=ermite-runner -p StateDirectoryMode=0700 \
			127	+	/usr/local/bin/ermite-runner register --token-stdin
109		sudo systemctl enable --now ermite-runner	128		sudo systemctl enable --now ermite-runner
			129	+	journalctl -u ermite-runner -f
110		```	130		```
111			131
112	-	`systemctl stop` sends SIGTERM: running jobs are cancelled with	132	+	`systemd-run` reads the same `EnvironmentFile` as the unit, so `register` and
			133	+	`run` agree on the URL, the name, the labels, the isolation mode and the state
			134	+	directory (`/var/lib/ermite-runner`, the default of `ERMITE_RUNNER_STATE_DIR`).
			135	+
			136	+	Logs. The runner writes to standard error, which journald collects. Set
			137	+	`ERMITE_RUNNER_LOG_FORMAT=json` under journald: one JSON event per line, with
			138	+	`job_id`, `run_id` and `attempt` on every line about a job, `warn` for what is
			139	+	retried and `error` for what is refused or leaked; the colours of the `console`
			140	+	format are noise in a journal. `ERMITE_RUNNER_LOG_LEVEL=debug` adds detail.
			141	+	Tokens are never logged.
			142	+
			143	+	Stopping. `systemctl stop` sends SIGTERM: running jobs are cancelled with
113		`runner_shutdown`, reported to ermite, and their sandboxes removed before the	144		`runner_shutdown`, reported to ermite, and their sandboxes removed before the
114	-	service exits (the unit allows 120 s). A crash or a power cut leaves sandboxes	145	+	service exits. The budget, with the defaults, is 90 s: 30 s for the executor
115	-	behind; the next start reaps them. Only one runner may live on a host (a	146	+	to stop its steps (SIGINT, SIGTERM after 7.5 s, SIGKILL 2.5 s later, then the
116	-	host-wide lock enforces it), because reaping removes everything labelled as the	147	+	post-steps), 30 s to destroy the sandbox, 30 s to flush the log and report; jobs
117	-	runner's own.	148	+	do this in parallel, so it does not grow with `ERMITE_RUNNER_CAPACITY`. The unit
			149	+	sets `TimeoutStopSec=120`, 30 s of margin (a test in `internal/host` fails if the
			150	+	two drift apart). A job whose first heartbeat is still pending when the signal
			151	+	arrives is not started: ermite gives it to another runner after 30 s. A crash or
			152	+	a power cut leaves sandboxes behind; the next start reaps them. Only one runner
			153	+	may live on a host (a host-wide lock enforces it), because reaping removes
			154	+	everything labelled as the runner's own.
			155	+
			156	+	Sandboxing of the unit. The runner has to be root (TAP devices, nftables,
			157	+	loop mounts, the jailer, Docker), so the unit only takes away what it does not
			158	+	use: `NoNewPrivileges`, `ProtectHome`, `PrivateTmp`, `ProtectClock`,
			159	+	`ProtectKernelModules`, `ProtectKernelLogs`, `RestrictSUIDSGID`,
			160	+	`LockPersonality`, `RestrictRealtime`, `SystemCallArchitectures=native`,
			161	+	`RestrictAddressFamilies` (UNIX, INET, INET6, NETLINK, VSOCK) and `UMask=0077`.
			162	+	Checked on a workstation (a user manager, no root): the micro-VM job test
			163	+	(`TestVMRunsAJob`: user and network namespaces, Firecracker, TAP, vsock) passes
			164	+	under every option above except `ProtectHome`, which hides the home directory
			165	+	where that test's tree lives; with it the binary itself was run (`version`) and
			166	+	`/home` is denied as intended. Not checked, because they need root on a real
			167	+	host: the jailer, loop mounts, nftables in the host namespace, Docker. Deliberately absent, and why:
			168	+	`Delegate=` (the runner manages no cgroup of its own: the jailer writes the
			169	+	VMs' cgroups as root, Docker has its own), `ProtectSystem=`,
			170	+	`ProtectControlGroups=`, `ProtectKernelTunables=` (the jailer writes cgroups, the
			171	+	runner turns on `net.ipv4.ip_forward`), `PrivateDevices=` (`/dev/kvm`,
			172	+	`/dev/net/tun`, `/dev/loop*`), `RestrictNamespaces=`, `PrivateNetwork=`,
			173	+	`CapabilityBoundingSet=` and `SystemCallFilter=` (network namespaces and
			174	+	devices, mounts, capabilities handed to the jailer and Docker),
			175	+	`MemoryDenyWriteExecute=` (Firecracker maps guest memory writable and
			176	+	executable). The boundary of a job is its micro-VM or container, not this unit;
			177	+	the unit cannot be tested end to end without a rootful host, so the options that
			178	+	would only break it silently are left out.
118			179
119		## The image	180		## The image
120			181

M contrib/ermite-runner.service +70 -12

@@ -1,15 +1,16 @@
1	1		# SPDX-License-Identifier: MIT
2	2		#
3	3		# systemd unit of the ermite runner. Install the static binary as
4		-	# /usr/local/bin/ermite-runner, write the settings to
5		-	# /etc/ermite-runner/env (mode 0600) and register the runner once:
	4	+	# /usr/local/bin/ermite-runner, write the settings to /etc/ermite-runner/env
	5	+	# (mode 0600) and register the runner once; the README has the commands.
6	6		#
7		-	# sudo systemd-run --pipe --wait -p EnvironmentFile=/etc/ermite-runner/env \
8		-	# /usr/local/bin/ermite-runner register --token <runner:register token>
	7	+	# The runner runs as root: it creates TAP devices and nftables rules (vm), loop
	8	+	# mounts and containers (container), and starts the jailer. Jobs never run as
	9	+	# root on the host: they run in a micro-VM or a container as user "runner".
9	10		#
10		-	# The runner runs as root: it creates TAP devices and firewall rules (vm), loop
11		-	# mounts (container), and starts the jailer or containers. Jobs never run as
12		-	# root on the host; they run in a micro-VM or a container as user "runner".
	11	+	# The sandboxing below is what can be added without breaking those jobs.
	12	+	# What is deliberately NOT set, and why, is listed at the end of the file and
	13	+	# in the README.
13	14		[Unit]
14	15		Description=ermite CI runner
15	16		Documentation=https://git.bnema.dev/bnema/ermite-runner

@@ -22,19 +23,76 @@ EnvironmentFile=/etc/ermite-runner/env
22	23		ExecStart=/usr/local/bin/ermite-runner run
23	24		Restart=on-failure
24	25		RestartSec=5
25		-	# SIGTERM cancels the running jobs with runner_shutdown, reports them and
26		-	# removes their sandboxes; give that time before systemd sends SIGKILL.
	26	+
	27	+	# SIGTERM cancels the running jobs with runner_shutdown, reports them and removes
	28	+	# their sandboxes. With the defaults that takes at most 90 s (30 s for the
	29	+	# executor to stop its steps, 30 s to destroy the sandbox, 30 s to report; the
	30	+	# jobs do this in parallel). A test checks that this value leaves 15 s of margin
	31	+	# over host.Timing.ShutdownBudget.
27	32		KillMode=mixed
28	33		KillSignal=SIGTERM
29	34		TimeoutStopSec=120
30		-	# The state directory holds the runner token (0600) and the VM images.
	35	+
	36	+	# The state directory holds the runner token (0600), the VM images and the
	37	+	# per-job files; RuntimeDirectory is for the lock and sockets.
31	38		StateDirectory=ermite-runner
32	39		StateDirectoryMode=0700
33	40		RuntimeDirectory=ermite-runner
34		-	# Jobs are many processes: do not let systemd's default task limit stop them.
	41	+	UMask=0077
	42	+
	43	+	# Jobs are many processes.
35	44		TasksMax=infinity
36	45		LimitNOFILE=1048576
37		-	Delegate=yes
	46	+
	47	+	# ---- sandboxing that keeps loop mounts, TAP, nftables, KVM and the jailer ----
	48	+	NoNewPrivileges=yes
	49	+	# Nobody else's files, and no interactive logins' data.
	50	+	ProtectHome=yes
	51	+	# A private /tmp: the runner and the jailer do not share files with the host
	52	+	# through it (the sandbox directories live in the state directory).
	53	+	PrivateTmp=yes
	54	+	# The runner does not set the clock, load modules or change kernel logging.
	55	+	ProtectClock=yes
	56	+	ProtectKernelModules=yes
	57	+	ProtectKernelLogs=yes
	58	+	# No setuid/setgid files created by the runner or its children.
	59	+	RestrictSUIDSGID=yes
	60	+	# Kernel tunables stay read-only for the unit... except where the runner needs
	61	+	# them: see the end of the file (net.ipv4.ip_forward, cgroups).
	62	+	LockPersonality=yes
	63	+	# The runner does not need real-time scheduling.
	64	+	RestrictRealtime=yes
	65	+	# Syscall architectures: native only.
	66	+	SystemCallArchitectures=native
	67	+	# Address families the runner and its children use: Unix sockets (Docker, the
	68	+	# executor stream), IPv4/IPv6 (the server, the DNS forwarder), netlink (ip, nft,
	69	+	# TAP devices), packet (none needed: left out), vsock (the executor stream in a
	70	+	# micro-VM).
	71	+	RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 AF_NETLINK AF_VSOCK
38	72
39	73		[Install]
40	74		WantedBy=multi-user.target
	75	+
	76	+	# ---- deliberately absent -----------------------------------------------------
	77	+	#
	78	+	# Delegate=yes The runner does not manage cgroups of its own. The jailer
	79	+	# creates the cgroups of the VMs below /sys/fs/cgroup itself
	80	+	# (it is root and writes there directly), and Docker has its
	81	+	# own. Delegation would hand a subtree to this unit that
	82	+	# nothing uses.
	83	+	# ProtectSystem= The runner writes to the state directory, /run, /sys/fs/cgroup,
	84	+	# /dev (mounts, TAP, loop devices), and Docker's data: a
	85	+	# read-only /usr is possible but would be the only gain, and
	86	+	# it breaks `ip`/`nft` helpers found by PATH on some distros.
	87	+	# PrivateDevices= The runner needs /dev/kvm, /dev/net/tun, /dev/loop*.
	88	+	# PrivateNetwork=, RestrictNamespaces=, ProtectKernelTunables=,
	89	+	# CapabilityBoundingSet=, MemoryDenyWriteExecute=, SystemCallFilter=
	90	+	# The runner creates network namespaces and devices (TAP,
	91	+	# veth, bridges) and mounts file systems, enables
	92	+	# net.ipv4.ip_forward, hands capabilities to the jailer and
	93	+	# to Docker, and starts Firecracker (which maps guest memory
	94	+	# writable and executable). Each of these would break a path
	95	+	# that cannot be tested without a rootful host; the
	96	+	# boundary of a job is the micro-VM or the container, not
	97	+	# this unit.
	98	+	# ProtectControlGroups= The jailer writes the cgroups of the VMs.

@@ -1,15 +1,16 @@
1		# SPDX-License-Identifier: MIT	1		# SPDX-License-Identifier: MIT
2		#	2		#
3		# systemd unit of the ermite runner. Install the static binary as	3		# systemd unit of the ermite runner. Install the static binary as
4	-	# /usr/local/bin/ermite-runner, write the settings to	4	+	# /usr/local/bin/ermite-runner, write the settings to /etc/ermite-runner/env
5	-	# /etc/ermite-runner/env (mode 0600) and register the runner once:	5	+	# (mode 0600) and register the runner once; the README has the commands.
6		#	6		#
7	-	# sudo systemd-run --pipe --wait -p EnvironmentFile=/etc/ermite-runner/env \	7	+	# The runner runs as root: it creates TAP devices and nftables rules (vm), loop
8	-	# /usr/local/bin/ermite-runner register --token <runner:register token>	8	+	# mounts and containers (container), and starts the jailer. Jobs never run as
			9	+	# root on the host: they run in a micro-VM or a container as user "runner".
9		#	10		#
10	-	# The runner runs as root: it creates TAP devices and firewall rules (vm), loop	11	+	# The sandboxing below is what can be added without breaking those jobs.
11	-	# mounts (container), and starts the jailer or containers. Jobs never run as	12	+	# What is deliberately NOT set, and why, is listed at the end of the file and
12	-	# root on the host; they run in a micro-VM or a container as user "runner".	13	+	# in the README.
13		[Unit]	14		[Unit]
14		Description=ermite CI runner	15		Description=ermite CI runner
15		Documentation=https://git.bnema.dev/bnema/ermite-runner	16		Documentation=https://git.bnema.dev/bnema/ermite-runner

@@ -22,19 +23,76 @@ EnvironmentFile=/etc/ermite-runner/env
22		ExecStart=/usr/local/bin/ermite-runner run	23		ExecStart=/usr/local/bin/ermite-runner run
23		Restart=on-failure	24		Restart=on-failure
24		RestartSec=5	25		RestartSec=5
25	-	# SIGTERM cancels the running jobs with runner_shutdown, reports them and	26	+
26	-	# removes their sandboxes; give that time before systemd sends SIGKILL.	27	+	# SIGTERM cancels the running jobs with runner_shutdown, reports them and removes
			28	+	# their sandboxes. With the defaults that takes at most 90 s (30 s for the
			29	+	# executor to stop its steps, 30 s to destroy the sandbox, 30 s to report; the
			30	+	# jobs do this in parallel). A test checks that this value leaves 15 s of margin
			31	+	# over host.Timing.ShutdownBudget.
27		KillMode=mixed	32		KillMode=mixed
28		KillSignal=SIGTERM	33		KillSignal=SIGTERM
29		TimeoutStopSec=120	34		TimeoutStopSec=120
30	-	# The state directory holds the runner token (0600) and the VM images.	35	+
			36	+	# The state directory holds the runner token (0600), the VM images and the
			37	+	# per-job files; RuntimeDirectory is for the lock and sockets.
31		StateDirectory=ermite-runner	38		StateDirectory=ermite-runner
32		StateDirectoryMode=0700	39		StateDirectoryMode=0700
33		RuntimeDirectory=ermite-runner	40		RuntimeDirectory=ermite-runner
34	-	# Jobs are many processes: do not let systemd's default task limit stop them.	41	+	UMask=0077
			42	+
			43	+	# Jobs are many processes.
35		TasksMax=infinity	44		TasksMax=infinity
36		LimitNOFILE=1048576	45		LimitNOFILE=1048576
37	-	Delegate=yes	46	+
			47	+	# ---- sandboxing that keeps loop mounts, TAP, nftables, KVM and the jailer ----
			48	+	NoNewPrivileges=yes
			49	+	# Nobody else's files, and no interactive logins' data.
			50	+	ProtectHome=yes
			51	+	# A private /tmp: the runner and the jailer do not share files with the host
			52	+	# through it (the sandbox directories live in the state directory).
			53	+	PrivateTmp=yes
			54	+	# The runner does not set the clock, load modules or change kernel logging.
			55	+	ProtectClock=yes
			56	+	ProtectKernelModules=yes
			57	+	ProtectKernelLogs=yes
			58	+	# No setuid/setgid files created by the runner or its children.
			59	+	RestrictSUIDSGID=yes
			60	+	# Kernel tunables stay read-only for the unit... except where the runner needs
			61	+	# them: see the end of the file (net.ipv4.ip_forward, cgroups).
			62	+	LockPersonality=yes
			63	+	# The runner does not need real-time scheduling.
			64	+	RestrictRealtime=yes
			65	+	# Syscall architectures: native only.
			66	+	SystemCallArchitectures=native
			67	+	# Address families the runner and its children use: Unix sockets (Docker, the
			68	+	# executor stream), IPv4/IPv6 (the server, the DNS forwarder), netlink (ip, nft,
			69	+	# TAP devices), packet (none needed: left out), vsock (the executor stream in a
			70	+	# micro-VM).
			71	+	RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 AF_NETLINK AF_VSOCK
38			72
39		[Install]	73		[Install]
40		WantedBy=multi-user.target	74		WantedBy=multi-user.target
			75	+
			76	+	# ---- deliberately absent -----------------------------------------------------
			77	+	#
			78	+	# Delegate=yes The runner does not manage cgroups of its own. The jailer
			79	+	# creates the cgroups of the VMs below /sys/fs/cgroup itself
			80	+	# (it is root and writes there directly), and Docker has its
			81	+	# own. Delegation would hand a subtree to this unit that
			82	+	# nothing uses.
			83	+	# ProtectSystem= The runner writes to the state directory, /run, /sys/fs/cgroup,
			84	+	# /dev (mounts, TAP, loop devices), and Docker's data: a
			85	+	# read-only /usr is possible but would be the only gain, and
			86	+	# it breaks `ip`/`nft` helpers found by PATH on some distros.
			87	+	# PrivateDevices= The runner needs /dev/kvm, /dev/net/tun, /dev/loop*.
			88	+	# PrivateNetwork=, RestrictNamespaces=, ProtectKernelTunables=,
			89	+	# CapabilityBoundingSet=, MemoryDenyWriteExecute=, SystemCallFilter=
			90	+	# The runner creates network namespaces and devices (TAP,
			91	+	# veth, bridges) and mounts file systems, enables
			92	+	# net.ipv4.ip_forward, hands capabilities to the jailer and
			93	+	# to Docker, and starts Firecracker (which maps guest memory
			94	+	# writable and executable). Each of these would break a path
			95	+	# that cannot be tested without a rootful host; the
			96	+	# boundary of a job is the micro-VM or the container, not
			97	+	# this unit.
			98	+	# ProtectControlGroups= The jailer writes the cgroups of the VMs.