detached 08776ef

feat(container): run as an unprivileged container with a guest forge relay and first-run registration

commit
08776ef1212e2cb6d9cc2ee6ccbe8303fe79ab78
author
brice · · 2026-10-18 20:41 UTC
parent
f9de495
signature
GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email
15 files · +672 -18
Split
UnifiedSplit
A contrib/container/Containerfile +50 -0
@@ -0,0 +1,50 @@
1+# SPDX-License-Identifier: MIT
2+#
3+# ermite-runner as an unprivileged container (vm isolation, no jailer), for a
4+# rootless Podman host. See docs/container.md.
5+#
6+# Build from the repository root, after scripts/build-kernel.sh and
7+# scripts/build-image.sh have written tmp/image/{vmlinux,base.ext4}:
8+#
9+# docker build -f contrib/container/Containerfile --build-arg VERSION=<tag> -t ermite-runner:<tag> .
10+#
11+# The guest kernel and the base image are part of the image: the runner then
12+# needs nothing on the host but /dev/kvm, /dev/net/tun and NET_ADMIN.
13+
14+ARG GO_VERSION=1.27
15+
16+FROM docker.io/library/golang:${GO_VERSION} AS build
17+WORKDIR /src
18+COPY go.mod go.sum ./
19+RUN go mod download
20+COPY . .
21+ARG VERSION=dev
22+RUN CGO_ENABLED=0 GOWORK=off go build -trimpath -ldflags "-s -w -X main.version=${VERSION}" -o /out/ermite-runner .
23+
24+FROM docker.io/library/alpine:3.22 AS firecracker
25+ARG FIRECRACKER_VERSION=v1.17.0
26+ARG FIRECRACKER_SHA256=06094a1108ae9e82aa4c23a775aa92758f53f1175d422270d9d6162cb9ade558
27+RUN apk add --no-cache curl \
28+ && curl -fsSL -o /tmp/fc.tgz "https://github.com/firecracker-microvm/firecracker/releases/download/${FIRECRACKER_VERSION}/firecracker-${FIRECRACKER_VERSION}-x86_64.tgz" \
29+ && echo "${FIRECRACKER_SHA256} /tmp/fc.tgz" | sha256sum -c - \
30+ && tar -xzf /tmp/fc.tgz -C /tmp \
31+ && install -m 0755 "/tmp/release-${FIRECRACKER_VERSION}-x86_64/firecracker-${FIRECRACKER_VERSION}-x86_64" /out-firecracker
32+
33+FROM docker.io/library/alpine:3.22
34+# ip (TAP devices), nft (job network policy), mkfs.ext4 (scratch and exec
35+# disks; e2fsprogs >= 1.47), ca-certificates (Node.js externals downloads).
36+RUN apk add --no-cache iproute2 nftables e2fsprogs ca-certificates tini
37+COPY --from=firecracker /out-firecracker /usr/local/bin/firecracker
38+COPY --from=build /out/ermite-runner /usr/local/bin/ermite-runner
39+COPY tmp/image/vmlinux tmp/image/base.ext4 /opt/ermite-runner/image/
40+
41+ENV ERMITE_RUNNER_ISOLATION=vm \
42+ ERMITE_RUNNER_JAILER_MODE=off \
43+ ERMITE_RUNNER_IMAGE=/opt/ermite-runner/image \
44+ ERMITE_RUNNER_STATE_DIR=/var/lib/ermite-runner \
45+ ERMITE_RUNNER_LOCK_FILE=/run/ermite-runner.lock \
46+ ERMITE_RUNNER_LOG_FORMAT=json
47+
48+VOLUME /var/lib/ermite-runner
49+ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/ermite-runner"]
50+CMD ["run"]
@@ -0,0 +1,50 @@
1+# SPDX-License-Identifier: MIT
2+#
3+# ermite-runner as an unprivileged container (vm isolation, no jailer), for a
4+# rootless Podman host. See docs/container.md.
5+#
6+# Build from the repository root, after scripts/build-kernel.sh and
7+# scripts/build-image.sh have written tmp/image/{vmlinux,base.ext4}:
8+#
9+# docker build -f contrib/container/Containerfile --build-arg VERSION=<tag> -t ermite-runner:<tag> .
10+#
11+# The guest kernel and the base image are part of the image: the runner then
12+# needs nothing on the host but /dev/kvm, /dev/net/tun and NET_ADMIN.
13+
14+ARG GO_VERSION=1.27
15+
16+FROM docker.io/library/golang:${GO_VERSION} AS build
17+WORKDIR /src
18+COPY go.mod go.sum ./
19+RUN go mod download
20+COPY . .
21+ARG VERSION=dev
22+RUN CGO_ENABLED=0 GOWORK=off go build -trimpath -ldflags "-s -w -X main.version=${VERSION}" -o /out/ermite-runner .
23+
24+FROM docker.io/library/alpine:3.22 AS firecracker
25+ARG FIRECRACKER_VERSION=v1.17.0
26+ARG FIRECRACKER_SHA256=06094a1108ae9e82aa4c23a775aa92758f53f1175d422270d9d6162cb9ade558
27+RUN apk add --no-cache curl \
28+ && curl -fsSL -o /tmp/fc.tgz "https://github.com/firecracker-microvm/firecracker/releases/download/${FIRECRACKER_VERSION}/firecracker-${FIRECRACKER_VERSION}-x86_64.tgz" \
29+ && echo "${FIRECRACKER_SHA256} /tmp/fc.tgz" | sha256sum -c - \
30+ && tar -xzf /tmp/fc.tgz -C /tmp \
31+ && install -m 0755 "/tmp/release-${FIRECRACKER_VERSION}-x86_64/firecracker-${FIRECRACKER_VERSION}-x86_64" /out-firecracker
32+
33+FROM docker.io/library/alpine:3.22
34+# ip (TAP devices), nft (job network policy), mkfs.ext4 (scratch and exec
35+# disks; e2fsprogs >= 1.47), ca-certificates (Node.js externals downloads).
36+RUN apk add --no-cache iproute2 nftables e2fsprogs ca-certificates tini
37+COPY --from=firecracker /out-firecracker /usr/local/bin/firecracker
38+COPY --from=build /out/ermite-runner /usr/local/bin/ermite-runner
39+COPY tmp/image/vmlinux tmp/image/base.ext4 /opt/ermite-runner/image/
40+
41+ENV ERMITE_RUNNER_ISOLATION=vm \
42+ ERMITE_RUNNER_JAILER_MODE=off \
43+ ERMITE_RUNNER_IMAGE=/opt/ermite-runner/image \
44+ ERMITE_RUNNER_STATE_DIR=/var/lib/ermite-runner \
45+ ERMITE_RUNNER_LOCK_FILE=/run/ermite-runner.lock \
46+ ERMITE_RUNNER_LOG_FORMAT=json
47+
48+VOLUME /var/lib/ermite-runner
49+ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/ermite-runner"]
50+CMD ["run"]
M docs/configuration.md +4 -2
@@ -12,7 +12,8 @@ Sizes accept `M`, `G` and `T`.
1212 | `ERMITE_RUNNER_NAME` | host name | Runner name: 1 to 64 letters, digits, `.`, `_` and `-` |
1313 | `ERMITE_RUNNER_LABELS` | | Extra labels, comma separated, same characters as the name |
1414 | `ERMITE_RUNNER_ENVIRONMENT` | `github-hosted` | `github-hosted` or `self-hosted`. Sets the `runner.environment` value that jobs see. `self-hosted` also makes "Set up job" print the runner and machine name |
15-| `ERMITE_RUNNER_REGISTER_TOKEN` | | Token for `register` |
15+| `ERMITE_RUNNER_REGISTER_TOKEN` | | Token for `register`, and for the first start of `run` when no registration exists |
16+| `ERMITE_RUNNER_REGISTER_TOKEN_FILE` | | File holding the register token. Wins over `ERMITE_RUNNER_REGISTER_TOKEN` for the first start of `run`. See [container](container.md#first-start) |
1617 | `ERMITE_RUNNER_INSECURE_HTTP` | `0` | `1` allows a non-loopback `http://` URL. Every token then travels in clear. For tests and trusted networks |
1718
1819 ### Server URL
@@ -42,7 +43,7 @@ The token has the scope `runner:register`. It can read the code and the secrets
4243 | `ERMITE_RUNNER_ISOLATION` | `auto` | `auto`, `vm` or `container` |
4344 | `ERMITE_RUNNER_STATE_DIR` | `/var/lib/ermite-runner` | Absolute path. See the ownership rules in [sandboxes](sandboxes.md#state-directory) |
4445 | `ERMITE_RUNNER_CAPACITY` | `1` | Jobs in parallel. One sandbox, one uid, one `/30` network and one disk per slot. The runner polls the server only when a slot is free |
45-| `ERMITE_RUNNER_IMAGE` | | `vm`: image directory with the kernel and `base.ext4` (default `<state dir>/image`). `container`: OCI reference (default `ermite-runner-ubuntu-24.04:latest`) |
46+| `ERMITE_RUNNER_IMAGE` | | `vm`: image directory with the kernel and `base.ext4` (default `<state dir>/image`). `container`: OCI reference (default `ermite-runner-ubuntu-26.04:latest`) |
4647 | `ERMITE_RUNNER_DENY_PRIVILEGED` | `0` | `1` refuses `container` mode without sysbox |
4748 | `ERMITE_RUNNER_NETWORK` | `10.200.0.0/16` | Pool of `/30` subnets, one per slot |
4849 | `ERMITE_RUNNER_UID_BASE` | `900000` | First uid of the jailer pool. A VM runs as this value plus its slot |
@@ -79,6 +80,7 @@ The token has the scope `runner:register`. It can read the code and the secrets
7980 | `ERMITE_RUNNER_ARTIFACTS` | `1` | `0` turns artifacts v4 off. Artifacts need the cache service |
8081 | `ERMITE_RUNNER_NODE` | `1` | `0` turns the Node.js externals off. See [externals](externals.md) |
8182 | `ERMITE_RUNNER_GUEST_FORGE` | `0` | `1` lets sandboxes reach the server host and port. Staging only. See [sandboxes](sandboxes.md#guest-forge-access) |
83+| `ERMITE_RUNNER_GUEST_FORGE_TARGET` | | `host:port` the runner relays guest forge traffic to. Empty: no relay, a service must listen on the gateway port |
8284
8385 ## Logging
8486
@@ -12,7 +12,8 @@ Sizes accept `M`, `G` and `T`.
12 | `ERMITE_RUNNER_NAME` | host name | Runner name: 1 to 64 letters, digits, `.`, `_` and `-` |12 | `ERMITE_RUNNER_NAME` | host name | Runner name: 1 to 64 letters, digits, `.`, `_` and `-` |
13 | `ERMITE_RUNNER_LABELS` | | Extra labels, comma separated, same characters as the name |13 | `ERMITE_RUNNER_LABELS` | | Extra labels, comma separated, same characters as the name |
14 | `ERMITE_RUNNER_ENVIRONMENT` | `github-hosted` | `github-hosted` or `self-hosted`. Sets the `runner.environment` value that jobs see. `self-hosted` also makes "Set up job" print the runner and machine name |14 | `ERMITE_RUNNER_ENVIRONMENT` | `github-hosted` | `github-hosted` or `self-hosted`. Sets the `runner.environment` value that jobs see. `self-hosted` also makes "Set up job" print the runner and machine name |
15-| `ERMITE_RUNNER_REGISTER_TOKEN` | | Token for `register` |15+| `ERMITE_RUNNER_REGISTER_TOKEN` | | Token for `register`, and for the first start of `run` when no registration exists |
16+| `ERMITE_RUNNER_REGISTER_TOKEN_FILE` | | File holding the register token. Wins over `ERMITE_RUNNER_REGISTER_TOKEN` for the first start of `run`. See [container](container.md#first-start) |
16 | `ERMITE_RUNNER_INSECURE_HTTP` | `0` | `1` allows a non-loopback `http://` URL. Every token then travels in clear. For tests and trusted networks |17 | `ERMITE_RUNNER_INSECURE_HTTP` | `0` | `1` allows a non-loopback `http://` URL. Every token then travels in clear. For tests and trusted networks |
17 18
18 ### Server URL19 ### Server URL
@@ -42,7 +43,7 @@ The token has the scope `runner:register`. It can read the code and the secrets
42 | `ERMITE_RUNNER_ISOLATION` | `auto` | `auto`, `vm` or `container` |43 | `ERMITE_RUNNER_ISOLATION` | `auto` | `auto`, `vm` or `container` |
43 | `ERMITE_RUNNER_STATE_DIR` | `/var/lib/ermite-runner` | Absolute path. See the ownership rules in [sandboxes](sandboxes.md#state-directory) |44 | `ERMITE_RUNNER_STATE_DIR` | `/var/lib/ermite-runner` | Absolute path. See the ownership rules in [sandboxes](sandboxes.md#state-directory) |
44 | `ERMITE_RUNNER_CAPACITY` | `1` | Jobs in parallel. One sandbox, one uid, one `/30` network and one disk per slot. The runner polls the server only when a slot is free |45 | `ERMITE_RUNNER_CAPACITY` | `1` | Jobs in parallel. One sandbox, one uid, one `/30` network and one disk per slot. The runner polls the server only when a slot is free |
45-| `ERMITE_RUNNER_IMAGE` | | `vm`: image directory with the kernel and `base.ext4` (default `<state dir>/image`). `container`: OCI reference (default `ermite-runner-ubuntu-24.04:latest`) |46+| `ERMITE_RUNNER_IMAGE` | | `vm`: image directory with the kernel and `base.ext4` (default `<state dir>/image`). `container`: OCI reference (default `ermite-runner-ubuntu-26.04:latest`) |
46 | `ERMITE_RUNNER_DENY_PRIVILEGED` | `0` | `1` refuses `container` mode without sysbox |47 | `ERMITE_RUNNER_DENY_PRIVILEGED` | `0` | `1` refuses `container` mode without sysbox |
47 | `ERMITE_RUNNER_NETWORK` | `10.200.0.0/16` | Pool of `/30` subnets, one per slot |48 | `ERMITE_RUNNER_NETWORK` | `10.200.0.0/16` | Pool of `/30` subnets, one per slot |
48 | `ERMITE_RUNNER_UID_BASE` | `900000` | First uid of the jailer pool. A VM runs as this value plus its slot |49 | `ERMITE_RUNNER_UID_BASE` | `900000` | First uid of the jailer pool. A VM runs as this value plus its slot |
@@ -79,6 +80,7 @@ The token has the scope `runner:register`. It can read the code and the secrets
79 | `ERMITE_RUNNER_ARTIFACTS` | `1` | `0` turns artifacts v4 off. Artifacts need the cache service |80 | `ERMITE_RUNNER_ARTIFACTS` | `1` | `0` turns artifacts v4 off. Artifacts need the cache service |
80 | `ERMITE_RUNNER_NODE` | `1` | `0` turns the Node.js externals off. See [externals](externals.md) |81 | `ERMITE_RUNNER_NODE` | `1` | `0` turns the Node.js externals off. See [externals](externals.md) |
81 | `ERMITE_RUNNER_GUEST_FORGE` | `0` | `1` lets sandboxes reach the server host and port. Staging only. See [sandboxes](sandboxes.md#guest-forge-access) |82 | `ERMITE_RUNNER_GUEST_FORGE` | `0` | `1` lets sandboxes reach the server host and port. Staging only. See [sandboxes](sandboxes.md#guest-forge-access) |
83+| `ERMITE_RUNNER_GUEST_FORGE_TARGET` | | `host:port` the runner relays guest forge traffic to. Empty: no relay, a service must listen on the gateway port |
82 84
83 ## Logging85 ## Logging
84 86
A docs/container.md +52 -0
@@ -0,0 +1,52 @@
1+# Unprivileged container
2+
3+`contrib/container/Containerfile` packages the runner for a rootless container host (Podman or Docker). Jobs run in Firecracker microVMs (`vm` isolation) inside the container. Nothing runs as root on the host.
4+
5+## What the image holds
6+
7+- `ermite-runner`, Firecracker and `tini`.
8+- The guest kernel (`vmlinux`) and the base disk (`base.ext4`), from [images](images.md).
9+- Defaults: `ERMITE_RUNNER_ISOLATION=vm`, `ERMITE_RUNNER_JAILER_MODE=off`, `ERMITE_RUNNER_STATE_DIR=/var/lib/ermite-runner`, JSON logs.
10+
11+## Build
12+
13+```sh
14+scripts/build-kernel.sh tmp/image
15+scripts/build-image.sh tmp/image
16+docker build -f contrib/container/Containerfile --build-arg VERSION=<tag> -t ermite-runner:<tag> .
17+```
18+
19+## What the container needs
20+
21+| Grant | Why |
22+| --- | --- |
23+| `/dev/kvm` | Firecracker |
24+| `/dev/net/tun` | One TAP device per job |
25+| `CAP_NET_ADMIN` | TAP devices and nftables rules, in the container's own network namespace only |
26+| A volume on `/var/lib/ermite-runner` | Registration, caches and job disks |
27+
28+No `--privileged`, no host network, no other capability. With rootless Podman, a CDI spec can expose the two device nodes.
29+
30+The jailer is off: it needs root to chroot and switch uid. The container and the user namespace are the outer boundary instead, and each job still runs in its own microVM.
31+
32+## First start
33+
34+With no registration in the state directory, the runner registers itself when a register token is set:
35+
36+- `ERMITE_RUNNER_REGISTER_TOKEN_FILE`: path of a file holding the token (a container secret), or
37+- `ERMITE_RUNNER_REGISTER_TOKEN`.
38+
39+The token is read only on that first start. Revoke it afterwards. Labels are fixed at registration: to change them, remove the runner on the server, delete `runner.json` from the state directory and restart with a fresh token.
40+
41+## Reaching the server
42+
43+When the server runs on the same container network, set `ERMITE_RUNNER_URL` to its internal address and turn on [guest forge access](sandboxes.md#guest-forge-access) with a relay:
44+
45+```sh
46+ERMITE_RUNNER_URL=http://forge:8080
47+ERMITE_RUNNER_INSECURE_HTTP=1
48+ERMITE_RUNNER_GUEST_FORGE=1
49+ERMITE_RUNNER_GUEST_FORGE_TARGET=forge:8080
50+```
51+
52+Jobs then clone from `http://forge:8080`, which the runner relays to the server.
@@ -0,0 +1,52 @@
1+# Unprivileged container
2+
3+`contrib/container/Containerfile` packages the runner for a rootless container host (Podman or Docker). Jobs run in Firecracker microVMs (`vm` isolation) inside the container. Nothing runs as root on the host.
4+
5+## What the image holds
6+
7+- `ermite-runner`, Firecracker and `tini`.
8+- The guest kernel (`vmlinux`) and the base disk (`base.ext4`), from [images](images.md).
9+- Defaults: `ERMITE_RUNNER_ISOLATION=vm`, `ERMITE_RUNNER_JAILER_MODE=off`, `ERMITE_RUNNER_STATE_DIR=/var/lib/ermite-runner`, JSON logs.
10+
11+## Build
12+
13+```sh
14+scripts/build-kernel.sh tmp/image
15+scripts/build-image.sh tmp/image
16+docker build -f contrib/container/Containerfile --build-arg VERSION=<tag> -t ermite-runner:<tag> .
17+```
18+
19+## What the container needs
20+
21+| Grant | Why |
22+| --- | --- |
23+| `/dev/kvm` | Firecracker |
24+| `/dev/net/tun` | One TAP device per job |
25+| `CAP_NET_ADMIN` | TAP devices and nftables rules, in the container's own network namespace only |
26+| A volume on `/var/lib/ermite-runner` | Registration, caches and job disks |
27+
28+No `--privileged`, no host network, no other capability. With rootless Podman, a CDI spec can expose the two device nodes.
29+
30+The jailer is off: it needs root to chroot and switch uid. The container and the user namespace are the outer boundary instead, and each job still runs in its own microVM.
31+
32+## First start
33+
34+With no registration in the state directory, the runner registers itself when a register token is set:
35+
36+- `ERMITE_RUNNER_REGISTER_TOKEN_FILE`: path of a file holding the token (a container secret), or
37+- `ERMITE_RUNNER_REGISTER_TOKEN`.
38+
39+The token is read only on that first start. Revoke it afterwards. Labels are fixed at registration: to change them, remove the runner on the server, delete `runner.json` from the state directory and restart with a fresh token.
40+
41+## Reaching the server
42+
43+When the server runs on the same container network, set `ERMITE_RUNNER_URL` to its internal address and turn on [guest forge access](sandboxes.md#guest-forge-access) with a relay:
44+
45+```sh
46+ERMITE_RUNNER_URL=http://forge:8080
47+ERMITE_RUNNER_INSECURE_HTTP=1
48+ERMITE_RUNNER_GUEST_FORGE=1
49+ERMITE_RUNNER_GUEST_FORGE_TARGET=forge:8080
50+```
51+
52+Jobs then clone from `http://forge:8080`, which the runner relays to the server.
M docs/sandboxes.md +2 -1
@@ -108,7 +108,8 @@ For every sandbox:
108108
109109 Security notes:
110110
111-- The runner does not proxy. A service must listen on that port on an address the gateway reaches (`*`, not `127.0.0.1`).
111+- Without `ERMITE_RUNNER_GUEST_FORGE_TARGET`, the runner does not proxy. A service must listen on that port on an address the gateway reaches (`*`, not `127.0.0.1`).
112+- With `ERMITE_RUNNER_GUEST_FORGE_TARGET=host:port`, the runner listens on each sandbox gateway on that port and relays TCP to the target. Use it when the server is only reachable from the runner, for example on a container network. See [container](container.md#reaching-the-server).
112113 - A shared reverse proxy routes by `Host` header. A job can reach every app that proxy serves on that port.
113114 - Traffic comes from a private address. Anything behind the proxy that trusts private sources trusts job code too.
114115 - Jobs see the server over plain HTTP in this setup.
@@ -108,7 +108,8 @@ For every sandbox:
108 108
109 Security notes:109 Security notes:
110 110
111-- The runner does not proxy. A service must listen on that port on an address the gateway reaches (`*`, not `127.0.0.1`).111+- Without `ERMITE_RUNNER_GUEST_FORGE_TARGET`, the runner does not proxy. A service must listen on that port on an address the gateway reaches (`*`, not `127.0.0.1`).
112+- With `ERMITE_RUNNER_GUEST_FORGE_TARGET=host:port`, the runner listens on each sandbox gateway on that port and relays TCP to the target. Use it when the server is only reachable from the runner, for example on a container network. See [container](container.md#reaching-the-server).
112 - A shared reverse proxy routes by `Host` header. A job can reach every app that proxy serves on that port.113 - A shared reverse proxy routes by `Host` header. A job can reach every app that proxy serves on that port.
113 - Traffic comes from a private address. Anything behind the proxy that trusts private sources trusts job code too.114 - Traffic comes from a private address. Anything behind the proxy that trusts private sources trusts job code too.
114 - Jobs see the server over plain HTTP in this setup.115 - Jobs see the server over plain HTTP in this setup.
M host_cmd.go +50 -1
@@ -193,6 +193,50 @@ func registerCmd(ctx context.Context, args []string) int {
193193 return 0
194194 }
195195
196+// registerOnFirstRun registers the runner when `run` finds no registration and
197+// a register token is given in ERMITE_RUNNER_REGISTER_TOKEN or in the file
198+// named by ERMITE_RUNNER_REGISTER_TOKEN_FILE. Without a token it returns
199+// host.ErrNotRegistered, as before.
200+func registerOnFirstRun(ctx context.Context, scfg sandbox.Config, hcfg host.Config) (host.State, error) {
201+ tok, err := firstRunToken(os.Getenv)
202+ if err != nil || tok == "" {
203+ return host.State{}, errors.Join(host.ErrNotRegistered, err)
204+ }
205+ if hcfg.URL == "" {
206+ return host.State{}, fmt.Errorf("%s is not set", host.EnvURL)
207+ }
208+ drv, err := buildDriver(ctx, scfg)
209+ if err != nil {
210+ return host.State{}, err
211+ }
212+ iso := drv.Isolation()
213+ _ = drv.Close()
214+ st, err := host.Register(ctx, hcfg, hcfg.Client(hcfg.URL, tok), iso, false)
215+ if err != nil {
216+ return host.State{}, fmt.Errorf("first-run registration: %w", err)
217+ }
218+ log := zerowrap.FromCtx(ctx)
219+ log.Info().Str("runner", st.Name).Str("isolation", string(iso)).Msg("registered on first run; the register token is no longer needed")
220+ return st, nil
221+}
222+
223+// envRegisterTokenFile names a file holding the register token (a container
224+// secret delivered as a file).
225+const envRegisterTokenFile = "ERMITE_RUNNER_REGISTER_TOKEN_FILE"
226+
227+// firstRunToken reads the register token of a first run: the file of
228+// envRegisterTokenFile when set, else host.EnvRegisterToken. "" means none.
229+func firstRunToken(getenv func(string) string) (string, error) {
230+ if p := strings.TrimSpace(getenv(envRegisterTokenFile)); p != "" {
231+ b, err := os.ReadFile(p)
232+ if err != nil {
233+ return "", fmt.Errorf("%s: %w", envRegisterTokenFile, err)
234+ }
235+ return strings.TrimSpace(string(b)), nil
236+ }
237+ return strings.TrimSpace(getenv(host.EnvRegisterToken)), nil
238+}
239+
196240 // runCmd is `ermite-runner run`: poll ermite for jobs and run each one in its
197241 // own sandbox until SIGINT or SIGTERM, then cancel the running jobs with
198242 // runner_shutdown and report them.
@@ -204,6 +248,11 @@ func runCmd(ctx context.Context) int {
204248 return 2
205249 }
206250 st, err := host.LoadState(hcfg.StateDir)
251+ if errors.Is(err, host.ErrNotRegistered) {
252+ // A container service has no separate install step: it registers on its
253+ // first start when it is given a register token.
254+ st, err = registerOnFirstRun(ctx, scfg, hcfg)
255+ }
207256 if err != nil {
208257 fmt.Fprintln(os.Stderr, "ermite-runner:", err)
209258 return 2
@@ -219,7 +268,7 @@ func runCmd(ctx context.Context) int {
219268 if scfg.GuestForgeEnabled {
220269 // The sandboxes reach the forge the way the runner does: by the host name
221270 // and port of the URL the runner itself uses.
222- gf, err := sandbox.ResolveGuestForge(ctx, url, hcfg.InsecureHTTP, net.DefaultResolver.LookupHost)
271+ gf, err := sandbox.ResolveGuestForge(ctx, url, os.Getenv(sandbox.EnvGuestForgeTarget), hcfg.InsecureHTTP, net.DefaultResolver.LookupHost)
223272 if err != nil {
224273 fmt.Fprintln(os.Stderr, "ermite-runner:", err)
225274 return 2
@@ -193,6 +193,50 @@ func registerCmd(ctx context.Context, args []string) int {
193 return 0193 return 0
194 }194 }
195 195
196+// registerOnFirstRun registers the runner when `run` finds no registration and
197+// a register token is given in ERMITE_RUNNER_REGISTER_TOKEN or in the file
198+// named by ERMITE_RUNNER_REGISTER_TOKEN_FILE. Without a token it returns
199+// host.ErrNotRegistered, as before.
200+func registerOnFirstRun(ctx context.Context, scfg sandbox.Config, hcfg host.Config) (host.State, error) {
201+ tok, err := firstRunToken(os.Getenv)
202+ if err != nil || tok == "" {
203+ return host.State{}, errors.Join(host.ErrNotRegistered, err)
204+ }
205+ if hcfg.URL == "" {
206+ return host.State{}, fmt.Errorf("%s is not set", host.EnvURL)
207+ }
208+ drv, err := buildDriver(ctx, scfg)
209+ if err != nil {
210+ return host.State{}, err
211+ }
212+ iso := drv.Isolation()
213+ _ = drv.Close()
214+ st, err := host.Register(ctx, hcfg, hcfg.Client(hcfg.URL, tok), iso, false)
215+ if err != nil {
216+ return host.State{}, fmt.Errorf("first-run registration: %w", err)
217+ }
218+ log := zerowrap.FromCtx(ctx)
219+ log.Info().Str("runner", st.Name).Str("isolation", string(iso)).Msg("registered on first run; the register token is no longer needed")
220+ return st, nil
221+}
222+
223+// envRegisterTokenFile names a file holding the register token (a container
224+// secret delivered as a file).
225+const envRegisterTokenFile = "ERMITE_RUNNER_REGISTER_TOKEN_FILE"
226+
227+// firstRunToken reads the register token of a first run: the file of
228+// envRegisterTokenFile when set, else host.EnvRegisterToken. "" means none.
229+func firstRunToken(getenv func(string) string) (string, error) {
230+ if p := strings.TrimSpace(getenv(envRegisterTokenFile)); p != "" {
231+ b, err := os.ReadFile(p)
232+ if err != nil {
233+ return "", fmt.Errorf("%s: %w", envRegisterTokenFile, err)
234+ }
235+ return strings.TrimSpace(string(b)), nil
236+ }
237+ return strings.TrimSpace(getenv(host.EnvRegisterToken)), nil
238+}
239+
196 // runCmd is `ermite-runner run`: poll ermite for jobs and run each one in its240 // runCmd is `ermite-runner run`: poll ermite for jobs and run each one in its
197 // own sandbox until SIGINT or SIGTERM, then cancel the running jobs with241 // own sandbox until SIGINT or SIGTERM, then cancel the running jobs with
198 // runner_shutdown and report them.242 // runner_shutdown and report them.
@@ -204,6 +248,11 @@ func runCmd(ctx context.Context) int {
204 return 2248 return 2
205 }249 }
206 st, err := host.LoadState(hcfg.StateDir)250 st, err := host.LoadState(hcfg.StateDir)
251+ if errors.Is(err, host.ErrNotRegistered) {
252+ // A container service has no separate install step: it registers on its
253+ // first start when it is given a register token.
254+ st, err = registerOnFirstRun(ctx, scfg, hcfg)
255+ }
207 if err != nil {256 if err != nil {
208 fmt.Fprintln(os.Stderr, "ermite-runner:", err)257 fmt.Fprintln(os.Stderr, "ermite-runner:", err)
209 return 2258 return 2
@@ -219,7 +268,7 @@ func runCmd(ctx context.Context) int {
219 if scfg.GuestForgeEnabled {268 if scfg.GuestForgeEnabled {
220 // The sandboxes reach the forge the way the runner does: by the host name269 // The sandboxes reach the forge the way the runner does: by the host name
221 // and port of the URL the runner itself uses.270 // and port of the URL the runner itself uses.
222- gf, err := sandbox.ResolveGuestForge(ctx, url, hcfg.InsecureHTTP, net.DefaultResolver.LookupHost)271+ gf, err := sandbox.ResolveGuestForge(ctx, url, os.Getenv(sandbox.EnvGuestForgeTarget), hcfg.InsecureHTTP, net.DefaultResolver.LookupHost)
223 if err != nil {272 if err != nil {
224 fmt.Fprintln(os.Stderr, "ermite-runner:", err)273 fmt.Fprintln(os.Stderr, "ermite-runner:", err)
225 return 2274 return 2
A host_cmd_test.go +34 -0
@@ -0,0 +1,34 @@
1+// SPDX-License-Identifier: MIT
2+
3+package main
4+
5+import (
6+ "os"
7+ "path/filepath"
8+ "testing"
9+
10+ "bnema.dev/ermite-runner/internal/host"
11+)
12+
13+func TestFirstRunToken(t *testing.T) {
14+ env := func(m map[string]string) func(string) string {
15+ return func(k string) string { return m[k] }
16+ }
17+ if tok, err := firstRunToken(env(nil)); err != nil || tok != "" {
18+ t.Fatalf("no token: %q %v", tok, err)
19+ }
20+ if tok, err := firstRunToken(env(map[string]string{host.EnvRegisterToken: " t1 \n"})); err != nil || tok != "t1" {
21+ t.Fatalf("env: %q %v", tok, err)
22+ }
23+ p := filepath.Join(t.TempDir(), "token")
24+ if err := os.WriteFile(p, []byte("t2\n"), 0o600); err != nil {
25+ t.Fatal(err)
26+ }
27+ // The file wins over the environment.
28+ if tok, err := firstRunToken(env(map[string]string{envRegisterTokenFile: p, host.EnvRegisterToken: "t1"})); err != nil || tok != "t2" {
29+ t.Fatalf("file: %q %v", tok, err)
30+ }
31+ if _, err := firstRunToken(env(map[string]string{envRegisterTokenFile: filepath.Join(t.TempDir(), "missing")})); err == nil {
32+ t.Fatal("a missing token file was accepted")
33+ }
34+}
@@ -0,0 +1,34 @@
1+// SPDX-License-Identifier: MIT
2+
3+package main
4+
5+import (
6+ "os"
7+ "path/filepath"
8+ "testing"
9+
10+ "bnema.dev/ermite-runner/internal/host"
11+)
12+
13+func TestFirstRunToken(t *testing.T) {
14+ env := func(m map[string]string) func(string) string {
15+ return func(k string) string { return m[k] }
16+ }
17+ if tok, err := firstRunToken(env(nil)); err != nil || tok != "" {
18+ t.Fatalf("no token: %q %v", tok, err)
19+ }
20+ if tok, err := firstRunToken(env(map[string]string{host.EnvRegisterToken: " t1 \n"})); err != nil || tok != "t1" {
21+ t.Fatalf("env: %q %v", tok, err)
22+ }
23+ p := filepath.Join(t.TempDir(), "token")
24+ if err := os.WriteFile(p, []byte("t2\n"), 0o600); err != nil {
25+ t.Fatal(err)
26+ }
27+ // The file wins over the environment.
28+ if tok, err := firstRunToken(env(map[string]string{envRegisterTokenFile: p, host.EnvRegisterToken: "t1"})); err != nil || tok != "t2" {
29+ t.Fatalf("file: %q %v", tok, err)
30+ }
31+ if _, err := firstRunToken(env(map[string]string{envRegisterTokenFile: filepath.Join(t.TempDir(), "missing")})); err == nil {
32+ t.Fatal("a missing token file was accepted")
33+ }
34+}
M internal/sandbox/container/container_test.go +3 -3
@@ -1014,10 +1014,10 @@ func TestSocketPathTooLongIsExplained(t *testing.T) {
10141014 // GO-002: only a podman daemon gets a DNS listener that may precede the gateway
10151015 // address; Docker keeps a normal bind.
10161016 func TestOnlyPodmanGetsAFreeBindDNSListener(t *testing.T) {
1017- if newNet(nil, 0, false, "").DNS.FreeBind {
1017+ if newNet(nil, 0, false, "", "").DNS.FreeBind {
10181018 t.Fatal("Docker must keep a normal bind")
10191019 }
1020- if !newNet(nil, 0, true, "").DNS.FreeBind {
1020+ if !newNet(nil, 0, true, "", "").DNS.FreeBind {
10211021 t.Fatal("podman needs the listener before the bridge address")
10221022 }
10231023 }
@@ -1080,7 +1080,7 @@ func TestNoGuestForgeByDefault(t *testing.T) {
10801080 if strings.Contains(buf.String(), "guest forge") {
10811081 t.Fatal("a guest forge warning without the setting")
10821082 }
1083- if n := newNet(nil, 0, false, "ermite.example.org").DNS.Override; n != "ermite.example.org" {
1083+ if n := newNet(nil, 0, false, "ermite.example.org", "").DNS.Override; n != "ermite.example.org" {
10841084 t.Fatalf("override %q", n)
10851085 }
10861086 }
@@ -1014,10 +1014,10 @@ func TestSocketPathTooLongIsExplained(t *testing.T) {
1014 // GO-002: only a podman daemon gets a DNS listener that may precede the gateway1014 // GO-002: only a podman daemon gets a DNS listener that may precede the gateway
1015 // address; Docker keeps a normal bind.1015 // address; Docker keeps a normal bind.
1016 func TestOnlyPodmanGetsAFreeBindDNSListener(t *testing.T) {1016 func TestOnlyPodmanGetsAFreeBindDNSListener(t *testing.T) {
1017- if newNet(nil, 0, false, "").DNS.FreeBind {1017+ if newNet(nil, 0, false, "", "").DNS.FreeBind {
1018 t.Fatal("Docker must keep a normal bind")1018 t.Fatal("Docker must keep a normal bind")
1019 }1019 }
1020- if !newNet(nil, 0, true, "").DNS.FreeBind {1020+ if !newNet(nil, 0, true, "", "").DNS.FreeBind {
1021 t.Fatal("podman needs the listener before the bridge address")1021 t.Fatal("podman needs the listener before the bridge address")
1022 }1022 }
1023 }1023 }
@@ -1080,7 +1080,7 @@ func TestNoGuestForgeByDefault(t *testing.T) {
1080 if strings.Contains(buf.String(), "guest forge") {1080 if strings.Contains(buf.String(), "guest forge") {
1081 t.Fatal("a guest forge warning without the setting")1081 t.Fatal("a guest forge warning without the setting")
1082 }1082 }
1083- if n := newNet(nil, 0, false, "ermite.example.org").DNS.Override; n != "ermite.example.org" {1083+ if n := newNet(nil, 0, false, "ermite.example.org", "").DNS.Override; n != "ermite.example.org" {
1084 t.Fatalf("override %q", n)1084 t.Fatalf("override %q", n)
1085 }1085 }
1086 }1086 }
M internal/sandbox/container/driver.go +6 -2
@@ -38,6 +38,7 @@ import (
3838 "bnema.dev/ermite-runner/internal/sandbox"
3939 "bnema.dev/ermite-runner/internal/sandbox/disk"
4040 "bnema.dev/ermite-runner/internal/sandbox/netsetup"
41+ "bnema.dev/ermite-runner/internal/tcprelay"
4142 )
4243
4344 // SysboxRuntime is the name of the sysbox runtime in `docker info`.
@@ -163,7 +164,7 @@ func New(ctx context.Context, cfg sandbox.Config, opt Options) (*Driver, error)
163164 }
164165 d.net = opt.Net
165166 if d.net == nil {
166- n := newNet(opt.Run, cfg.DNSPort, info.Podman, cfg.GuestForge.DNSName())
167+ n := newNet(opt.Run, cfg.DNSPort, info.Podman, cfg.GuestForge.DNSName(), cfg.GuestForge.RelayTarget())
167168 d.net = n
168169 }
169170 if err := d.net.EnableForwarding(); err != nil {
@@ -178,10 +179,13 @@ func New(ctx context.Context, cfg sandbox.Config, opt Options) (*Driver, error)
178179 // newNet builds the host network plumbing. Only podman creates the bridge
179180 // address late (see daemonInfo.Podman), so only there the DNS listener may come
180181 // before the address; on Docker a missing gateway address stays a startup error.
181-func newNet(run cmdrun.Runner, dnsPort int, podman bool, forgeHost string) *netsetup.Net {
182+func newNet(run cmdrun.Runner, dnsPort int, podman bool, forgeHost, relay string) *netsetup.Net {
182183 n := netsetup.New(run, dnsPort, "")
183184 n.DNS.FreeBind = podman
184185 n.DNS.Override = forgeHost
186+ if relay != "" {
187+ n.Relay = tcprelay.New(relay)
188+ }
185189 return n
186190 }
187191
@@ -38,6 +38,7 @@ import (
38 "bnema.dev/ermite-runner/internal/sandbox"38 "bnema.dev/ermite-runner/internal/sandbox"
39 "bnema.dev/ermite-runner/internal/sandbox/disk"39 "bnema.dev/ermite-runner/internal/sandbox/disk"
40 "bnema.dev/ermite-runner/internal/sandbox/netsetup"40 "bnema.dev/ermite-runner/internal/sandbox/netsetup"
41+ "bnema.dev/ermite-runner/internal/tcprelay"
41 )42 )
42 43
43 // SysboxRuntime is the name of the sysbox runtime in `docker info`.44 // SysboxRuntime is the name of the sysbox runtime in `docker info`.
@@ -163,7 +164,7 @@ func New(ctx context.Context, cfg sandbox.Config, opt Options) (*Driver, error)
163 }164 }
164 d.net = opt.Net165 d.net = opt.Net
165 if d.net == nil {166 if d.net == nil {
166- n := newNet(opt.Run, cfg.DNSPort, info.Podman, cfg.GuestForge.DNSName())167+ n := newNet(opt.Run, cfg.DNSPort, info.Podman, cfg.GuestForge.DNSName(), cfg.GuestForge.RelayTarget())
167 d.net = n168 d.net = n
168 }169 }
169 if err := d.net.EnableForwarding(); err != nil {170 if err := d.net.EnableForwarding(); err != nil {
@@ -178,10 +179,13 @@ func New(ctx context.Context, cfg sandbox.Config, opt Options) (*Driver, error)
178 // newNet builds the host network plumbing. Only podman creates the bridge179 // newNet builds the host network plumbing. Only podman creates the bridge
179 // address late (see daemonInfo.Podman), so only there the DNS listener may come180 // address late (see daemonInfo.Podman), so only there the DNS listener may come
180 // before the address; on Docker a missing gateway address stays a startup error.181 // before the address; on Docker a missing gateway address stays a startup error.
181-func newNet(run cmdrun.Runner, dnsPort int, podman bool, forgeHost string) *netsetup.Net {182+func newNet(run cmdrun.Runner, dnsPort int, podman bool, forgeHost, relay string) *netsetup.Net {
182 n := netsetup.New(run, dnsPort, "")183 n := netsetup.New(run, dnsPort, "")
183 n.DNS.FreeBind = podman184 n.DNS.FreeBind = podman
184 n.DNS.Override = forgeHost185 n.DNS.Override = forgeHost
186+ if relay != "" {
187+ n.Relay = tcprelay.New(relay)
188+ }
185 return n189 return n
186 }190 }
187 191
M internal/sandbox/guestforge.go +59 -4
@@ -6,6 +6,7 @@ import (
66 "context"
77 "errors"
88 "fmt"
9+ "net"
910 "net/netip"
1011 "net/url"
1112 "strconv"
@@ -19,6 +20,13 @@ import (
1920 // internet side of the NAT (see the README). Off by default.
2021 const EnvGuestForge = "ERMITE_RUNNER_GUEST_FORGE"
2122
23+// EnvGuestForgeTarget is ERMITE_RUNNER_GUEST_FORGE_TARGET=host:port: with the
24+// guest forge access on, the runner relays the forge port of each sandbox's
25+// gateway to that address instead of letting the host's own listener answer.
26+// For a runner in a container, where the forge is another container reached
27+// by name (ermite:7341) and nothing listens on the forge port locally.
28+const EnvGuestForgeTarget = "ERMITE_RUNNER_GUEST_FORGE_TARGET"
29+
2230 // GuestForge is the one forge endpoint that sandboxes may reach on the host.
2331 // It is derived from the server URL by ParseGuestForge.
2432 type GuestForge struct {
@@ -30,6 +38,39 @@ type GuestForge struct {
3038 // Origin is scheme://host[:port] of the URL: the jobs' server URLs are
3139 // rewritten to it.
3240 Origin string
41+ // Target, when not empty, is the host:port the runner relays the sandboxes'
42+ // forge connections to (EnvGuestForgeTarget).
43+ Target string
44+}
45+
46+// RelayTarget is Target, "" when the access is off or not relayed. Safe on nil.
47+func (g *GuestForge) RelayTarget() string {
48+ if g == nil {
49+ return ""
50+ }
51+ return g.Target
52+}
53+
54+// ParseGuestForgeTarget checks a relay target: host:port with a host name or
55+// an IP address and a port in 1-65535. "" is accepted (no relay).
56+func ParseGuestForgeTarget(raw string) (string, error) {
57+ raw = strings.TrimSpace(raw)
58+ if raw == "" {
59+ return "", nil
60+ }
61+ h, p, err := net.SplitHostPort(raw)
62+ if err != nil || h == "" {
63+ return "", fmt.Errorf("%s: %q is not host:port", EnvGuestForgeTarget, raw)
64+ }
65+ if n, err := strconv.Atoi(p); err != nil || n < 1 || n > 65535 {
66+ return "", fmt.Errorf("%s: port %q is not in 1-65535", EnvGuestForgeTarget, p)
67+ }
68+ if _, err := netip.ParseAddr(h); err != nil {
69+ if err := CheckForgeHostname(strings.ToLower(h)); err != nil {
70+ return "", fmt.Errorf("%s: %w", EnvGuestForgeTarget, err)
71+ }
72+ }
73+ return raw, nil
3374 }
3475
3576 // Warning is the line logged at start and for each job.
@@ -138,19 +179,33 @@ func CheckForgeHostname(h string) error {
138179 // resolves on the host (lookup is net.DefaultResolver.LookupHost in
139180 // production): the runner reaches the forge by that name, so a name that
140181 // resolves to nothing is a configuration error to report at start.
141-func ResolveGuestForge(ctx context.Context, raw string, insecureHTTP bool, lookup func(ctx context.Context, host string) ([]string, error)) (*GuestForge, error) {
182+//
183+// target is the relay target (EnvGuestForgeTarget, "" for none). With a
184+// target, the sandboxes still see the forge under the host name of raw, and it
185+// is the target's host that must resolve: the runner connects there.
186+func ResolveGuestForge(ctx context.Context, raw, target string, insecureHTTP bool, lookup func(ctx context.Context, host string) ([]string, error)) (*GuestForge, error) {
142187 g, err := ParseGuestForge(raw, insecureHTTP)
143188 if err != nil {
144189 return nil, err
145190 }
191+ if g.Target, err = ParseGuestForgeTarget(target); err != nil {
192+ return nil, err
193+ }
194+ host := g.Host
195+ if g.Target != "" {
196+ host, _, _ = net.SplitHostPort(g.Target)
197+ if _, err := netip.ParseAddr(host); err == nil {
198+ return g, nil
199+ }
200+ }
146201 ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
147202 defer cancel()
148- addrs, err := lookup(ctx, g.Host)
203+ addrs, err := lookup(ctx, host)
149204 if err != nil {
150- return nil, fmt.Errorf("%s: %s does not resolve on this host: %w", EnvGuestForge, g.Host, err)
205+ return nil, fmt.Errorf("%s: %s does not resolve on this host: %w", EnvGuestForge, host, err)
151206 }
152207 if len(addrs) == 0 {
153- return nil, fmt.Errorf("%s: %s resolves to no address on this host", EnvGuestForge, g.Host)
208+ return nil, fmt.Errorf("%s: %s resolves to no address on this host", EnvGuestForge, host)
154209 }
155210 return g, nil
156211 }
@@ -6,6 +6,7 @@ import (
6 "context"6 "context"
7 "errors"7 "errors"
8 "fmt"8 "fmt"
9+ "net"
9 "net/netip"10 "net/netip"
10 "net/url"11 "net/url"
11 "strconv"12 "strconv"
@@ -19,6 +20,13 @@ import (
19 // internet side of the NAT (see the README). Off by default.20 // internet side of the NAT (see the README). Off by default.
20 const EnvGuestForge = "ERMITE_RUNNER_GUEST_FORGE"21 const EnvGuestForge = "ERMITE_RUNNER_GUEST_FORGE"
21 22
23+// EnvGuestForgeTarget is ERMITE_RUNNER_GUEST_FORGE_TARGET=host:port: with the
24+// guest forge access on, the runner relays the forge port of each sandbox's
25+// gateway to that address instead of letting the host's own listener answer.
26+// For a runner in a container, where the forge is another container reached
27+// by name (ermite:7341) and nothing listens on the forge port locally.
28+const EnvGuestForgeTarget = "ERMITE_RUNNER_GUEST_FORGE_TARGET"
29+
22 // GuestForge is the one forge endpoint that sandboxes may reach on the host.30 // GuestForge is the one forge endpoint that sandboxes may reach on the host.
23 // It is derived from the server URL by ParseGuestForge.31 // It is derived from the server URL by ParseGuestForge.
24 type GuestForge struct {32 type GuestForge struct {
@@ -30,6 +38,39 @@ type GuestForge struct {
30 // Origin is scheme://host[:port] of the URL: the jobs' server URLs are38 // Origin is scheme://host[:port] of the URL: the jobs' server URLs are
31 // rewritten to it.39 // rewritten to it.
32 Origin string40 Origin string
41+ // Target, when not empty, is the host:port the runner relays the sandboxes'
42+ // forge connections to (EnvGuestForgeTarget).
43+ Target string
44+}
45+
46+// RelayTarget is Target, "" when the access is off or not relayed. Safe on nil.
47+func (g *GuestForge) RelayTarget() string {
48+ if g == nil {
49+ return ""
50+ }
51+ return g.Target
52+}
53+
54+// ParseGuestForgeTarget checks a relay target: host:port with a host name or
55+// an IP address and a port in 1-65535. "" is accepted (no relay).
56+func ParseGuestForgeTarget(raw string) (string, error) {
57+ raw = strings.TrimSpace(raw)
58+ if raw == "" {
59+ return "", nil
60+ }
61+ h, p, err := net.SplitHostPort(raw)
62+ if err != nil || h == "" {
63+ return "", fmt.Errorf("%s: %q is not host:port", EnvGuestForgeTarget, raw)
64+ }
65+ if n, err := strconv.Atoi(p); err != nil || n < 1 || n > 65535 {
66+ return "", fmt.Errorf("%s: port %q is not in 1-65535", EnvGuestForgeTarget, p)
67+ }
68+ if _, err := netip.ParseAddr(h); err != nil {
69+ if err := CheckForgeHostname(strings.ToLower(h)); err != nil {
70+ return "", fmt.Errorf("%s: %w", EnvGuestForgeTarget, err)
71+ }
72+ }
73+ return raw, nil
33 }74 }
34 75
35 // Warning is the line logged at start and for each job.76 // Warning is the line logged at start and for each job.
@@ -138,19 +179,33 @@ func CheckForgeHostname(h string) error {
138 // resolves on the host (lookup is net.DefaultResolver.LookupHost in179 // resolves on the host (lookup is net.DefaultResolver.LookupHost in
139 // production): the runner reaches the forge by that name, so a name that180 // production): the runner reaches the forge by that name, so a name that
140 // resolves to nothing is a configuration error to report at start.181 // resolves to nothing is a configuration error to report at start.
141-func ResolveGuestForge(ctx context.Context, raw string, insecureHTTP bool, lookup func(ctx context.Context, host string) ([]string, error)) (*GuestForge, error) {182+//
183+// target is the relay target (EnvGuestForgeTarget, "" for none). With a
184+// target, the sandboxes still see the forge under the host name of raw, and it
185+// is the target's host that must resolve: the runner connects there.
186+func ResolveGuestForge(ctx context.Context, raw, target string, insecureHTTP bool, lookup func(ctx context.Context, host string) ([]string, error)) (*GuestForge, error) {
142 g, err := ParseGuestForge(raw, insecureHTTP)187 g, err := ParseGuestForge(raw, insecureHTTP)
143 if err != nil {188 if err != nil {
144 return nil, err189 return nil, err
145 }190 }
191+ if g.Target, err = ParseGuestForgeTarget(target); err != nil {
192+ return nil, err
193+ }
194+ host := g.Host
195+ if g.Target != "" {
196+ host, _, _ = net.SplitHostPort(g.Target)
197+ if _, err := netip.ParseAddr(host); err == nil {
198+ return g, nil
199+ }
200+ }
146 ctx, cancel := context.WithTimeout(ctx, 10*time.Second)201 ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
147 defer cancel()202 defer cancel()
148- addrs, err := lookup(ctx, g.Host)203+ addrs, err := lookup(ctx, host)
149 if err != nil {204 if err != nil {
150- return nil, fmt.Errorf("%s: %s does not resolve on this host: %w", EnvGuestForge, g.Host, err)205+ return nil, fmt.Errorf("%s: %s does not resolve on this host: %w", EnvGuestForge, host, err)
151 }206 }
152 if len(addrs) == 0 {207 if len(addrs) == 0 {
153- return nil, fmt.Errorf("%s: %s resolves to no address on this host", EnvGuestForge, g.Host)208+ return nil, fmt.Errorf("%s: %s resolves to no address on this host", EnvGuestForge, host)
154 }209 }
155 return g, nil210 return g, nil
156 }211 }
M internal/sandbox/guestforge_test.go +26 -4
@@ -74,22 +74,22 @@ func TestResolveGuestForge(t *testing.T) {
7474 ctx := context.Background()
7575 var asked string
7676 lookup := func(_ context.Context, h string) ([]string, error) { asked = h; return []string{"127.0.0.1"}, nil }
77- g, err := ResolveGuestForge(ctx, "http://ermite.bnema.dev:8088", true, lookup)
77+ g, err := ResolveGuestForge(ctx, "http://ermite.bnema.dev:8088", "", true, lookup)
7878 if err != nil || g.Port != 8088 || asked != "ermite.bnema.dev" {
7979 t.Fatalf("%+v %v %q", g, err, asked)
8080 }
81- _, err = ResolveGuestForge(ctx, "https://nx.example.org", false, func(context.Context, string) ([]string, error) {
81+ _, err = ResolveGuestForge(ctx, "https://nx.example.org", "", false, func(context.Context, string) ([]string, error) {
8282 return nil, errors.New("no such host")
8383 })
8484 if err == nil || !strings.Contains(err.Error(), "does not resolve") {
8585 t.Fatalf("%v", err)
8686 }
87- _, err = ResolveGuestForge(ctx, "https://empty.example.org", false, func(context.Context, string) ([]string, error) { return nil, nil })
87+ _, err = ResolveGuestForge(ctx, "https://empty.example.org", "", false, func(context.Context, string) ([]string, error) { return nil, nil })
8888 if err == nil || !strings.Contains(err.Error(), "no address") {
8989 t.Fatalf("%v", err)
9090 }
9191 called := false
92- _, err = ResolveGuestForge(ctx, "https://10.0.0.1", false, func(context.Context, string) ([]string, error) { called = true; return nil, nil })
92+ _, err = ResolveGuestForge(ctx, "https://10.0.0.1", "", false, func(context.Context, string) ([]string, error) { called = true; return nil, nil })
9393 if err == nil || called {
9494 t.Fatalf("an IP literal was looked up or accepted: %v", err)
9595 }
@@ -106,6 +106,28 @@ func TestGuestForgeNilIsOff(t *testing.T) {
106106 }
107107 }
108108
109+func TestResolveGuestForgeTarget(t *testing.T) {
110+ ctx := context.Background()
111+ var asked string
112+ lookup := func(_ context.Context, h string) ([]string, error) { asked = h; return []string{"10.89.0.5"}, nil }
113+ g, err := ResolveGuestForge(ctx, "http://ermite:7341", "ermite:7341", true, lookup)
114+ if err != nil || g.Target != "ermite:7341" || g.RelayTarget() != "ermite:7341" || asked != "ermite" || g.Port != 7341 {
115+ t.Fatalf("%+v %v %q", g, err, asked)
116+ }
117+ asked = ""
118+ if g, err = ResolveGuestForge(ctx, "https://git.example.org", "10.89.0.5:7341", false, lookup); err != nil || asked != "" || g.Target != "10.89.0.5:7341" {
119+ t.Fatalf("an IP target must not be looked up: %+v %v %q", g, err, asked)
120+ }
121+ for _, bad := range []string{"ermite", "ermite:0", "ermite:70000", ":7341", "bad_host:1", "*.x:1"} {
122+ if _, err := ResolveGuestForge(ctx, "http://ermite:7341", bad, true, lookup); err == nil || !strings.Contains(err.Error(), EnvGuestForgeTarget) {
123+ t.Errorf("%q: %v", bad, err)
124+ }
125+ }
126+ if (*GuestForge)(nil).RelayTarget() != "" {
127+ t.Fatal("nil RelayTarget")
128+ }
129+}
130+
109131 func TestLoadConfigGuestForge(t *testing.T) {
110132 get := func(v string) func(string) string {
111133 return func(k string) string {
@@ -74,22 +74,22 @@ func TestResolveGuestForge(t *testing.T) {
74 ctx := context.Background()74 ctx := context.Background()
75 var asked string75 var asked string
76 lookup := func(_ context.Context, h string) ([]string, error) { asked = h; return []string{"127.0.0.1"}, nil }76 lookup := func(_ context.Context, h string) ([]string, error) { asked = h; return []string{"127.0.0.1"}, nil }
77- g, err := ResolveGuestForge(ctx, "http://ermite.bnema.dev:8088", true, lookup)77+ g, err := ResolveGuestForge(ctx, "http://ermite.bnema.dev:8088", "", true, lookup)
78 if err != nil || g.Port != 8088 || asked != "ermite.bnema.dev" {78 if err != nil || g.Port != 8088 || asked != "ermite.bnema.dev" {
79 t.Fatalf("%+v %v %q", g, err, asked)79 t.Fatalf("%+v %v %q", g, err, asked)
80 }80 }
81- _, err = ResolveGuestForge(ctx, "https://nx.example.org", false, func(context.Context, string) ([]string, error) {81+ _, err = ResolveGuestForge(ctx, "https://nx.example.org", "", false, func(context.Context, string) ([]string, error) {
82 return nil, errors.New("no such host")82 return nil, errors.New("no such host")
83 })83 })
84 if err == nil || !strings.Contains(err.Error(), "does not resolve") {84 if err == nil || !strings.Contains(err.Error(), "does not resolve") {
85 t.Fatalf("%v", err)85 t.Fatalf("%v", err)
86 }86 }
87- _, err = ResolveGuestForge(ctx, "https://empty.example.org", false, func(context.Context, string) ([]string, error) { return nil, nil })87+ _, err = ResolveGuestForge(ctx, "https://empty.example.org", "", false, func(context.Context, string) ([]string, error) { return nil, nil })
88 if err == nil || !strings.Contains(err.Error(), "no address") {88 if err == nil || !strings.Contains(err.Error(), "no address") {
89 t.Fatalf("%v", err)89 t.Fatalf("%v", err)
90 }90 }
91 called := false91 called := false
92- _, err = ResolveGuestForge(ctx, "https://10.0.0.1", false, func(context.Context, string) ([]string, error) { called = true; return nil, nil })92+ _, err = ResolveGuestForge(ctx, "https://10.0.0.1", "", false, func(context.Context, string) ([]string, error) { called = true; return nil, nil })
93 if err == nil || called {93 if err == nil || called {
94 t.Fatalf("an IP literal was looked up or accepted: %v", err)94 t.Fatalf("an IP literal was looked up or accepted: %v", err)
95 }95 }
@@ -106,6 +106,28 @@ func TestGuestForgeNilIsOff(t *testing.T) {
106 }106 }
107 }107 }
108 108
109+func TestResolveGuestForgeTarget(t *testing.T) {
110+ ctx := context.Background()
111+ var asked string
112+ lookup := func(_ context.Context, h string) ([]string, error) { asked = h; return []string{"10.89.0.5"}, nil }
113+ g, err := ResolveGuestForge(ctx, "http://ermite:7341", "ermite:7341", true, lookup)
114+ if err != nil || g.Target != "ermite:7341" || g.RelayTarget() != "ermite:7341" || asked != "ermite" || g.Port != 7341 {
115+ t.Fatalf("%+v %v %q", g, err, asked)
116+ }
117+ asked = ""
118+ if g, err = ResolveGuestForge(ctx, "https://git.example.org", "10.89.0.5:7341", false, lookup); err != nil || asked != "" || g.Target != "10.89.0.5:7341" {
119+ t.Fatalf("an IP target must not be looked up: %+v %v %q", g, err, asked)
120+ }
121+ for _, bad := range []string{"ermite", "ermite:0", "ermite:70000", ":7341", "bad_host:1", "*.x:1"} {
122+ if _, err := ResolveGuestForge(ctx, "http://ermite:7341", bad, true, lookup); err == nil || !strings.Contains(err.Error(), EnvGuestForgeTarget) {
123+ t.Errorf("%q: %v", bad, err)
124+ }
125+ }
126+ if (*GuestForge)(nil).RelayTarget() != "" {
127+ t.Fatal("nil RelayTarget")
128+ }
129+}
130+
109 func TestLoadConfigGuestForge(t *testing.T) {131 func TestLoadConfigGuestForge(t *testing.T) {
110 get := func(v string) func(string) string {132 get := func(v string) func(string) string {
111 return func(k string) string {133 return func(k string) string {
M internal/sandbox/netsetup/netsetup.go +20 -1
@@ -17,6 +17,7 @@ import (
1717 "bnema.dev/ermite-runner/internal/cmdrun"
1818 "bnema.dev/ermite-runner/internal/dnsfwd"
1919 "bnema.dev/ermite-runner/internal/netpolicy"
20+ "bnema.dev/ermite-runner/internal/tcprelay"
2021 )
2122
2223 // Plumbing is the host network surface a driver needs. Net implements it; tests
@@ -38,6 +39,10 @@ type Net struct {
3839 DNS *dnsfwd.Forwarder
3940 // ForwardFile is the ip_forward sysctl; overridden in tests.
4041 ForwardFile string
42+ // Relay, when not nil, forwards the guest forge port of every gateway to
43+ // the forge (ERMITE_RUNNER_GUEST_FORGE_TARGET): the forge then does not
44+ // need to listen in the runner's network namespace.
45+ Relay *tcprelay.Relay
4146 }
4247
4348 // New builds a Net: nft runs through run, DNS listens on dnsPort and forwards
@@ -52,7 +57,9 @@ func New(run cmdrun.Runner, dnsPort int, resolvConf string) *Net {
5257 }
5358
5459 // EnableForwarding turns on IPv4 forwarding (NAT needs it). It is a no-op when
55-// already on, and an error that names the fix when it cannot be turned on.
60+// already on, and an error that names the fix when it cannot be turned on. In
61+// a container /proc/sys is read-only: the runtime must start the container's
62+// network namespace with net.ipv4.ip_forward=1 (rootless Podman does).
5663 func (n *Net) EnableForwarding() error {
5764 b, err := os.ReadFile(n.ForwardFile)
5865 if err == nil && strings.TrimSpace(string(b)) == "1" {
@@ -74,11 +81,20 @@ func (n *Net) Attach(ctx context.Context, g netpolicy.Guest) error {
7481 if err := n.DNS.Listen(ctx, g.Gateway); err != nil {
7582 return errors.Join(err, n.Policy.Remove(context.WithoutCancel(ctx), g.Iface))
7683 }
84+ if n.Relay != nil && g.ForgePort != 0 {
85+ if err := n.Relay.Listen(ctx, g.Gateway, g.ForgePort); err != nil {
86+ n.DNS.Close(g.Gateway)
87+ return errors.Join(err, n.Policy.Remove(context.WithoutCancel(ctx), g.Iface))
88+ }
89+ }
7790 return nil
7891 }
7992
8093 // Detach undoes Attach. It is idempotent and keeps going after an error.
8194 func (n *Net) Detach(ctx context.Context, iface string, gateway netip.Addr) error {
95+ if n.Relay != nil {
96+ n.Relay.Close(gateway)
97+ }
8298 n.DNS.Close(gateway)
8399 return n.Policy.Remove(ctx, iface)
84100 }
@@ -89,5 +105,8 @@ func (n *Net) Reset(ctx context.Context) error { return n.Policy.Reset(ctx) }
89105
90106 // Close stops every DNS listener.
91107 func (n *Net) Close() {
108+ if n.Relay != nil {
109+ n.Relay.CloseAll()
110+ }
92111 n.DNS.CloseAll()
93112 }
@@ -17,6 +17,7 @@ import (
17 "bnema.dev/ermite-runner/internal/cmdrun"17 "bnema.dev/ermite-runner/internal/cmdrun"
18 "bnema.dev/ermite-runner/internal/dnsfwd"18 "bnema.dev/ermite-runner/internal/dnsfwd"
19 "bnema.dev/ermite-runner/internal/netpolicy"19 "bnema.dev/ermite-runner/internal/netpolicy"
20+ "bnema.dev/ermite-runner/internal/tcprelay"
20 )21 )
21 22
22 // Plumbing is the host network surface a driver needs. Net implements it; tests23 // Plumbing is the host network surface a driver needs. Net implements it; tests
@@ -38,6 +39,10 @@ type Net struct {
38 DNS *dnsfwd.Forwarder39 DNS *dnsfwd.Forwarder
39 // ForwardFile is the ip_forward sysctl; overridden in tests.40 // ForwardFile is the ip_forward sysctl; overridden in tests.
40 ForwardFile string41 ForwardFile string
42+ // Relay, when not nil, forwards the guest forge port of every gateway to
43+ // the forge (ERMITE_RUNNER_GUEST_FORGE_TARGET): the forge then does not
44+ // need to listen in the runner's network namespace.
45+ Relay *tcprelay.Relay
41 }46 }
42 47
43 // New builds a Net: nft runs through run, DNS listens on dnsPort and forwards48 // New builds a Net: nft runs through run, DNS listens on dnsPort and forwards
@@ -52,7 +57,9 @@ func New(run cmdrun.Runner, dnsPort int, resolvConf string) *Net {
52 }57 }
53 58
54 // EnableForwarding turns on IPv4 forwarding (NAT needs it). It is a no-op when59 // EnableForwarding turns on IPv4 forwarding (NAT needs it). It is a no-op when
55-// already on, and an error that names the fix when it cannot be turned on.60+// already on, and an error that names the fix when it cannot be turned on. In
61+// a container /proc/sys is read-only: the runtime must start the container's
62+// network namespace with net.ipv4.ip_forward=1 (rootless Podman does).
56 func (n *Net) EnableForwarding() error {63 func (n *Net) EnableForwarding() error {
57 b, err := os.ReadFile(n.ForwardFile)64 b, err := os.ReadFile(n.ForwardFile)
58 if err == nil && strings.TrimSpace(string(b)) == "1" {65 if err == nil && strings.TrimSpace(string(b)) == "1" {
@@ -74,11 +81,20 @@ func (n *Net) Attach(ctx context.Context, g netpolicy.Guest) error {
74 if err := n.DNS.Listen(ctx, g.Gateway); err != nil {81 if err := n.DNS.Listen(ctx, g.Gateway); err != nil {
75 return errors.Join(err, n.Policy.Remove(context.WithoutCancel(ctx), g.Iface))82 return errors.Join(err, n.Policy.Remove(context.WithoutCancel(ctx), g.Iface))
76 }83 }
84+ if n.Relay != nil && g.ForgePort != 0 {
85+ if err := n.Relay.Listen(ctx, g.Gateway, g.ForgePort); err != nil {
86+ n.DNS.Close(g.Gateway)
87+ return errors.Join(err, n.Policy.Remove(context.WithoutCancel(ctx), g.Iface))
88+ }
89+ }
77 return nil90 return nil
78 }91 }
79 92
80 // Detach undoes Attach. It is idempotent and keeps going after an error.93 // Detach undoes Attach. It is idempotent and keeps going after an error.
81 func (n *Net) Detach(ctx context.Context, iface string, gateway netip.Addr) error {94 func (n *Net) Detach(ctx context.Context, iface string, gateway netip.Addr) error {
95+ if n.Relay != nil {
96+ n.Relay.Close(gateway)
97+ }
82 n.DNS.Close(gateway)98 n.DNS.Close(gateway)
83 return n.Policy.Remove(ctx, iface)99 return n.Policy.Remove(ctx, iface)
84 }100 }
@@ -89,5 +105,8 @@ func (n *Net) Reset(ctx context.Context) error { return n.Policy.Reset(ctx) }
89 105
90 // Close stops every DNS listener.106 // Close stops every DNS listener.
91 func (n *Net) Close() {107 func (n *Net) Close() {
108+ if n.Relay != nil {
109+ n.Relay.CloseAll()
110+ }
92 n.DNS.CloseAll()111 n.DNS.CloseAll()
93 }112 }
M internal/sandbox/netsetup/netsetup_test.go +56 -0
@@ -13,6 +13,7 @@ import (
1313
1414 "bnema.dev/ermite-runner/internal/cmdrun"
1515 "bnema.dev/ermite-runner/internal/netpolicy"
16+ "bnema.dev/ermite-runner/internal/tcprelay"
1617 )
1718
1819 func guest() netpolicy.Guest {
@@ -81,6 +82,61 @@ func TestAttachRollsBackPolicyWhenDNSFails(t *testing.T) {
8182 }
8283 }
8384
85+// With a relay, Attach relays the guest forge port of the gateway, Detach stops
86+// it, and a guest without a forge port gets no relay listener.
87+func TestAttachStartsAndStopsTheForgeRelay(t *testing.T) {
88+ f := &cmdrun.Fake{}
89+ n := New(f, 0, "")
90+ n.Relay = tcprelay.New("127.0.0.1:1")
91+ t.Cleanup(n.Close)
92+ ctx := context.Background()
93+ g := guest()
94+ if err := n.Attach(ctx, g); err != nil {
95+ t.Skip("cannot bind loopback:", err)
96+ }
97+ if _, ok := n.Relay.Addr(g.Gateway); ok {
98+ t.Fatal("a relay was started for a guest without a forge port")
99+ }
100+ if err := n.Detach(ctx, g.Iface, g.Gateway); err != nil {
101+ t.Fatal(err)
102+ }
103+ // Port 0 lets the kernel pick a free port for the relay.
104+ g.ForgePort = 0
105+ n.Relay = tcprelay.New("127.0.0.1:1")
106+ if err := n.Relay.Listen(ctx, g.Gateway, 0); err != nil {
107+ t.Skip(err)
108+ }
109+ if _, ok := n.Relay.Addr(g.Gateway); !ok {
110+ t.Fatal("relay not listening")
111+ }
112+ if err := n.Detach(ctx, g.Iface, g.Gateway); err != nil {
113+ t.Fatal(err)
114+ }
115+ if _, ok := n.Relay.Addr(g.Gateway); ok {
116+ t.Fatal("relay still listening after Detach")
117+ }
118+}
119+
120+// A relay that cannot bind rolls back the DNS listener and the policy.
121+func TestAttachRollsBackWhenTheRelayFails(t *testing.T) {
122+ f := &cmdrun.Fake{}
123+ n := New(f, 0, "")
124+ n.Relay = tcprelay.New("127.0.0.1:1")
125+ t.Cleanup(n.Close)
126+ ctx := context.Background()
127+ g := guest()
128+ g.ForgePort = 1 // privileged port: binding fails without CAP_NET_BIND_SERVICE
129+ if err := n.Attach(ctx, g); err == nil {
130+ t.Skip("port 1 could be bound here")
131+ }
132+ if _, ok := n.DNS.Addr(g.Gateway); ok {
133+ t.Fatal("DNS still listening after a failed Attach")
134+ }
135+ if len(f.Calls) != 2 {
136+ t.Fatalf("expected add then rollback remove, got %v", f.Lines())
137+ }
138+}
139+
84140 func TestAttachPolicyError(t *testing.T) {
85141 boom := errors.New("nft failed")
86142 n := New(&cmdrun.Fake{Handle: func(cmdrun.Call) ([]byte, error) { return nil, boom }}, 0, "")
@@ -13,6 +13,7 @@ import (
13 13
14 "bnema.dev/ermite-runner/internal/cmdrun"14 "bnema.dev/ermite-runner/internal/cmdrun"
15 "bnema.dev/ermite-runner/internal/netpolicy"15 "bnema.dev/ermite-runner/internal/netpolicy"
16+ "bnema.dev/ermite-runner/internal/tcprelay"
16 )17 )
17 18
18 func guest() netpolicy.Guest {19 func guest() netpolicy.Guest {
@@ -81,6 +82,61 @@ func TestAttachRollsBackPolicyWhenDNSFails(t *testing.T) {
81 }82 }
82 }83 }
83 84
85+// With a relay, Attach relays the guest forge port of the gateway, Detach stops
86+// it, and a guest without a forge port gets no relay listener.
87+func TestAttachStartsAndStopsTheForgeRelay(t *testing.T) {
88+ f := &cmdrun.Fake{}
89+ n := New(f, 0, "")
90+ n.Relay = tcprelay.New("127.0.0.1:1")
91+ t.Cleanup(n.Close)
92+ ctx := context.Background()
93+ g := guest()
94+ if err := n.Attach(ctx, g); err != nil {
95+ t.Skip("cannot bind loopback:", err)
96+ }
97+ if _, ok := n.Relay.Addr(g.Gateway); ok {
98+ t.Fatal("a relay was started for a guest without a forge port")
99+ }
100+ if err := n.Detach(ctx, g.Iface, g.Gateway); err != nil {
101+ t.Fatal(err)
102+ }
103+ // Port 0 lets the kernel pick a free port for the relay.
104+ g.ForgePort = 0
105+ n.Relay = tcprelay.New("127.0.0.1:1")
106+ if err := n.Relay.Listen(ctx, g.Gateway, 0); err != nil {
107+ t.Skip(err)
108+ }
109+ if _, ok := n.Relay.Addr(g.Gateway); !ok {
110+ t.Fatal("relay not listening")
111+ }
112+ if err := n.Detach(ctx, g.Iface, g.Gateway); err != nil {
113+ t.Fatal(err)
114+ }
115+ if _, ok := n.Relay.Addr(g.Gateway); ok {
116+ t.Fatal("relay still listening after Detach")
117+ }
118+}
119+
120+// A relay that cannot bind rolls back the DNS listener and the policy.
121+func TestAttachRollsBackWhenTheRelayFails(t *testing.T) {
122+ f := &cmdrun.Fake{}
123+ n := New(f, 0, "")
124+ n.Relay = tcprelay.New("127.0.0.1:1")
125+ t.Cleanup(n.Close)
126+ ctx := context.Background()
127+ g := guest()
128+ g.ForgePort = 1 // privileged port: binding fails without CAP_NET_BIND_SERVICE
129+ if err := n.Attach(ctx, g); err == nil {
130+ t.Skip("port 1 could be bound here")
131+ }
132+ if _, ok := n.DNS.Addr(g.Gateway); ok {
133+ t.Fatal("DNS still listening after a failed Attach")
134+ }
135+ if len(f.Calls) != 2 {
136+ t.Fatalf("expected add then rollback remove, got %v", f.Lines())
137+ }
138+}
139+
84 func TestAttachPolicyError(t *testing.T) {140 func TestAttachPolicyError(t *testing.T) {
85 boom := errors.New("nft failed")141 boom := errors.New("nft failed")
86 n := New(&cmdrun.Fake{Handle: func(cmdrun.Call) ([]byte, error) { return nil, boom }}, 0, "")142 n := New(&cmdrun.Fake{Handle: func(cmdrun.Call) ([]byte, error) { return nil, boom }}, 0, "")
M internal/sandbox/vm/driver.go +4 -0
@@ -33,6 +33,7 @@ import (
3333 "bnema.dev/ermite-runner/internal/sandbox"
3434 "bnema.dev/ermite-runner/internal/sandbox/disk"
3535 "bnema.dev/ermite-runner/internal/sandbox/netsetup"
36+ "bnema.dev/ermite-runner/internal/tcprelay"
3637 )
3738
3839 // Options are the testable knobs of the driver.
@@ -152,6 +153,9 @@ func New(ctx context.Context, cfg sandbox.Config, opt Options) (*Driver, error)
152153 if d.net == nil {
153154 n := netsetup.New(opt.Run, cfg.DNSPort, "")
154155 n.DNS.Override = cfg.GuestForge.DNSName()
156+ if t := cfg.GuestForge.RelayTarget(); t != "" {
157+ n.Relay = tcprelay.New(t)
158+ }
155159 d.net = n
156160 }
157161 if cfg.GuestForge != nil {
@@ -33,6 +33,7 @@ import (
33 "bnema.dev/ermite-runner/internal/sandbox"33 "bnema.dev/ermite-runner/internal/sandbox"
34 "bnema.dev/ermite-runner/internal/sandbox/disk"34 "bnema.dev/ermite-runner/internal/sandbox/disk"
35 "bnema.dev/ermite-runner/internal/sandbox/netsetup"35 "bnema.dev/ermite-runner/internal/sandbox/netsetup"
36+ "bnema.dev/ermite-runner/internal/tcprelay"
36 )37 )
37 38
38 // Options are the testable knobs of the driver.39 // Options are the testable knobs of the driver.
@@ -152,6 +153,9 @@ func New(ctx context.Context, cfg sandbox.Config, opt Options) (*Driver, error)
152 if d.net == nil {153 if d.net == nil {
153 n := netsetup.New(opt.Run, cfg.DNSPort, "")154 n := netsetup.New(opt.Run, cfg.DNSPort, "")
154 n.DNS.Override = cfg.GuestForge.DNSName()155 n.DNS.Override = cfg.GuestForge.DNSName()
156+ if t := cfg.GuestForge.RelayTarget(); t != "" {
157+ n.Relay = tcprelay.New(t)
158+ }
155 d.net = n159 d.net = n
156 }160 }
157 if cfg.GuestForge != nil {161 if cfg.GuestForge != nil {
A internal/tcprelay/tcprelay.go +183 -0
@@ -0,0 +1,183 @@
1+// SPDX-License-Identifier: MIT
2+
3+// Package tcprelay forwards TCP connections accepted on a sandbox's gateway
4+// address to one fixed target. It carries the guest forge access when the
5+// forge does not listen on the runner's own network namespace: a runner in a
6+// container reaches the forge by name on its container network, and the
7+// sandboxes, which only reach their gateway, are relayed there.
8+package tcprelay
9+
10+import (
11+ "context"
12+ "errors"
13+ "fmt"
14+ "io"
15+ "net"
16+ "net/netip"
17+ "strconv"
18+ "sync"
19+ "time"
20+
21+ "github.com/bnema/zerowrap"
22+)
23+
24+// MaxConns bounds the connections relayed at once for one listener: a guest
25+// cannot exhaust the runner's file descriptors.
26+const MaxConns = 64
27+
28+// dialTimeout bounds the connection to the target.
29+const dialTimeout = 10 * time.Second
30+
31+// Relay forwards every connection accepted on its listeners to Target.
32+type Relay struct {
33+ // Target is host:port, resolved again for every connection.
34+ Target string
35+ // Dial connects to Target; nil means a net.Dialer.
36+ Dial func(ctx context.Context, network, addr string) (net.Conn, error)
37+
38+ mu sync.Mutex
39+ listeners map[netip.Addr]*listener
40+}
41+
42+type listener struct {
43+ ln net.Listener
44+ cancel context.CancelFunc
45+ done sync.WaitGroup
46+ slots chan struct{}
47+}
48+
49+// New returns a relay to target (host:port).
50+func New(target string) *Relay {
51+ return &Relay{Target: target, listeners: map[netip.Addr]*listener{}}
52+}
53+
54+// Listen starts relaying connections to addr:port. It returns once the socket
55+// is bound. Listening twice on the same address is an error.
56+func (r *Relay) Listen(ctx context.Context, addr netip.Addr, port uint16) error {
57+ r.mu.Lock()
58+ defer r.mu.Unlock()
59+ if r.listeners == nil {
60+ r.listeners = map[netip.Addr]*listener{}
61+ }
62+ if _, ok := r.listeners[addr]; ok {
63+ return fmt.Errorf("tcprelay: already listening on %s", addr)
64+ }
65+ var lc net.ListenConfig
66+ ln, err := lc.Listen(ctx, "tcp", net.JoinHostPort(addr.String(), strconv.Itoa(int(port))))
67+ if err != nil {
68+ return fmt.Errorf("tcprelay: listen %s:%d: %w", addr, port, err)
69+ }
70+ lctx, cancel := context.WithCancel(context.WithoutCancel(ctx))
71+ l := &listener{ln: ln, cancel: cancel, slots: make(chan struct{}, MaxConns)}
72+ r.listeners[addr] = l
73+ l.done.Add(1)
74+ go func() {
75+ defer l.done.Done()
76+ r.serve(lctx, l)
77+ }()
78+ return nil
79+}
80+
81+// Addr returns the bound address for addr, useful when port is 0.
82+func (r *Relay) Addr(addr netip.Addr) (net.Addr, bool) {
83+ r.mu.Lock()
84+ defer r.mu.Unlock()
85+ l, ok := r.listeners[addr]
86+ if !ok {
87+ return nil, false
88+ }
89+ return l.ln.Addr(), true
90+}
91+
92+// Close stops the listener of addr and its relayed connections. It is a no-op
93+// when there is none.
94+func (r *Relay) Close(addr netip.Addr) {
95+ r.mu.Lock()
96+ l, ok := r.listeners[addr]
97+ delete(r.listeners, addr)
98+ r.mu.Unlock()
99+ if ok {
100+ l.stop()
101+ }
102+}
103+
104+// CloseAll stops every listener.
105+func (r *Relay) CloseAll() {
106+ r.mu.Lock()
107+ ls := r.listeners
108+ r.listeners = map[netip.Addr]*listener{}
109+ r.mu.Unlock()
110+ for _, l := range ls {
111+ l.stop()
112+ }
113+}
114+
115+func (l *listener) stop() {
116+ l.cancel()
117+ _ = l.ln.Close()
118+ l.done.Wait()
119+}
120+
121+func (r *Relay) serve(ctx context.Context, l *listener) {
122+ log := zerowrap.FromCtx(ctx)
123+ for {
124+ c, err := l.ln.Accept()
125+ if err != nil {
126+ if ctx.Err() != nil || errors.Is(err, net.ErrClosed) {
127+ return
128+ }
129+ log.Warn().Err(err).Msg("tcprelay: accept")
130+ continue
131+ }
132+ select {
133+ case l.slots <- struct{}{}:
134+ default:
135+ log.Warn().Str("target", r.Target).Msg("tcprelay: too many relayed connections, refusing one")
136+ _ = c.Close()
137+ continue
138+ }
139+ l.done.Add(1)
140+ go func() {
141+ defer l.done.Done()
142+ defer func() { <-l.slots }()
143+ r.relay(ctx, c)
144+ }()
145+ }
146+}
147+
148+func (r *Relay) relay(ctx context.Context, in net.Conn) {
149+ defer func() { _ = in.Close() }()
150+ dial := r.Dial
151+ if dial == nil {
152+ d := net.Dialer{}
153+ dial = d.DialContext
154+ }
155+ dctx, cancel := context.WithTimeout(ctx, dialTimeout)
156+ out, err := dial(dctx, "tcp", r.Target)
157+ cancel()
158+ if err != nil {
159+ log := zerowrap.FromCtx(ctx)
160+ log.Warn().Err(err).Str("target", r.Target).Msg("tcprelay: dial")
161+ return
162+ }
163+ defer func() { _ = out.Close() }()
164+ // Closing both ends when the listener stops ends the copies.
165+ stop := context.AfterFunc(ctx, func() { _ = in.Close(); _ = out.Close() })
166+ defer stop()
167+ var wg sync.WaitGroup
168+ wg.Add(2)
169+ go func() { defer wg.Done(); pipe(out, in) }()
170+ go func() { defer wg.Done(); pipe(in, out) }()
171+ wg.Wait()
172+}
173+
174+// pipe copies src to dst, then half-closes dst so the other side sees the end
175+// of the stream while the reverse direction may still be sending.
176+func pipe(dst, src net.Conn) {
177+ _, _ = io.Copy(dst, src)
178+ if cw, ok := dst.(interface{ CloseWrite() error }); ok {
179+ _ = cw.CloseWrite()
180+ } else {
181+ _ = dst.Close()
182+ }
183+}
@@ -0,0 +1,183 @@
1+// SPDX-License-Identifier: MIT
2+
3+// Package tcprelay forwards TCP connections accepted on a sandbox's gateway
4+// address to one fixed target. It carries the guest forge access when the
5+// forge does not listen on the runner's own network namespace: a runner in a
6+// container reaches the forge by name on its container network, and the
7+// sandboxes, which only reach their gateway, are relayed there.
8+package tcprelay
9+
10+import (
11+ "context"
12+ "errors"
13+ "fmt"
14+ "io"
15+ "net"
16+ "net/netip"
17+ "strconv"
18+ "sync"
19+ "time"
20+
21+ "github.com/bnema/zerowrap"
22+)
23+
24+// MaxConns bounds the connections relayed at once for one listener: a guest
25+// cannot exhaust the runner's file descriptors.
26+const MaxConns = 64
27+
28+// dialTimeout bounds the connection to the target.
29+const dialTimeout = 10 * time.Second
30+
31+// Relay forwards every connection accepted on its listeners to Target.
32+type Relay struct {
33+ // Target is host:port, resolved again for every connection.
34+ Target string
35+ // Dial connects to Target; nil means a net.Dialer.
36+ Dial func(ctx context.Context, network, addr string) (net.Conn, error)
37+
38+ mu sync.Mutex
39+ listeners map[netip.Addr]*listener
40+}
41+
42+type listener struct {
43+ ln net.Listener
44+ cancel context.CancelFunc
45+ done sync.WaitGroup
46+ slots chan struct{}
47+}
48+
49+// New returns a relay to target (host:port).
50+func New(target string) *Relay {
51+ return &Relay{Target: target, listeners: map[netip.Addr]*listener{}}
52+}
53+
54+// Listen starts relaying connections to addr:port. It returns once the socket
55+// is bound. Listening twice on the same address is an error.
56+func (r *Relay) Listen(ctx context.Context, addr netip.Addr, port uint16) error {
57+ r.mu.Lock()
58+ defer r.mu.Unlock()
59+ if r.listeners == nil {
60+ r.listeners = map[netip.Addr]*listener{}
61+ }
62+ if _, ok := r.listeners[addr]; ok {
63+ return fmt.Errorf("tcprelay: already listening on %s", addr)
64+ }
65+ var lc net.ListenConfig
66+ ln, err := lc.Listen(ctx, "tcp", net.JoinHostPort(addr.String(), strconv.Itoa(int(port))))
67+ if err != nil {
68+ return fmt.Errorf("tcprelay: listen %s:%d: %w", addr, port, err)
69+ }
70+ lctx, cancel := context.WithCancel(context.WithoutCancel(ctx))
71+ l := &listener{ln: ln, cancel: cancel, slots: make(chan struct{}, MaxConns)}
72+ r.listeners[addr] = l
73+ l.done.Add(1)
74+ go func() {
75+ defer l.done.Done()
76+ r.serve(lctx, l)
77+ }()
78+ return nil
79+}
80+
81+// Addr returns the bound address for addr, useful when port is 0.
82+func (r *Relay) Addr(addr netip.Addr) (net.Addr, bool) {
83+ r.mu.Lock()
84+ defer r.mu.Unlock()
85+ l, ok := r.listeners[addr]
86+ if !ok {
87+ return nil, false
88+ }
89+ return l.ln.Addr(), true
90+}
91+
92+// Close stops the listener of addr and its relayed connections. It is a no-op
93+// when there is none.
94+func (r *Relay) Close(addr netip.Addr) {
95+ r.mu.Lock()
96+ l, ok := r.listeners[addr]
97+ delete(r.listeners, addr)
98+ r.mu.Unlock()
99+ if ok {
100+ l.stop()
101+ }
102+}
103+
104+// CloseAll stops every listener.
105+func (r *Relay) CloseAll() {
106+ r.mu.Lock()
107+ ls := r.listeners
108+ r.listeners = map[netip.Addr]*listener{}
109+ r.mu.Unlock()
110+ for _, l := range ls {
111+ l.stop()
112+ }
113+}
114+
115+func (l *listener) stop() {
116+ l.cancel()
117+ _ = l.ln.Close()
118+ l.done.Wait()
119+}
120+
121+func (r *Relay) serve(ctx context.Context, l *listener) {
122+ log := zerowrap.FromCtx(ctx)
123+ for {
124+ c, err := l.ln.Accept()
125+ if err != nil {
126+ if ctx.Err() != nil || errors.Is(err, net.ErrClosed) {
127+ return
128+ }
129+ log.Warn().Err(err).Msg("tcprelay: accept")
130+ continue
131+ }
132+ select {
133+ case l.slots <- struct{}{}:
134+ default:
135+ log.Warn().Str("target", r.Target).Msg("tcprelay: too many relayed connections, refusing one")
136+ _ = c.Close()
137+ continue
138+ }
139+ l.done.Add(1)
140+ go func() {
141+ defer l.done.Done()
142+ defer func() { <-l.slots }()
143+ r.relay(ctx, c)
144+ }()
145+ }
146+}
147+
148+func (r *Relay) relay(ctx context.Context, in net.Conn) {
149+ defer func() { _ = in.Close() }()
150+ dial := r.Dial
151+ if dial == nil {
152+ d := net.Dialer{}
153+ dial = d.DialContext
154+ }
155+ dctx, cancel := context.WithTimeout(ctx, dialTimeout)
156+ out, err := dial(dctx, "tcp", r.Target)
157+ cancel()
158+ if err != nil {
159+ log := zerowrap.FromCtx(ctx)
160+ log.Warn().Err(err).Str("target", r.Target).Msg("tcprelay: dial")
161+ return
162+ }
163+ defer func() { _ = out.Close() }()
164+ // Closing both ends when the listener stops ends the copies.
165+ stop := context.AfterFunc(ctx, func() { _ = in.Close(); _ = out.Close() })
166+ defer stop()
167+ var wg sync.WaitGroup
168+ wg.Add(2)
169+ go func() { defer wg.Done(); pipe(out, in) }()
170+ go func() { defer wg.Done(); pipe(in, out) }()
171+ wg.Wait()
172+}
173+
174+// pipe copies src to dst, then half-closes dst so the other side sees the end
175+// of the stream while the reverse direction may still be sending.
176+func pipe(dst, src net.Conn) {
177+ _, _ = io.Copy(dst, src)
178+ if cw, ok := dst.(interface{ CloseWrite() error }); ok {
179+ _ = cw.CloseWrite()
180+ } else {
181+ _ = dst.Close()
182+ }
183+}
A internal/tcprelay/tcprelay_test.go +123 -0
@@ -0,0 +1,123 @@
1+// SPDX-License-Identifier: MIT
2+
3+package tcprelay
4+
5+import (
6+ "bufio"
7+ "context"
8+ "io"
9+ "net"
10+ "net/netip"
11+ "strconv"
12+ "testing"
13+ "time"
14+)
15+
16+var lo = netip.MustParseAddr("127.0.0.1")
17+
18+// echo starts a line echo server and returns its address.
19+func echo(t *testing.T) string {
20+ t.Helper()
21+ ln, err := net.Listen("tcp", "127.0.0.1:0")
22+ if err != nil {
23+ t.Skip("cannot bind loopback:", err)
24+ }
25+ t.Cleanup(func() { _ = ln.Close() })
26+ go func() {
27+ for {
28+ c, err := ln.Accept()
29+ if err != nil {
30+ return
31+ }
32+ go func() { defer func() { _ = c.Close() }(); _, _ = io.Copy(c, c) }()
33+ }
34+ }()
35+ return ln.Addr().String()
36+}
37+
38+func dialRelay(t *testing.T, r *Relay) net.Conn {
39+ t.Helper()
40+ a, ok := r.Addr(lo)
41+ if !ok {
42+ t.Fatal("relay not listening")
43+ }
44+ c, err := net.DialTimeout("tcp", a.String(), 2*time.Second)
45+ if err != nil {
46+ t.Fatal(err)
47+ }
48+ t.Cleanup(func() { _ = c.Close() })
49+ return c
50+}
51+
52+func TestRelayForwardsBothWays(t *testing.T) {
53+ r := New(echo(t))
54+ if err := r.Listen(context.Background(), lo, 0); err != nil {
55+ t.Fatal(err)
56+ }
57+ t.Cleanup(r.CloseAll)
58+ c := dialRelay(t, r)
59+ _ = c.SetDeadline(time.Now().Add(5 * time.Second))
60+ if _, err := c.Write([]byte("hello\n")); err != nil {
61+ t.Fatal(err)
62+ }
63+ line, err := bufio.NewReader(c).ReadString('\n')
64+ if err != nil || line != "hello\n" {
65+ t.Fatalf("got %q, %v", line, err)
66+ }
67+}
68+
69+func TestRelayListenTwiceFails(t *testing.T) {
70+ r := New(echo(t))
71+ if err := r.Listen(context.Background(), lo, 0); err != nil {
72+ t.Fatal(err)
73+ }
74+ t.Cleanup(r.CloseAll)
75+ if err := r.Listen(context.Background(), lo, 0); err == nil {
76+ t.Fatal("a second listener on the same address was accepted")
77+ }
78+}
79+
80+func TestRelayCloseStopsListenerAndConnections(t *testing.T) {
81+ r := New(echo(t))
82+ if err := r.Listen(context.Background(), lo, 0); err != nil {
83+ t.Fatal(err)
84+ }
85+ a, _ := r.Addr(lo)
86+ c := dialRelay(t, r)
87+ if _, err := c.Write([]byte("x\n")); err != nil {
88+ t.Fatal(err)
89+ }
90+ _, _ = bufio.NewReader(c).ReadString('\n')
91+ r.Close(lo)
92+ if _, ok := r.Addr(lo); ok {
93+ t.Fatal("still listening after Close")
94+ }
95+ _ = c.SetReadDeadline(time.Now().Add(2 * time.Second))
96+ if _, err := c.Read(make([]byte, 1)); err == nil {
97+ t.Fatal("relayed connection still open after Close")
98+ }
99+ if c2, err := net.DialTimeout("tcp", a.String(), time.Second); err == nil {
100+ _ = c2.Close()
101+ t.Fatal("new connections still accepted after Close")
102+ }
103+ r.Close(lo) // idempotent
104+}
105+
106+func TestRelayUnreachableTargetClosesTheGuestSide(t *testing.T) {
107+ ln, err := net.Listen("tcp", "127.0.0.1:0")
108+ if err != nil {
109+ t.Skip(err)
110+ }
111+ port := ln.Addr().(*net.TCPAddr).Port
112+ _ = ln.Close() // nothing listens there now
113+ r := New(net.JoinHostPort("127.0.0.1", strconv.Itoa(port)))
114+ if err := r.Listen(context.Background(), lo, 0); err != nil {
115+ t.Fatal(err)
116+ }
117+ t.Cleanup(r.CloseAll)
118+ c := dialRelay(t, r)
119+ _ = c.SetReadDeadline(time.Now().Add(5 * time.Second))
120+ if _, err := c.Read(make([]byte, 1)); err == nil {
121+ t.Fatal("expected the connection to be closed")
122+ }
123+}
@@ -0,0 +1,123 @@
1+// SPDX-License-Identifier: MIT
2+
3+package tcprelay
4+
5+import (
6+ "bufio"
7+ "context"
8+ "io"
9+ "net"
10+ "net/netip"
11+ "strconv"
12+ "testing"
13+ "time"
14+)
15+
16+var lo = netip.MustParseAddr("127.0.0.1")
17+
18+// echo starts a line echo server and returns its address.
19+func echo(t *testing.T) string {
20+ t.Helper()
21+ ln, err := net.Listen("tcp", "127.0.0.1:0")
22+ if err != nil {
23+ t.Skip("cannot bind loopback:", err)
24+ }
25+ t.Cleanup(func() { _ = ln.Close() })
26+ go func() {
27+ for {
28+ c, err := ln.Accept()
29+ if err != nil {
30+ return
31+ }
32+ go func() { defer func() { _ = c.Close() }(); _, _ = io.Copy(c, c) }()
33+ }
34+ }()
35+ return ln.Addr().String()
36+}
37+
38+func dialRelay(t *testing.T, r *Relay) net.Conn {
39+ t.Helper()
40+ a, ok := r.Addr(lo)
41+ if !ok {
42+ t.Fatal("relay not listening")
43+ }
44+ c, err := net.DialTimeout("tcp", a.String(), 2*time.Second)
45+ if err != nil {
46+ t.Fatal(err)
47+ }
48+ t.Cleanup(func() { _ = c.Close() })
49+ return c
50+}
51+
52+func TestRelayForwardsBothWays(t *testing.T) {
53+ r := New(echo(t))
54+ if err := r.Listen(context.Background(), lo, 0); err != nil {
55+ t.Fatal(err)
56+ }
57+ t.Cleanup(r.CloseAll)
58+ c := dialRelay(t, r)
59+ _ = c.SetDeadline(time.Now().Add(5 * time.Second))
60+ if _, err := c.Write([]byte("hello\n")); err != nil {
61+ t.Fatal(err)
62+ }
63+ line, err := bufio.NewReader(c).ReadString('\n')
64+ if err != nil || line != "hello\n" {
65+ t.Fatalf("got %q, %v", line, err)
66+ }
67+}
68+
69+func TestRelayListenTwiceFails(t *testing.T) {
70+ r := New(echo(t))
71+ if err := r.Listen(context.Background(), lo, 0); err != nil {
72+ t.Fatal(err)
73+ }
74+ t.Cleanup(r.CloseAll)
75+ if err := r.Listen(context.Background(), lo, 0); err == nil {
76+ t.Fatal("a second listener on the same address was accepted")
77+ }
78+}
79+
80+func TestRelayCloseStopsListenerAndConnections(t *testing.T) {
81+ r := New(echo(t))
82+ if err := r.Listen(context.Background(), lo, 0); err != nil {
83+ t.Fatal(err)
84+ }
85+ a, _ := r.Addr(lo)
86+ c := dialRelay(t, r)
87+ if _, err := c.Write([]byte("x\n")); err != nil {
88+ t.Fatal(err)
89+ }
90+ _, _ = bufio.NewReader(c).ReadString('\n')
91+ r.Close(lo)
92+ if _, ok := r.Addr(lo); ok {
93+ t.Fatal("still listening after Close")
94+ }
95+ _ = c.SetReadDeadline(time.Now().Add(2 * time.Second))
96+ if _, err := c.Read(make([]byte, 1)); err == nil {
97+ t.Fatal("relayed connection still open after Close")
98+ }
99+ if c2, err := net.DialTimeout("tcp", a.String(), time.Second); err == nil {
100+ _ = c2.Close()
101+ t.Fatal("new connections still accepted after Close")
102+ }
103+ r.Close(lo) // idempotent
104+}
105+
106+func TestRelayUnreachableTargetClosesTheGuestSide(t *testing.T) {
107+ ln, err := net.Listen("tcp", "127.0.0.1:0")
108+ if err != nil {
109+ t.Skip(err)
110+ }
111+ port := ln.Addr().(*net.TCPAddr).Port
112+ _ = ln.Close() // nothing listens there now
113+ r := New(net.JoinHostPort("127.0.0.1", strconv.Itoa(port)))
114+ if err := r.Listen(context.Background(), lo, 0); err != nil {
115+ t.Fatal(err)
116+ }
117+ t.Cleanup(r.CloseAll)
118+ c := dialRelay(t, r)
119+ _ = c.SetReadDeadline(time.Now().Add(5 * time.Second))
120+ if _, err := c.Read(make([]byte, 1)); err == nil {
121+ t.Fatal("expected the connection to be closed")
122+ }
123+}

Keyboard shortcuts

Single-key shortcuts are on. ctrl k and esc work whatever this says.

ctrl k :
go to a section, branch, tag, file or repository
?
this list
j k
move the row cursor down / up
enter
open the row under the cursor; on Files, a directory opens or closes in place
↑ ↓ ← → home end
in the file tree (focus on a row): previous / next row, close / open a directory or go to its parent / first entry, first / last row. They never open a file
l h
on Files: open a directory or a file / close a directory or go up to its parent, then to the parent page. Elsewhere: into the side panel (on a merge request, its files) / back to the content; there j k move between its controls
g g G
first / last row
/
filter the list; on Files, find a file anywhere in the repository (esc clears, enter opens the best match)
b
switch branch or tag
g s g f g c g b g m g i g r g a
tabs: summary, files, commits, branches & tags, merge requests, issues, releases, actions (when available)
g h
home
w
wide mode: use the whole width of the screen, or not (remembered)
y
copy a permalink pinned to the commit
d u x
diff: split / unified / collapse or expand all
e E
diff: the whole file in view / every file, with the lines the hunks hide
n
merge request: next note in the diff
esc
close a dialog, leave a field

Branches 4

Tags 11

↑↓ moveenter openesc close