detached a743879

feat(sandbox): add the Sandbox and Driver interfaces, config and isolation selection

commit
a743879da66a62cb640b3cd58e0cacce21c1b1da
author
brice · · 2026-10-12 15:35 UTC
parent
4f2d9a2
signature
GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email
9 files · +1130 -0
Unified
UnifiedSplit
A internal/sandbox/config.go +262 -0
@@ -0,0 +1,262 @@
1+// SPDX-License-Identifier: MIT
2+
3+package sandbox
4+
5+import (
6+ "errors"
7+ "fmt"
8+ "net/netip"
9+ "os"
10+ "path/filepath"
11+ "runtime"
12+ "strconv"
13+ "strings"
14+)
15+
16+// Environment variables read by LoadConfig. The other ERMITE_RUNNER_* variables
17+// (URL, NAME, LABELS, ...) belong to the host process, not to the sandboxes.
18+const (
19+ EnvIsolation = "ERMITE_RUNNER_ISOLATION"
20+ EnvStateDir = "ERMITE_RUNNER_STATE_DIR"
21+ EnvCapacity = "ERMITE_RUNNER_CAPACITY"
22+ EnvImage = "ERMITE_RUNNER_IMAGE"
23+ EnvJobCPUs = "ERMITE_RUNNER_JOB_CPUS"
24+ EnvJobMemory = "ERMITE_RUNNER_JOB_MEMORY"
25+ EnvJobDisk = "ERMITE_RUNNER_JOB_DISK"
26+ EnvUIDBase = "ERMITE_RUNNER_UID_BASE"
27+ EnvNetwork = "ERMITE_RUNNER_NETWORK"
28+ EnvFirecracker = "ERMITE_RUNNER_FIRECRACKER"
29+ EnvJailer = "ERMITE_RUNNER_JAILER"
30+ EnvJailerMode = "ERMITE_RUNNER_JAILER_MODE"
31+ EnvKernel = "ERMITE_RUNNER_KERNEL"
32+)
33+
34+// Defaults.
35+const (
36+ DefaultStateDir = "/var/lib/ermite-runner"
37+ DefaultOCIImage = "ermite-runner-ubuntu-24.04:latest"
38+ DefaultUIDBase = 900000
39+ DefaultNetwork = "10.200.0.0/16"
40+ DefaultCPUs = 2
41+ DefaultMemoryMiB = 4096
42+ DefaultDiskMiB = 64 * 1024
43+)
44+
45+// Config is the sandbox part of the runner configuration.
46+type Config struct {
47+ Isolation Mode
48+ StateDir string
49+ Capacity int
50+
51+ // Image is ERMITE_RUNNER_IMAGE: a directory (VM: holds the kernel and
52+ // base.ext4) or an OCI reference (container). A value that is empty or does
53+ // not fit the selected mode falls back to the default for that mode.
54+ Image string
55+
56+ JobCPUs int
57+ JobMemoryMiB int
58+ JobDiskMiB int
59+
60+ // UIDBase+slot is the uid and gid the jailer drops to.
61+ UIDBase int
62+ // Network is the pool of /30 subnets, one per slot (VM TAPs, container
63+ // bridges).
64+ Network netip.Prefix
65+
66+ // Firecracker and Jailer are binary paths; empty means look in PATH.
67+ Firecracker string
68+ Jailer string
69+ // Kernel overrides <image dir>/vmlinux (Image on arm64).
70+ Kernel string
71+ // JailerOff runs Firecracker without the jailer (development and tests as
72+ // an unprivileged user). Never use it in production: it only drops the
73+ // chroot, uid and namespace confinement of the VMM process.
74+ JailerOff bool
75+
76+ // KVMPath is the KVM device; tests point it elsewhere to hide /dev/kvm.
77+ KVMPath string
78+ // Executable is attached read-only to every sandbox. Empty means
79+ // os.Executable(). It must be a static binary (CGO_ENABLED=0).
80+ Executable string
81+ // DNSPort is the port of the DNS forwarder; 53 in production. Tests that
82+ // cannot bind it use another port with no guest.
83+ DNSPort int
84+}
85+
86+// LoadConfig reads the sandbox settings from getenv (os.Getenv in production).
87+func LoadConfig(getenv func(string) string) (Config, error) {
88+ c := Config{
89+ Isolation: Auto,
90+ StateDir: DefaultStateDir,
91+ Capacity: 1,
92+ JobCPUs: DefaultCPUs,
93+ JobMemoryMiB: DefaultMemoryMiB,
94+ JobDiskMiB: DefaultDiskMiB,
95+ UIDBase: DefaultUIDBase,
96+ KVMPath: "/dev/kvm",
97+ DNSPort: 53,
98+ }
99+ var err error
100+ if v := getenv(EnvIsolation); v != "" {
101+ switch m := Mode(strings.ToLower(strings.TrimSpace(v))); m {
102+ case Auto, VM, Container:
103+ c.Isolation = m
104+ default:
105+ return c, fmt.Errorf("%s: %q is not auto, vm or container", EnvIsolation, v)
106+ }
107+ }
108+ if v := getenv(EnvStateDir); v != "" {
109+ c.StateDir = v
110+ }
111+ if !filepath.IsAbs(c.StateDir) {
112+ return c, fmt.Errorf("%s: %q must be an absolute path", EnvStateDir, c.StateDir)
113+ }
114+ if c.Capacity, err = intEnv(getenv, EnvCapacity, c.Capacity, 1, 1024); err != nil {
115+ return c, err
116+ }
117+ c.Image = getenv(EnvImage)
118+ if c.JobCPUs, err = intEnv(getenv, EnvJobCPUs, c.JobCPUs, 1, 128); err != nil {
119+ return c, err
120+ }
121+ if v := getenv(EnvJobMemory); v != "" {
122+ if c.JobMemoryMiB, err = ParseSizeMiB(v); err != nil || c.JobMemoryMiB < 256 {
123+ return c, fmt.Errorf("%s: %q: need a size of at least 256M", EnvJobMemory, v)
124+ }
125+ }
126+ if v := getenv(EnvJobDisk); v != "" {
127+ if c.JobDiskMiB, err = ParseSizeMiB(v); err != nil || c.JobDiskMiB < 1024 {
128+ return c, fmt.Errorf("%s: %q: need a size of at least 1G", EnvJobDisk, v)
129+ }
130+ }
131+ if c.UIDBase, err = intEnv(getenv, EnvUIDBase, c.UIDBase, 1000, 4_000_000_000); err != nil {
132+ return c, err
133+ }
134+ if c.UIDBase+c.Capacity > 4_294_967_294 {
135+ return c, fmt.Errorf("%s: %d + capacity overflows the uid range", EnvUIDBase, c.UIDBase)
136+ }
137+ net := DefaultNetwork
138+ if v := getenv(EnvNetwork); v != "" {
139+ net = v
140+ }
141+ if c.Network, err = netip.ParsePrefix(net); err != nil || !c.Network.Addr().Is4() || c.Network.Bits() > 30 {
142+ return c, fmt.Errorf("%s: %q is not an IPv4 prefix of at most /30", EnvNetwork, net)
143+ }
144+ if 1<<(30-c.Network.Bits()) < c.Capacity {
145+ return c, fmt.Errorf("%s: %s has fewer /30 subnets than the capacity %d", EnvNetwork, c.Network, c.Capacity)
146+ }
147+ c.Firecracker = getenv(EnvFirecracker)
148+ c.Jailer = getenv(EnvJailer)
149+ c.Kernel = getenv(EnvKernel)
150+ switch strings.ToLower(getenv(EnvJailerMode)) {
151+ case "", "on":
152+ case "off":
153+ c.JailerOff = true
154+ default:
155+ return c, fmt.Errorf("%s: %q is not on or off", EnvJailerMode, getenv(EnvJailerMode))
156+ }
157+ return c, nil
158+}
159+
160+func intEnv(getenv func(string) string, key string, def, lo, hi int) (int, error) {
161+ v := getenv(key)
162+ if v == "" {
163+ return def, nil
164+ }
165+ n, err := strconv.Atoi(strings.TrimSpace(v))
166+ if err != nil || n < lo || n > hi {
167+ return 0, fmt.Errorf("%s: %q is not an integer in [%d, %d]", key, v, lo, hi)
168+ }
169+ return n, nil
170+}
171+
172+// ParseSizeMiB parses "4096", "512M", "4G", "64GiB", "1T" into MiB. A bare
173+// number is MiB. Suffixes are binary (1G = 1024 MiB) and case-insensitive.
174+func ParseSizeMiB(s string) (int, error) {
175+ s = strings.ToLower(strings.TrimSpace(s))
176+ s = strings.TrimSuffix(strings.TrimSuffix(s, "b"), "i")
177+ mult := 1
178+ if s != "" {
179+ switch s[len(s)-1] {
180+ case 'k':
181+ mult, s = 0, s[:len(s)-1]
182+ case 'm':
183+ mult, s = 1, s[:len(s)-1]
184+ case 'g':
185+ mult, s = 1024, s[:len(s)-1]
186+ case 't':
187+ mult, s = 1024*1024, s[:len(s)-1]
188+ }
189+ }
190+ n, err := strconv.Atoi(s)
191+ if err != nil || n <= 0 || mult == 0 {
192+ return 0, errors.New("invalid size")
193+ }
194+ if n > (1<<31-1)/mult {
195+ return 0, errors.New("size too large")
196+ }
197+ return n * mult, nil
198+}
199+
200+// ImageDir returns the VM image directory: Image when it is a directory path,
201+// else <StateDir>/image.
202+func (c Config) ImageDir() string {
203+ if c.Image != "" && looksLikePath(c.Image) {
204+ return c.Image
205+ }
206+ return filepath.Join(c.StateDir, "image")
207+}
208+
209+// OCIImage returns the container image reference: Image when it is not a
210+// directory path, else DefaultOCIImage.
211+func (c Config) OCIImage() string {
212+ if c.Image != "" && !looksLikePath(c.Image) {
213+ return c.Image
214+ }
215+ return DefaultOCIImage
216+}
217+
218+func looksLikePath(s string) bool {
219+ return strings.HasPrefix(s, "/") || strings.HasPrefix(s, "./") || strings.HasPrefix(s, "../")
220+}
221+
222+// KernelPath returns the guest kernel: Kernel, else <ImageDir>/vmlinux (x86-64)
223+// or <ImageDir>/Image (arm64).
224+func (c Config) KernelPath() string {
225+ if c.Kernel != "" {
226+ return c.Kernel
227+ }
228+ name := "vmlinux"
229+ if runtime.GOARCH == "arm64" {
230+ name = "Image"
231+ }
232+ return filepath.Join(c.ImageDir(), name)
233+}
234+
235+// BaseImagePath returns the shared read-only base disk.
236+func (c Config) BaseImagePath() string { return filepath.Join(c.ImageDir(), "base.ext4") }
237+
238+// ExecutablePath returns the binary attached to sandboxes.
239+func (c Config) ExecutablePath() (string, error) {
240+ if c.Executable != "" {
241+ return c.Executable, nil
242+ }
243+ p, err := os.Executable()
244+ if err != nil {
245+ return "", fmt.Errorf("sandbox: locate own executable: %w", err)
246+ }
247+ return filepath.EvalSymlinks(p)
248+}
249+
250+// Resolve fills the zero resource fields of a JobSpec from the config.
251+func (c Config) Resolve(s JobSpec) JobSpec {
252+ if s.CPUs <= 0 {
253+ s.CPUs = c.JobCPUs
254+ }
255+ if s.MemoryMiB <= 0 {
256+ s.MemoryMiB = c.JobMemoryMiB
257+ }
258+ if s.DiskMiB <= 0 {
259+ s.DiskMiB = c.JobDiskMiB
260+ }
261+ return s
262+}
@@ -0,0 +1,262 @@
1+// SPDX-License-Identifier: MIT
2+
3+package sandbox
4+
5+import (
6+ "errors"
7+ "fmt"
8+ "net/netip"
9+ "os"
10+ "path/filepath"
11+ "runtime"
12+ "strconv"
13+ "strings"
14+)
15+
16+// Environment variables read by LoadConfig. The other ERMITE_RUNNER_* variables
17+// (URL, NAME, LABELS, ...) belong to the host process, not to the sandboxes.
18+const (
19+ EnvIsolation = "ERMITE_RUNNER_ISOLATION"
20+ EnvStateDir = "ERMITE_RUNNER_STATE_DIR"
21+ EnvCapacity = "ERMITE_RUNNER_CAPACITY"
22+ EnvImage = "ERMITE_RUNNER_IMAGE"
23+ EnvJobCPUs = "ERMITE_RUNNER_JOB_CPUS"
24+ EnvJobMemory = "ERMITE_RUNNER_JOB_MEMORY"
25+ EnvJobDisk = "ERMITE_RUNNER_JOB_DISK"
26+ EnvUIDBase = "ERMITE_RUNNER_UID_BASE"
27+ EnvNetwork = "ERMITE_RUNNER_NETWORK"
28+ EnvFirecracker = "ERMITE_RUNNER_FIRECRACKER"
29+ EnvJailer = "ERMITE_RUNNER_JAILER"
30+ EnvJailerMode = "ERMITE_RUNNER_JAILER_MODE"
31+ EnvKernel = "ERMITE_RUNNER_KERNEL"
32+)
33+
34+// Defaults.
35+const (
36+ DefaultStateDir = "/var/lib/ermite-runner"
37+ DefaultOCIImage = "ermite-runner-ubuntu-24.04:latest"
38+ DefaultUIDBase = 900000
39+ DefaultNetwork = "10.200.0.0/16"
40+ DefaultCPUs = 2
41+ DefaultMemoryMiB = 4096
42+ DefaultDiskMiB = 64 * 1024
43+)
44+
45+// Config is the sandbox part of the runner configuration.
46+type Config struct {
47+ Isolation Mode
48+ StateDir string
49+ Capacity int
50+
51+ // Image is ERMITE_RUNNER_IMAGE: a directory (VM: holds the kernel and
52+ // base.ext4) or an OCI reference (container). A value that is empty or does
53+ // not fit the selected mode falls back to the default for that mode.
54+ Image string
55+
56+ JobCPUs int
57+ JobMemoryMiB int
58+ JobDiskMiB int
59+
60+ // UIDBase+slot is the uid and gid the jailer drops to.
61+ UIDBase int
62+ // Network is the pool of /30 subnets, one per slot (VM TAPs, container
63+ // bridges).
64+ Network netip.Prefix
65+
66+ // Firecracker and Jailer are binary paths; empty means look in PATH.
67+ Firecracker string
68+ Jailer string
69+ // Kernel overrides <image dir>/vmlinux (Image on arm64).
70+ Kernel string
71+ // JailerOff runs Firecracker without the jailer (development and tests as
72+ // an unprivileged user). Never use it in production: it only drops the
73+ // chroot, uid and namespace confinement of the VMM process.
74+ JailerOff bool
75+
76+ // KVMPath is the KVM device; tests point it elsewhere to hide /dev/kvm.
77+ KVMPath string
78+ // Executable is attached read-only to every sandbox. Empty means
79+ // os.Executable(). It must be a static binary (CGO_ENABLED=0).
80+ Executable string
81+ // DNSPort is the port of the DNS forwarder; 53 in production. Tests that
82+ // cannot bind it use another port with no guest.
83+ DNSPort int
84+}
85+
86+// LoadConfig reads the sandbox settings from getenv (os.Getenv in production).
87+func LoadConfig(getenv func(string) string) (Config, error) {
88+ c := Config{
89+ Isolation: Auto,
90+ StateDir: DefaultStateDir,
91+ Capacity: 1,
92+ JobCPUs: DefaultCPUs,
93+ JobMemoryMiB: DefaultMemoryMiB,
94+ JobDiskMiB: DefaultDiskMiB,
95+ UIDBase: DefaultUIDBase,
96+ KVMPath: "/dev/kvm",
97+ DNSPort: 53,
98+ }
99+ var err error
100+ if v := getenv(EnvIsolation); v != "" {
101+ switch m := Mode(strings.ToLower(strings.TrimSpace(v))); m {
102+ case Auto, VM, Container:
103+ c.Isolation = m
104+ default:
105+ return c, fmt.Errorf("%s: %q is not auto, vm or container", EnvIsolation, v)
106+ }
107+ }
108+ if v := getenv(EnvStateDir); v != "" {
109+ c.StateDir = v
110+ }
111+ if !filepath.IsAbs(c.StateDir) {
112+ return c, fmt.Errorf("%s: %q must be an absolute path", EnvStateDir, c.StateDir)
113+ }
114+ if c.Capacity, err = intEnv(getenv, EnvCapacity, c.Capacity, 1, 1024); err != nil {
115+ return c, err
116+ }
117+ c.Image = getenv(EnvImage)
118+ if c.JobCPUs, err = intEnv(getenv, EnvJobCPUs, c.JobCPUs, 1, 128); err != nil {
119+ return c, err
120+ }
121+ if v := getenv(EnvJobMemory); v != "" {
122+ if c.JobMemoryMiB, err = ParseSizeMiB(v); err != nil || c.JobMemoryMiB < 256 {
123+ return c, fmt.Errorf("%s: %q: need a size of at least 256M", EnvJobMemory, v)
124+ }
125+ }
126+ if v := getenv(EnvJobDisk); v != "" {
127+ if c.JobDiskMiB, err = ParseSizeMiB(v); err != nil || c.JobDiskMiB < 1024 {
128+ return c, fmt.Errorf("%s: %q: need a size of at least 1G", EnvJobDisk, v)
129+ }
130+ }
131+ if c.UIDBase, err = intEnv(getenv, EnvUIDBase, c.UIDBase, 1000, 4_000_000_000); err != nil {
132+ return c, err
133+ }
134+ if c.UIDBase+c.Capacity > 4_294_967_294 {
135+ return c, fmt.Errorf("%s: %d + capacity overflows the uid range", EnvUIDBase, c.UIDBase)
136+ }
137+ net := DefaultNetwork
138+ if v := getenv(EnvNetwork); v != "" {
139+ net = v
140+ }
141+ if c.Network, err = netip.ParsePrefix(net); err != nil || !c.Network.Addr().Is4() || c.Network.Bits() > 30 {
142+ return c, fmt.Errorf("%s: %q is not an IPv4 prefix of at most /30", EnvNetwork, net)
143+ }
144+ if 1<<(30-c.Network.Bits()) < c.Capacity {
145+ return c, fmt.Errorf("%s: %s has fewer /30 subnets than the capacity %d", EnvNetwork, c.Network, c.Capacity)
146+ }
147+ c.Firecracker = getenv(EnvFirecracker)
148+ c.Jailer = getenv(EnvJailer)
149+ c.Kernel = getenv(EnvKernel)
150+ switch strings.ToLower(getenv(EnvJailerMode)) {
151+ case "", "on":
152+ case "off":
153+ c.JailerOff = true
154+ default:
155+ return c, fmt.Errorf("%s: %q is not on or off", EnvJailerMode, getenv(EnvJailerMode))
156+ }
157+ return c, nil
158+}
159+
160+func intEnv(getenv func(string) string, key string, def, lo, hi int) (int, error) {
161+ v := getenv(key)
162+ if v == "" {
163+ return def, nil
164+ }
165+ n, err := strconv.Atoi(strings.TrimSpace(v))
166+ if err != nil || n < lo || n > hi {
167+ return 0, fmt.Errorf("%s: %q is not an integer in [%d, %d]", key, v, lo, hi)
168+ }
169+ return n, nil
170+}
171+
172+// ParseSizeMiB parses "4096", "512M", "4G", "64GiB", "1T" into MiB. A bare
173+// number is MiB. Suffixes are binary (1G = 1024 MiB) and case-insensitive.
174+func ParseSizeMiB(s string) (int, error) {
175+ s = strings.ToLower(strings.TrimSpace(s))
176+ s = strings.TrimSuffix(strings.TrimSuffix(s, "b"), "i")
177+ mult := 1
178+ if s != "" {
179+ switch s[len(s)-1] {
180+ case 'k':
181+ mult, s = 0, s[:len(s)-1]
182+ case 'm':
183+ mult, s = 1, s[:len(s)-1]
184+ case 'g':
185+ mult, s = 1024, s[:len(s)-1]
186+ case 't':
187+ mult, s = 1024*1024, s[:len(s)-1]
188+ }
189+ }
190+ n, err := strconv.Atoi(s)
191+ if err != nil || n <= 0 || mult == 0 {
192+ return 0, errors.New("invalid size")
193+ }
194+ if n > (1<<31-1)/mult {
195+ return 0, errors.New("size too large")
196+ }
197+ return n * mult, nil
198+}
199+
200+// ImageDir returns the VM image directory: Image when it is a directory path,
201+// else <StateDir>/image.
202+func (c Config) ImageDir() string {
203+ if c.Image != "" && looksLikePath(c.Image) {
204+ return c.Image
205+ }
206+ return filepath.Join(c.StateDir, "image")
207+}
208+
209+// OCIImage returns the container image reference: Image when it is not a
210+// directory path, else DefaultOCIImage.
211+func (c Config) OCIImage() string {
212+ if c.Image != "" && !looksLikePath(c.Image) {
213+ return c.Image
214+ }
215+ return DefaultOCIImage
216+}
217+
218+func looksLikePath(s string) bool {
219+ return strings.HasPrefix(s, "/") || strings.HasPrefix(s, "./") || strings.HasPrefix(s, "../")
220+}
221+
222+// KernelPath returns the guest kernel: Kernel, else <ImageDir>/vmlinux (x86-64)
223+// or <ImageDir>/Image (arm64).
224+func (c Config) KernelPath() string {
225+ if c.Kernel != "" {
226+ return c.Kernel
227+ }
228+ name := "vmlinux"
229+ if runtime.GOARCH == "arm64" {
230+ name = "Image"
231+ }
232+ return filepath.Join(c.ImageDir(), name)
233+}
234+
235+// BaseImagePath returns the shared read-only base disk.
236+func (c Config) BaseImagePath() string { return filepath.Join(c.ImageDir(), "base.ext4") }
237+
238+// ExecutablePath returns the binary attached to sandboxes.
239+func (c Config) ExecutablePath() (string, error) {
240+ if c.Executable != "" {
241+ return c.Executable, nil
242+ }
243+ p, err := os.Executable()
244+ if err != nil {
245+ return "", fmt.Errorf("sandbox: locate own executable: %w", err)
246+ }
247+ return filepath.EvalSymlinks(p)
248+}
249+
250+// Resolve fills the zero resource fields of a JobSpec from the config.
251+func (c Config) Resolve(s JobSpec) JobSpec {
252+ if s.CPUs <= 0 {
253+ s.CPUs = c.JobCPUs
254+ }
255+ if s.MemoryMiB <= 0 {
256+ s.MemoryMiB = c.JobMemoryMiB
257+ }
258+ if s.DiskMiB <= 0 {
259+ s.DiskMiB = c.JobDiskMiB
260+ }
261+ return s
262+}
A internal/sandbox/contract.go +30 -0
@@ -0,0 +1,30 @@
1+// SPDX-License-Identifier: MIT
2+
3+package sandbox
4+
5+// The host/guest contract shared by the drivers and the guest side
6+// (internal/guest). Changing a value here changes both sides at once because
7+// the host always attaches its own executable to the sandbox.
8+const (
9+ // VsockPort is the vsock port the executor connects to on the host (CID 2).
10+ VsockPort = 1024
11+ // GuestExecDir and GuestExecName locate the runner executable inside the
12+ // sandbox (read-only), in both modes.
13+ GuestExecDir = "/opt/ermite"
14+ GuestExecName = "ermite-runner"
15+ // ContainerRunDir is the per-job directory bind-mounted into containers;
16+ // ContainerSocket is the executor stream socket inside it.
17+ ContainerRunDir = "/run/ermite"
18+ ContainerSocket = "exec.sock"
19+ // GuestScratch is where the per-job scratch disk (VM) or volume (container)
20+ // is mounted; dockerd keeps its data root there.
21+ GuestScratch = "/scratch"
22+
23+ // LabelJob marks every Docker object created for a job; reaping lists by it.
24+ LabelJob = "dev.bnema.ermite-runner.job"
25+
26+ // Kernel command line keys of the VM (read from /proc/cmdline by init).
27+ CmdlineSandbox = "ermite.sandbox" // "vm"
28+ CmdlineConnect = "ermite.connect" // "vsock:1024"
29+ CmdlineGateway = "ermite.gw" // host side of the /30, also the DNS server
30+)
@@ -0,0 +1,30 @@
1+// SPDX-License-Identifier: MIT
2+
3+package sandbox
4+
5+// The host/guest contract shared by the drivers and the guest side
6+// (internal/guest). Changing a value here changes both sides at once because
7+// the host always attaches its own executable to the sandbox.
8+const (
9+ // VsockPort is the vsock port the executor connects to on the host (CID 2).
10+ VsockPort = 1024
11+ // GuestExecDir and GuestExecName locate the runner executable inside the
12+ // sandbox (read-only), in both modes.
13+ GuestExecDir = "/opt/ermite"
14+ GuestExecName = "ermite-runner"
15+ // ContainerRunDir is the per-job directory bind-mounted into containers;
16+ // ContainerSocket is the executor stream socket inside it.
17+ ContainerRunDir = "/run/ermite"
18+ ContainerSocket = "exec.sock"
19+ // GuestScratch is where the per-job scratch disk (VM) or volume (container)
20+ // is mounted; dockerd keeps its data root there.
21+ GuestScratch = "/scratch"
22+
23+ // LabelJob marks every Docker object created for a job; reaping lists by it.
24+ LabelJob = "dev.bnema.ermite-runner.job"
25+
26+ // Kernel command line keys of the VM (read from /proc/cmdline by init).
27+ CmdlineSandbox = "ermite.sandbox" // "vm"
28+ CmdlineConnect = "ermite.connect" // "vsock:1024"
29+ CmdlineGateway = "ermite.gw" // host side of the /30, also the DNS server
30+)
A internal/sandbox/id.go +30 -0
@@ -0,0 +1,30 @@
1+// SPDX-License-Identifier: MIT
2+
3+package sandbox
4+
5+import (
6+ "crypto/rand"
7+ "encoding/hex"
8+ "fmt"
9+ "strings"
10+)
11+
12+// NewID returns "er" + 8 random hex digits: short (socket paths), unique enough
13+// for concurrent sandboxes, and safe in file, interface and Docker object names.
14+func NewID() (string, error) {
15+ var b [4]byte
16+ if _, err := rand.Read(b[:]); err != nil {
17+ return "", fmt.Errorf("random id: %w", err)
18+ }
19+ return "er" + hex.EncodeToString(b[:]), nil
20+}
21+
22+// ValidID reports whether id has the form produced by NewID. It guards paths and
23+// process matches built from ids read back from disk or from /proc.
24+func ValidID(id string) bool {
25+ if len(id) != 10 || !strings.HasPrefix(id, "er") {
26+ return false
27+ }
28+ _, err := hex.DecodeString(id[2:])
29+ return err == nil
30+}
@@ -0,0 +1,30 @@
1+// SPDX-License-Identifier: MIT
2+
3+package sandbox
4+
5+import (
6+ "crypto/rand"
7+ "encoding/hex"
8+ "fmt"
9+ "strings"
10+)
11+
12+// NewID returns "er" + 8 random hex digits: short (socket paths), unique enough
13+// for concurrent sandboxes, and safe in file, interface and Docker object names.
14+func NewID() (string, error) {
15+ var b [4]byte
16+ if _, err := rand.Read(b[:]); err != nil {
17+ return "", fmt.Errorf("random id: %w", err)
18+ }
19+ return "er" + hex.EncodeToString(b[:]), nil
20+}
21+
22+// ValidID reports whether id has the form produced by NewID. It guards paths and
23+// process matches built from ids read back from disk or from /proc.
24+func ValidID(id string) bool {
25+ if len(id) != 10 || !strings.HasPrefix(id, "er") {
26+ return false
27+ }
28+ _, err := hex.DecodeString(id[2:])
29+ return err == nil
30+}
A internal/sandbox/netsetup/netsetup.go +91 -0
@@ -0,0 +1,91 @@
1+// SPDX-License-Identifier: MIT
2+
3+// Package netsetup is the host-side network plumbing shared by both sandbox
4+// drivers: the nftables policy (package netpolicy), the DNS forwarder (package
5+// dnsfwd) and IPv4 forwarding. Each driver creates its own interface (a TAP or
6+// a Docker bridge) and then Attaches it here.
7+package netsetup
8+
9+import (
10+ "context"
11+ "errors"
12+ "fmt"
13+ "net/netip"
14+ "os"
15+ "strings"
16+
17+ "bnema.dev/ermite-runner/internal/cmdrun"
18+ "bnema.dev/ermite-runner/internal/dnsfwd"
19+ "bnema.dev/ermite-runner/internal/netpolicy"
20+)
21+
22+// Plumbing is the host network surface a driver needs. Net implements it; tests
23+// of the drivers use a fake because binding gateway addresses and loading nft
24+// rules needs privileges.
25+type Plumbing interface {
26+ EnableForwarding() error
27+ Attach(ctx context.Context, g netpolicy.Guest) error
28+ Detach(ctx context.Context, iface string, gateway netip.Addr) error
29+ Reset(ctx context.Context) error
30+ Close()
31+}
32+
33+var _ Plumbing = (*Net)(nil)
34+
35+// Net is the host network state of one driver.
36+type Net struct {
37+ Policy *netpolicy.Manager
38+ DNS *dnsfwd.Forwarder
39+ // ForwardFile is the ip_forward sysctl; overridden in tests.
40+ ForwardFile string
41+}
42+
43+// New builds a Net: nft runs through run, DNS listens on dnsPort and forwards
44+// to the resolvers of resolvConf ("" = /etc/resolv.conf).
45+func New(run cmdrun.Runner, dnsPort int, resolvConf string) *Net {
46+ r := dnsfwd.NewResolvers(resolvConf)
47+ return &Net{
48+ Policy: netpolicy.NewManager(run, "nft"),
49+ DNS: dnsfwd.New(r.Addrs, dnsPort),
50+ ForwardFile: "/proc/sys/net/ipv4/ip_forward",
51+ }
52+}
53+
54+// EnableForwarding turns on IPv4 forwarding (NAT needs it). It is a no-op when
55+// already on, and an error that names the fix when it cannot be turned on.
56+func (n *Net) EnableForwarding() error {
57+ b, err := os.ReadFile(n.ForwardFile)
58+ if err == nil && strings.TrimSpace(string(b)) == "1" {
59+ return nil
60+ }
61+ if err := os.WriteFile(n.ForwardFile, []byte("1\n"), 0o644); err != nil {
62+ return fmt.Errorf("enable IPv4 forwarding (%s): %w; set net.ipv4.ip_forward=1", n.ForwardFile, err)
63+ }
64+ return nil
65+}
66+
67+// Attach installs the firewall policy of g and starts the DNS forwarder on its
68+// gateway address, which must already be configured on the host interface.
69+// On error nothing stays installed.
70+func (n *Net) Attach(ctx context.Context, g netpolicy.Guest) error {
71+ if err := n.Policy.Add(ctx, g); err != nil {
72+ return err
73+ }
74+ if err := n.DNS.Listen(ctx, g.Gateway); err != nil {
75+ return errors.Join(err, n.Policy.Remove(context.WithoutCancel(ctx), g.Iface))
76+ }
77+ return nil
78+}
79+
80+// Detach undoes Attach. It is idempotent and keeps going after an error.
81+func (n *Net) Detach(ctx context.Context, iface string, gateway netip.Addr) error {
82+ n.DNS.Close(gateway)
83+ return n.Policy.Remove(ctx, iface)
84+}
85+
86+// Reset removes the runner's nftables table: used at startup to reap a previous
87+// run. Only the table is touched, never other tables.
88+func (n *Net) Reset(ctx context.Context) error { return n.Policy.Reset(ctx) }
89+
90+// Close stops every DNS listener.
91+func (n *Net) Close() { n.DNS.CloseAll() }
@@ -0,0 +1,91 @@
1+// SPDX-License-Identifier: MIT
2+
3+// Package netsetup is the host-side network plumbing shared by both sandbox
4+// drivers: the nftables policy (package netpolicy), the DNS forwarder (package
5+// dnsfwd) and IPv4 forwarding. Each driver creates its own interface (a TAP or
6+// a Docker bridge) and then Attaches it here.
7+package netsetup
8+
9+import (
10+ "context"
11+ "errors"
12+ "fmt"
13+ "net/netip"
14+ "os"
15+ "strings"
16+
17+ "bnema.dev/ermite-runner/internal/cmdrun"
18+ "bnema.dev/ermite-runner/internal/dnsfwd"
19+ "bnema.dev/ermite-runner/internal/netpolicy"
20+)
21+
22+// Plumbing is the host network surface a driver needs. Net implements it; tests
23+// of the drivers use a fake because binding gateway addresses and loading nft
24+// rules needs privileges.
25+type Plumbing interface {
26+ EnableForwarding() error
27+ Attach(ctx context.Context, g netpolicy.Guest) error
28+ Detach(ctx context.Context, iface string, gateway netip.Addr) error
29+ Reset(ctx context.Context) error
30+ Close()
31+}
32+
33+var _ Plumbing = (*Net)(nil)
34+
35+// Net is the host network state of one driver.
36+type Net struct {
37+ Policy *netpolicy.Manager
38+ DNS *dnsfwd.Forwarder
39+ // ForwardFile is the ip_forward sysctl; overridden in tests.
40+ ForwardFile string
41+}
42+
43+// New builds a Net: nft runs through run, DNS listens on dnsPort and forwards
44+// to the resolvers of resolvConf ("" = /etc/resolv.conf).
45+func New(run cmdrun.Runner, dnsPort int, resolvConf string) *Net {
46+ r := dnsfwd.NewResolvers(resolvConf)
47+ return &Net{
48+ Policy: netpolicy.NewManager(run, "nft"),
49+ DNS: dnsfwd.New(r.Addrs, dnsPort),
50+ ForwardFile: "/proc/sys/net/ipv4/ip_forward",
51+ }
52+}
53+
54+// EnableForwarding turns on IPv4 forwarding (NAT needs it). It is a no-op when
55+// already on, and an error that names the fix when it cannot be turned on.
56+func (n *Net) EnableForwarding() error {
57+ b, err := os.ReadFile(n.ForwardFile)
58+ if err == nil && strings.TrimSpace(string(b)) == "1" {
59+ return nil
60+ }
61+ if err := os.WriteFile(n.ForwardFile, []byte("1\n"), 0o644); err != nil {
62+ return fmt.Errorf("enable IPv4 forwarding (%s): %w; set net.ipv4.ip_forward=1", n.ForwardFile, err)
63+ }
64+ return nil
65+}
66+
67+// Attach installs the firewall policy of g and starts the DNS forwarder on its
68+// gateway address, which must already be configured on the host interface.
69+// On error nothing stays installed.
70+func (n *Net) Attach(ctx context.Context, g netpolicy.Guest) error {
71+ if err := n.Policy.Add(ctx, g); err != nil {
72+ return err
73+ }
74+ if err := n.DNS.Listen(ctx, g.Gateway); err != nil {
75+ return errors.Join(err, n.Policy.Remove(context.WithoutCancel(ctx), g.Iface))
76+ }
77+ return nil
78+}
79+
80+// Detach undoes Attach. It is idempotent and keeps going after an error.
81+func (n *Net) Detach(ctx context.Context, iface string, gateway netip.Addr) error {
82+ n.DNS.Close(gateway)
83+ return n.Policy.Remove(ctx, iface)
84+}
85+
86+// Reset removes the runner's nftables table: used at startup to reap a previous
87+// run. Only the table is touched, never other tables.
88+func (n *Net) Reset(ctx context.Context) error { return n.Policy.Reset(ctx) }
89+
90+// Close stops every DNS listener.
91+func (n *Net) Close() { n.DNS.CloseAll() }
A internal/sandbox/netsetup/netsetup_test.go +89 -0
@@ -0,0 +1,89 @@
1+// SPDX-License-Identifier: MIT
2+
3+package netsetup
4+
5+import (
6+ "context"
7+ "errors"
8+ "net/netip"
9+ "os"
10+ "path/filepath"
11+ "strings"
12+ "testing"
13+
14+ "bnema.dev/ermite-runner/internal/cmdrun"
15+ "bnema.dev/ermite-runner/internal/netpolicy"
16+)
17+
18+func guest() netpolicy.Guest {
19+ return netpolicy.Guest{Iface: "ertap0", Gateway: netip.MustParseAddr("127.0.0.1"), Addr: netip.MustParseAddr("10.200.0.2")}
20+}
21+
22+func TestAttachDetach(t *testing.T) {
23+ f := &cmdrun.Fake{}
24+ n := New(f, 0, "")
25+ t.Cleanup(n.Close)
26+ ctx := context.Background()
27+ if err := n.Attach(ctx, guest()); err != nil {
28+ t.Skip("cannot bind loopback:", err)
29+ }
30+ if _, ok := n.DNS.Addr(guest().Gateway); !ok {
31+ t.Fatal("DNS forwarder not listening on the gateway")
32+ }
33+ if err := n.Detach(ctx, "ertap0", guest().Gateway); err != nil {
34+ t.Fatal(err)
35+ }
36+ if _, ok := n.DNS.Addr(guest().Gateway); ok {
37+ t.Fatal("DNS forwarder still listening after Detach")
38+ }
39+ if len(f.Calls) != 2 || f.Calls[1].Stdin != netpolicy.Ruleset(nil) {
40+ t.Fatalf("calls: %v", f.Lines())
41+ }
42+ if err := n.Detach(ctx, "ertap0", guest().Gateway); err != nil {
43+ t.Fatalf("Detach must be idempotent: %v", err)
44+ }
45+}
46+
47+func TestAttachRollsBackPolicyWhenDNSFails(t *testing.T) {
48+ f := &cmdrun.Fake{}
49+ n := New(f, 0, "")
50+ t.Cleanup(n.Close)
51+ ctx := context.Background()
52+ if err := n.Attach(ctx, guest()); err != nil {
53+ t.Skip(err)
54+ }
55+ before := len(f.Calls)
56+ if err := n.Attach(ctx, guest()); err == nil { // already listening
57+ t.Fatal("second Attach on the same gateway must fail")
58+ }
59+ if len(f.Calls) != before+2 {
60+ t.Fatalf("expected add then rollback remove, got %v", f.Lines())
61+ }
62+}
63+
64+func TestAttachPolicyError(t *testing.T) {
65+ boom := errors.New("nft failed")
66+ n := New(&cmdrun.Fake{Handle: func(cmdrun.Call) ([]byte, error) { return nil, boom }}, 0, "")
67+ if err := n.Attach(context.Background(), guest()); !errors.Is(err, boom) {
68+ t.Fatalf("got %v", err)
69+ }
70+ if _, ok := n.DNS.Addr(guest().Gateway); ok {
71+ t.Fatal("DNS must not start when the policy failed")
72+ }
73+}
74+
75+func TestEnableForwarding(t *testing.T) {
76+ n := New(&cmdrun.Fake{}, 0, "")
77+ n.ForwardFile = filepath.Join(t.TempDir(), "ip_forward")
78+ _ = os.WriteFile(n.ForwardFile, []byte("0\n"), 0o644)
79+ if err := n.EnableForwarding(); err != nil {
80+ t.Fatal(err)
81+ }
82+ if b, _ := os.ReadFile(n.ForwardFile); strings.TrimSpace(string(b)) != "1" {
83+ t.Fatalf("not enabled: %q", b)
84+ }
85+ n.ForwardFile = filepath.Join(t.TempDir(), "missing", "ip_forward")
86+ if err := n.EnableForwarding(); err == nil || !strings.Contains(err.Error(), "ip_forward") {
87+ t.Fatalf("want a clear error, got %v", err)
88+ }
89+}
@@ -0,0 +1,89 @@
1+// SPDX-License-Identifier: MIT
2+
3+package netsetup
4+
5+import (
6+ "context"
7+ "errors"
8+ "net/netip"
9+ "os"
10+ "path/filepath"
11+ "strings"
12+ "testing"
13+
14+ "bnema.dev/ermite-runner/internal/cmdrun"
15+ "bnema.dev/ermite-runner/internal/netpolicy"
16+)
17+
18+func guest() netpolicy.Guest {
19+ return netpolicy.Guest{Iface: "ertap0", Gateway: netip.MustParseAddr("127.0.0.1"), Addr: netip.MustParseAddr("10.200.0.2")}
20+}
21+
22+func TestAttachDetach(t *testing.T) {
23+ f := &cmdrun.Fake{}
24+ n := New(f, 0, "")
25+ t.Cleanup(n.Close)
26+ ctx := context.Background()
27+ if err := n.Attach(ctx, guest()); err != nil {
28+ t.Skip("cannot bind loopback:", err)
29+ }
30+ if _, ok := n.DNS.Addr(guest().Gateway); !ok {
31+ t.Fatal("DNS forwarder not listening on the gateway")
32+ }
33+ if err := n.Detach(ctx, "ertap0", guest().Gateway); err != nil {
34+ t.Fatal(err)
35+ }
36+ if _, ok := n.DNS.Addr(guest().Gateway); ok {
37+ t.Fatal("DNS forwarder still listening after Detach")
38+ }
39+ if len(f.Calls) != 2 || f.Calls[1].Stdin != netpolicy.Ruleset(nil) {
40+ t.Fatalf("calls: %v", f.Lines())
41+ }
42+ if err := n.Detach(ctx, "ertap0", guest().Gateway); err != nil {
43+ t.Fatalf("Detach must be idempotent: %v", err)
44+ }
45+}
46+
47+func TestAttachRollsBackPolicyWhenDNSFails(t *testing.T) {
48+ f := &cmdrun.Fake{}
49+ n := New(f, 0, "")
50+ t.Cleanup(n.Close)
51+ ctx := context.Background()
52+ if err := n.Attach(ctx, guest()); err != nil {
53+ t.Skip(err)
54+ }
55+ before := len(f.Calls)
56+ if err := n.Attach(ctx, guest()); err == nil { // already listening
57+ t.Fatal("second Attach on the same gateway must fail")
58+ }
59+ if len(f.Calls) != before+2 {
60+ t.Fatalf("expected add then rollback remove, got %v", f.Lines())
61+ }
62+}
63+
64+func TestAttachPolicyError(t *testing.T) {
65+ boom := errors.New("nft failed")
66+ n := New(&cmdrun.Fake{Handle: func(cmdrun.Call) ([]byte, error) { return nil, boom }}, 0, "")
67+ if err := n.Attach(context.Background(), guest()); !errors.Is(err, boom) {
68+ t.Fatalf("got %v", err)
69+ }
70+ if _, ok := n.DNS.Addr(guest().Gateway); ok {
71+ t.Fatal("DNS must not start when the policy failed")
72+ }
73+}
74+
75+func TestEnableForwarding(t *testing.T) {
76+ n := New(&cmdrun.Fake{}, 0, "")
77+ n.ForwardFile = filepath.Join(t.TempDir(), "ip_forward")
78+ _ = os.WriteFile(n.ForwardFile, []byte("0\n"), 0o644)
79+ if err := n.EnableForwarding(); err != nil {
80+ t.Fatal(err)
81+ }
82+ if b, _ := os.ReadFile(n.ForwardFile); strings.TrimSpace(string(b)) != "1" {
83+ t.Fatalf("not enabled: %q", b)
84+ }
85+ n.ForwardFile = filepath.Join(t.TempDir(), "missing", "ip_forward")
86+ if err := n.EnableForwarding(); err == nil || !strings.Contains(err.Error(), "ip_forward") {
87+ t.Fatalf("want a clear error, got %v", err)
88+ }
89+}
A internal/sandbox/sandbox.go +110 -0
@@ -0,0 +1,110 @@
1+// SPDX-License-Identifier: MIT
2+
3+// Package sandbox defines the isolation boundary of a CI job: one Sandbox per
4+// job, either a Firecracker micro-VM (package vm) or a container with its own
5+// Docker daemon (package container). Both run the same executable
6+// (`ermite-runner init`, then `ermite-runner exec`) and expose the same framed
7+// stream (package execstream) to the host process.
8+package sandbox
9+
10+import (
11+ "context"
12+ "errors"
13+ "io"
14+)
15+
16+// Mode is an isolation mode.
17+type Mode string
18+
19+// Isolation modes. Auto is a setting only: a Driver always reports VM or
20+// Container.
21+const (
22+ Auto Mode = "auto"
23+ VM Mode = "vm"
24+ Container Mode = "container"
25+)
26+
27+// ErrCapacity is returned by Driver.New when every slot is in use.
28+var ErrCapacity = errors.New("sandbox: no free slot")
29+
30+// JobSpec describes the sandbox of one job. Zero resource fields take the
31+// configured defaults.
32+type JobSpec struct {
33+ // JobID identifies the job in logs, labels and leftover reports. It never
34+ // appears in file or interface names.
35+ JobID string
36+ // CPUs, MemoryMiB and DiskMiB size the sandbox (ERMITE_RUNNER_JOB_*).
37+ CPUs int
38+ MemoryMiB int
39+ DiskMiB int
40+}
41+
42+// Conn is the executor stream: bytes in both directions, framed by execstream.
43+// Closing it ends the session from the host side.
44+type Conn io.ReadWriteCloser
45+
46+// Sandbox isolates one job. A Sandbox is single use: Start once, then Stop.
47+type Sandbox interface {
48+ // Start creates the sandbox, boots it and waits for the executor inside to
49+ // connect back. On error everything created so far has been removed.
50+ Start(ctx context.Context, spec JobSpec) (Conn, error)
51+ // Stop destroys the sandbox and everything it created (processes, network
52+ // devices, firewall rules, files). It is idempotent, safe after a failed or
53+ // never-made Start, and runs to completion even when ctx is already done.
54+ Stop(ctx context.Context) error
55+}
56+
57+// Driver creates sandboxes of one mode and reaps what a previous run left.
58+type Driver interface {
59+ // Mode is VM or Container.
60+ Mode() Mode
61+ // New returns a fresh Sandbox bound to a free slot (ErrCapacity if none).
62+ // The slot is released by Stop.
63+ New() (Sandbox, error)
64+ // Reap removes everything a previous run of the runner left behind
65+ // (leftover processes, devices, firewall rules, containers, files). It must
66+ // be called once at startup, before any New.
67+ Reap(ctx context.Context) error
68+ // Close releases the driver's own resources (DNS forwarder, API client).
69+ // Sandboxes still running are not stopped.
70+ Close() error
71+}
72+
73+// Slots hands out small integers, lowest first. Slots identify a sandbox's uid,
74+// network device and /30 subnet, so two live sandboxes never collide.
75+type Slots struct {
76+ used []bool
77+}
78+
79+// NewSlots returns a pool of n slots.
80+func NewSlots(n int) *Slots { return &Slots{used: make([]bool, n)} }
81+
82+// Acquire returns the lowest free slot, or ErrCapacity. Callers serialise
83+// access (the drivers hold their mutex).
84+func (s *Slots) Acquire() (int, error) {
85+ for i, u := range s.used {
86+ if !u {
87+ s.used[i] = true
88+ return i, nil
89+ }
90+ }
91+ return 0, ErrCapacity
92+}
93+
94+// Release frees a slot; releasing a free or unknown slot is a no-op.
95+func (s *Slots) Release(i int) {
96+ if i >= 0 && i < len(s.used) {
97+ s.used[i] = false
98+ }
99+}
100+
101+// InUse reports how many slots are taken.
102+func (s *Slots) InUse() int {
103+ n := 0
104+ for _, u := range s.used {
105+ if u {
106+ n++
107+ }
108+ }
109+ return n
110+}
@@ -0,0 +1,110 @@
1+// SPDX-License-Identifier: MIT
2+
3+// Package sandbox defines the isolation boundary of a CI job: one Sandbox per
4+// job, either a Firecracker micro-VM (package vm) or a container with its own
5+// Docker daemon (package container). Both run the same executable
6+// (`ermite-runner init`, then `ermite-runner exec`) and expose the same framed
7+// stream (package execstream) to the host process.
8+package sandbox
9+
10+import (
11+ "context"
12+ "errors"
13+ "io"
14+)
15+
16+// Mode is an isolation mode.
17+type Mode string
18+
19+// Isolation modes. Auto is a setting only: a Driver always reports VM or
20+// Container.
21+const (
22+ Auto Mode = "auto"
23+ VM Mode = "vm"
24+ Container Mode = "container"
25+)
26+
27+// ErrCapacity is returned by Driver.New when every slot is in use.
28+var ErrCapacity = errors.New("sandbox: no free slot")
29+
30+// JobSpec describes the sandbox of one job. Zero resource fields take the
31+// configured defaults.
32+type JobSpec struct {
33+ // JobID identifies the job in logs, labels and leftover reports. It never
34+ // appears in file or interface names.
35+ JobID string
36+ // CPUs, MemoryMiB and DiskMiB size the sandbox (ERMITE_RUNNER_JOB_*).
37+ CPUs int
38+ MemoryMiB int
39+ DiskMiB int
40+}
41+
42+// Conn is the executor stream: bytes in both directions, framed by execstream.
43+// Closing it ends the session from the host side.
44+type Conn io.ReadWriteCloser
45+
46+// Sandbox isolates one job. A Sandbox is single use: Start once, then Stop.
47+type Sandbox interface {
48+ // Start creates the sandbox, boots it and waits for the executor inside to
49+ // connect back. On error everything created so far has been removed.
50+ Start(ctx context.Context, spec JobSpec) (Conn, error)
51+ // Stop destroys the sandbox and everything it created (processes, network
52+ // devices, firewall rules, files). It is idempotent, safe after a failed or
53+ // never-made Start, and runs to completion even when ctx is already done.
54+ Stop(ctx context.Context) error
55+}
56+
57+// Driver creates sandboxes of one mode and reaps what a previous run left.
58+type Driver interface {
59+ // Mode is VM or Container.
60+ Mode() Mode
61+ // New returns a fresh Sandbox bound to a free slot (ErrCapacity if none).
62+ // The slot is released by Stop.
63+ New() (Sandbox, error)
64+ // Reap removes everything a previous run of the runner left behind
65+ // (leftover processes, devices, firewall rules, containers, files). It must
66+ // be called once at startup, before any New.
67+ Reap(ctx context.Context) error
68+ // Close releases the driver's own resources (DNS forwarder, API client).
69+ // Sandboxes still running are not stopped.
70+ Close() error
71+}
72+
73+// Slots hands out small integers, lowest first. Slots identify a sandbox's uid,
74+// network device and /30 subnet, so two live sandboxes never collide.
75+type Slots struct {
76+ used []bool
77+}
78+
79+// NewSlots returns a pool of n slots.
80+func NewSlots(n int) *Slots { return &Slots{used: make([]bool, n)} }
81+
82+// Acquire returns the lowest free slot, or ErrCapacity. Callers serialise
83+// access (the drivers hold their mutex).
84+func (s *Slots) Acquire() (int, error) {
85+ for i, u := range s.used {
86+ if !u {
87+ s.used[i] = true
88+ return i, nil
89+ }
90+ }
91+ return 0, ErrCapacity
92+}
93+
94+// Release frees a slot; releasing a free or unknown slot is a no-op.
95+func (s *Slots) Release(i int) {
96+ if i >= 0 && i < len(s.used) {
97+ s.used[i] = false
98+ }
99+}
100+
101+// InUse reports how many slots are taken.
102+func (s *Slots) InUse() int {
103+ n := 0
104+ for _, u := range s.used {
105+ if u {
106+ n++
107+ }
108+ }
109+ return n
110+}
A internal/sandbox/sandbox_test.go +270 -0
@@ -0,0 +1,270 @@
1+// SPDX-License-Identifier: MIT
2+
3+package sandbox
4+
5+import (
6+ "bytes"
7+ "context"
8+ "errors"
9+ "os"
10+ "path/filepath"
11+ "strings"
12+ "testing"
13+
14+ "github.com/bnema/zerowrap"
15+)
16+
17+func env(m map[string]string) func(string) string { return func(k string) string { return m[k] } }
18+
19+func TestLoadConfigDefaults(t *testing.T) {
20+ c, err := LoadConfig(env(nil))
21+ if err != nil {
22+ t.Fatal(err)
23+ }
24+ if c.Isolation != Auto || c.Capacity != 1 || c.UIDBase != 900000 || c.StateDir != DefaultStateDir ||
25+ c.JobCPUs != 2 || c.JobMemoryMiB != 4096 || c.JobDiskMiB != 64*1024 || c.Network.String() != "10.200.0.0/16" || c.DNSPort != 53 {
26+ t.Fatalf("%+v", c)
27+ }
28+}
29+
30+func TestLoadConfigValues(t *testing.T) {
31+ c, err := LoadConfig(env(map[string]string{
32+ EnvIsolation: "VM", EnvStateDir: "/srv/er", EnvCapacity: "4", EnvImage: "/images/u24",
33+ EnvJobCPUs: "8", EnvJobMemory: "16G", EnvJobDisk: "100G", EnvUIDBase: "910000",
34+ EnvNetwork: "10.50.0.0/24", EnvJailerMode: "off", EnvFirecracker: "/x/fc", EnvKernel: "/k",
35+ }))
36+ if err != nil {
37+ t.Fatal(err)
38+ }
39+ if c.Isolation != VM || c.Capacity != 4 || c.JobMemoryMiB != 16384 || c.JobDiskMiB != 100*1024 || c.UIDBase != 910000 ||
40+ !c.JailerOff || c.Network.String() != "10.50.0.0/24" || c.ImageDir() != "/images/u24" || c.KernelPath() != "/k" ||
41+ c.BaseImagePath() != "/images/u24/base.ext4" || c.OCIImage() != DefaultOCIImage {
42+ t.Fatalf("%+v", c)
43+ }
44+ c, _ = LoadConfig(env(map[string]string{EnvImage: "registry.example/ermite:1"}))
45+ if c.OCIImage() != "registry.example/ermite:1" || c.ImageDir() != filepath.Join(DefaultStateDir, "image") {
46+ t.Fatalf("OCI ref misread: %q %q", c.OCIImage(), c.ImageDir())
47+ }
48+}
49+
50+func TestLoadConfigErrors(t *testing.T) {
51+ for name, m := range map[string]map[string]string{
52+ "isolation": {EnvIsolation: "docker"},
53+ "relative": {EnvStateDir: "state"},
54+ "capacity 0": {EnvCapacity: "0"},
55+ "capacity abc": {EnvCapacity: "abc"},
56+ "cpus": {EnvJobCPUs: "0"},
57+ "memory": {EnvJobMemory: "10M"},
58+ "disk": {EnvJobDisk: "100M"},
59+ "uid base": {EnvUIDBase: "10"},
60+ "network v6": {EnvNetwork: "fd00::/64"},
61+ "network /31": {EnvNetwork: "10.0.0.0/31"},
62+ "network few": {EnvNetwork: "10.0.0.0/30", EnvCapacity: "2"},
63+ "jailer mode": {EnvJailerMode: "maybe"},
64+ } {
65+ if _, err := LoadConfig(env(m)); err == nil {
66+ t.Errorf("%s: accepted", name)
67+ }
68+ }
69+}
70+
71+func TestParseSizeMiB(t *testing.T) {
72+ for in, want := range map[string]int{"4096": 4096, "512M": 512, "4G": 4096, "64GiB": 65536, "1t": 1048576, " 2g ": 2048, "1024MB": 1024} {
73+ if got, err := ParseSizeMiB(in); err != nil || got != want {
74+ t.Errorf("%q: %d %v", in, got, err)
75+ }
76+ }
77+ for _, in := range []string{"", "0", "-1", "G", "12K", "1x", "99999999999T"} {
78+ if _, err := ParseSizeMiB(in); err == nil {
79+ t.Errorf("%q accepted", in)
80+ }
81+ }
82+}
83+
84+func TestResolve(t *testing.T) {
85+ c, _ := LoadConfig(env(nil))
86+ s := c.Resolve(JobSpec{JobID: "j", CPUs: 6})
87+ if s.CPUs != 6 || s.MemoryMiB != 4096 || s.DiskMiB != 64*1024 || s.JobID != "j" {
88+ t.Fatalf("%+v", s)
89+ }
90+}
91+
92+func TestSlots(t *testing.T) {
93+ s := NewSlots(2)
94+ a, _ := s.Acquire()
95+ b, _ := s.Acquire()
96+ if a != 0 || b != 1 || s.InUse() != 2 {
97+ t.Fatal(a, b)
98+ }
99+ if _, err := s.Acquire(); !errors.Is(err, ErrCapacity) {
100+ t.Fatal(err)
101+ }
102+ s.Release(0)
103+ s.Release(0)
104+ s.Release(99)
105+ if c, _ := s.Acquire(); c != 0 {
106+ t.Fatalf("lowest free slot first, got %d", c)
107+ }
108+}
109+
110+func TestIDs(t *testing.T) {
111+ id, err := NewID()
112+ if err != nil || !ValidID(id) {
113+ t.Fatal(id, err)
114+ }
115+ for _, bad := range []string{"", "er", "er12345", "xx12345678", "er1234567g", "er../../etc", "er123456789"} {
116+ if ValidID(bad) {
117+ t.Errorf("%q valid", bad)
118+ }
119+ }
120+}
121+
122+// fakeDriver is a Driver that records its construction.
123+type fakeDriver struct{ mode Mode }
124+
125+func (f fakeDriver) Mode() Mode { return f.mode }
126+func (fakeDriver) New() (Sandbox, error) { return nil, nil }
127+func (fakeDriver) Reap(context.Context) error { return nil }
128+func (fakeDriver) Close() error { return nil }
129+
130+func facts(built *[]Mode) Factories {
131+ return Factories{
132+ VM: func(context.Context, Config) (Driver, error) { *built = append(*built, VM); return fakeDriver{VM}, nil },
133+ Container: func(context.Context, Config) (Driver, error) {
134+ *built = append(*built, Container)
135+ return fakeDriver{Container}, nil
136+ },
137+ }
138+}
139+
140+// goodProber reports a host where every vm prerequisite exists.
141+func goodProber() Prober {
142+ return Prober{
143+ OpenKVM: func(string) error { return nil },
144+ LookPath: func(n string) (string, error) { return "/usr/bin/" + n, nil },
145+ Stat: func(string) error { return nil },
146+ }
147+}
148+
149+func logCtx(buf *bytes.Buffer) context.Context {
150+ return zerowrap.WithCtx(context.Background(), zerowrap.New(zerowrap.Config{Level: "debug", Format: "json", Output: buf}))
151+}
152+
153+func TestSelectAutoPicksVMWhenEverythingIsThere(t *testing.T) {
154+ var built []Mode
155+ var log bytes.Buffer
156+ cfg, _ := LoadConfig(env(nil))
157+ d, err := Select(logCtx(&log), cfg, facts(&built), goodProber())
158+ if err != nil || d.Mode() != VM || len(built) != 1 {
159+ t.Fatalf("%v %v %v", d, err, built)
160+ }
161+ if !strings.Contains(log.String(), `"mode":"vm"`) || !strings.Contains(log.String(), "reason") {
162+ t.Fatalf("mode and reason must be logged: %s", log.String())
163+ }
164+}
165+
166+// AC4: auto picks container when /dev/kvm is hidden, and says why.
167+func TestSelectAutoFallsBackWithoutKVM(t *testing.T) {
168+ var built []Mode
169+ var log bytes.Buffer
170+ cfg, _ := LoadConfig(env(nil))
171+ cfg.KVMPath = filepath.Join(t.TempDir(), "kvm") // hidden: does not exist
172+ p := goodProber()
173+ p.OpenKVM = Prober{}.withDefaults().OpenKVM // the real open, on the hidden path
174+ d, err := Select(logCtx(&log), cfg, facts(&built), p)
175+ if err != nil || d.Mode() != Container {
176+ t.Fatalf("%v %v", d, err)
177+ }
178+ out := log.String()
179+ if !strings.Contains(out, `"mode":"container"`) || !strings.Contains(out, "not usable read-write") {
180+ t.Fatalf("fallback reason not logged: %s", out)
181+ }
182+}
183+
184+func TestSelectAutoFallbackReasons(t *testing.T) {
185+ cfg, _ := LoadConfig(env(nil))
186+ cases := map[string]Prober{
187+ "firecracker not found": {OpenKVM: func(string) error { return nil }, LookPath: func(n string) (string, error) {
188+ if n == "firecracker" {
189+ return "", os.ErrNotExist
190+ }
191+ return "/x", nil
192+ }, Stat: func(string) error { return nil }},
193+ "jailer not found": {OpenKVM: func(string) error { return nil }, LookPath: func(n string) (string, error) {
194+ if n == "jailer" {
195+ return "", os.ErrNotExist
196+ }
197+ return "/x", nil
198+ }, Stat: func(string) error { return nil }},
199+ "guest kernel": {OpenKVM: func(string) error { return nil }, LookPath: goodProber().LookPath, Stat: func(p string) error {
200+ if strings.HasSuffix(p, "vmlinux") {
201+ return os.ErrNotExist
202+ }
203+ return nil
204+ }},
205+ "base image": {OpenKVM: func(string) error { return nil }, LookPath: goodProber().LookPath, Stat: func(p string) error {
206+ if strings.HasSuffix(p, "base.ext4") {
207+ return os.ErrNotExist
208+ }
209+ return nil
210+ }},
211+ }
212+ for want, p := range cases {
213+ var built []Mode
214+ d, err := Select(context.Background(), cfg, facts(&built), p)
215+ if err != nil || d.Mode() != Container {
216+ t.Errorf("%s: %v %v", want, d, err)
217+ }
218+ if why := VMUnavailable(cfg, p); len(why) != 1 || !strings.Contains(why[0], want) {
219+ t.Errorf("%s: reasons %v", want, why)
220+ }
221+ }
222+}
223+
224+func TestSelectExplicitModes(t *testing.T) {
225+ cfg, _ := LoadConfig(env(map[string]string{EnvIsolation: "vm"}))
226+ var built []Mode
227+ bad := Prober{OpenKVM: func(string) error { return os.ErrPermission }, LookPath: goodProber().LookPath, Stat: goodProber().Stat}
228+ if _, err := Select(context.Background(), cfg, facts(&built), bad); err == nil || !strings.Contains(err.Error(), "kvm") || len(built) != 0 {
229+ t.Fatalf("vm must fail loudly without KVM and build nothing: %v %v", err, built)
230+ }
231+ cfg.Isolation = Container
232+ d, err := Select(context.Background(), cfg, facts(&built), bad)
233+ if err != nil || d.Mode() != Container {
234+ t.Fatalf("container never probes KVM: %v %v", d, err)
235+ }
236+ cfg.Isolation = Mode("x")
237+ if _, err := Select(context.Background(), cfg, facts(&built), bad); err == nil {
238+ t.Fatal("unknown mode accepted")
239+ }
240+}
241+
242+func TestSelectDoesNotMaskBrokenVM(t *testing.T) {
243+ cfg, _ := LoadConfig(env(nil))
244+ boom := errors.New("kernel does not boot")
245+ f := Factories{
246+ VM: func(context.Context, Config) (Driver, error) { return nil, boom },
247+ Container: func(context.Context, Config) (Driver, error) { t.Fatal("must not fall back silently"); return nil, nil },
248+ }
249+ if _, err := Select(context.Background(), cfg, f, goodProber()); !errors.Is(err, boom) {
250+ t.Fatalf("got %v", err)
251+ }
252+ if _, err := Select(context.Background(), cfg, Factories{}, goodProber()); err == nil {
253+ t.Fatal("missing factory accepted")
254+ }
255+}
256+
257+func TestCheckStatic(t *testing.T) {
258+ dir := t.TempDir()
259+ if err := CheckStatic(filepath.Join(dir, "missing")); err == nil {
260+ t.Fatal("missing file accepted")
261+ }
262+ txt := filepath.Join(dir, "script")
263+ _ = os.WriteFile(txt, []byte("#!/bin/sh\n"), 0o755)
264+ if err := CheckStatic(txt); err == nil || !strings.Contains(err.Error(), "CGO_ENABLED=0") {
265+ t.Fatalf("script accepted: %v", err)
266+ }
267+ if err := CheckStatic("/bin/sh"); err == nil {
268+ t.Skip("/bin/sh is static here")
269+ }
270+}
@@ -0,0 +1,270 @@
1+// SPDX-License-Identifier: MIT
2+
3+package sandbox
4+
5+import (
6+ "bytes"
7+ "context"
8+ "errors"
9+ "os"
10+ "path/filepath"
11+ "strings"
12+ "testing"
13+
14+ "github.com/bnema/zerowrap"
15+)
16+
17+func env(m map[string]string) func(string) string { return func(k string) string { return m[k] } }
18+
19+func TestLoadConfigDefaults(t *testing.T) {
20+ c, err := LoadConfig(env(nil))
21+ if err != nil {
22+ t.Fatal(err)
23+ }
24+ if c.Isolation != Auto || c.Capacity != 1 || c.UIDBase != 900000 || c.StateDir != DefaultStateDir ||
25+ c.JobCPUs != 2 || c.JobMemoryMiB != 4096 || c.JobDiskMiB != 64*1024 || c.Network.String() != "10.200.0.0/16" || c.DNSPort != 53 {
26+ t.Fatalf("%+v", c)
27+ }
28+}
29+
30+func TestLoadConfigValues(t *testing.T) {
31+ c, err := LoadConfig(env(map[string]string{
32+ EnvIsolation: "VM", EnvStateDir: "/srv/er", EnvCapacity: "4", EnvImage: "/images/u24",
33+ EnvJobCPUs: "8", EnvJobMemory: "16G", EnvJobDisk: "100G", EnvUIDBase: "910000",
34+ EnvNetwork: "10.50.0.0/24", EnvJailerMode: "off", EnvFirecracker: "/x/fc", EnvKernel: "/k",
35+ }))
36+ if err != nil {
37+ t.Fatal(err)
38+ }
39+ if c.Isolation != VM || c.Capacity != 4 || c.JobMemoryMiB != 16384 || c.JobDiskMiB != 100*1024 || c.UIDBase != 910000 ||
40+ !c.JailerOff || c.Network.String() != "10.50.0.0/24" || c.ImageDir() != "/images/u24" || c.KernelPath() != "/k" ||
41+ c.BaseImagePath() != "/images/u24/base.ext4" || c.OCIImage() != DefaultOCIImage {
42+ t.Fatalf("%+v", c)
43+ }
44+ c, _ = LoadConfig(env(map[string]string{EnvImage: "registry.example/ermite:1"}))
45+ if c.OCIImage() != "registry.example/ermite:1" || c.ImageDir() != filepath.Join(DefaultStateDir, "image") {
46+ t.Fatalf("OCI ref misread: %q %q", c.OCIImage(), c.ImageDir())
47+ }
48+}
49+
50+func TestLoadConfigErrors(t *testing.T) {
51+ for name, m := range map[string]map[string]string{
52+ "isolation": {EnvIsolation: "docker"},
53+ "relative": {EnvStateDir: "state"},
54+ "capacity 0": {EnvCapacity: "0"},
55+ "capacity abc": {EnvCapacity: "abc"},
56+ "cpus": {EnvJobCPUs: "0"},
57+ "memory": {EnvJobMemory: "10M"},
58+ "disk": {EnvJobDisk: "100M"},
59+ "uid base": {EnvUIDBase: "10"},
60+ "network v6": {EnvNetwork: "fd00::/64"},
61+ "network /31": {EnvNetwork: "10.0.0.0/31"},
62+ "network few": {EnvNetwork: "10.0.0.0/30", EnvCapacity: "2"},
63+ "jailer mode": {EnvJailerMode: "maybe"},
64+ } {
65+ if _, err := LoadConfig(env(m)); err == nil {
66+ t.Errorf("%s: accepted", name)
67+ }
68+ }
69+}
70+
71+func TestParseSizeMiB(t *testing.T) {
72+ for in, want := range map[string]int{"4096": 4096, "512M": 512, "4G": 4096, "64GiB": 65536, "1t": 1048576, " 2g ": 2048, "1024MB": 1024} {
73+ if got, err := ParseSizeMiB(in); err != nil || got != want {
74+ t.Errorf("%q: %d %v", in, got, err)
75+ }
76+ }
77+ for _, in := range []string{"", "0", "-1", "G", "12K", "1x", "99999999999T"} {
78+ if _, err := ParseSizeMiB(in); err == nil {
79+ t.Errorf("%q accepted", in)
80+ }
81+ }
82+}
83+
84+func TestResolve(t *testing.T) {
85+ c, _ := LoadConfig(env(nil))
86+ s := c.Resolve(JobSpec{JobID: "j", CPUs: 6})
87+ if s.CPUs != 6 || s.MemoryMiB != 4096 || s.DiskMiB != 64*1024 || s.JobID != "j" {
88+ t.Fatalf("%+v", s)
89+ }
90+}
91+
92+func TestSlots(t *testing.T) {
93+ s := NewSlots(2)
94+ a, _ := s.Acquire()
95+ b, _ := s.Acquire()
96+ if a != 0 || b != 1 || s.InUse() != 2 {
97+ t.Fatal(a, b)
98+ }
99+ if _, err := s.Acquire(); !errors.Is(err, ErrCapacity) {
100+ t.Fatal(err)
101+ }
102+ s.Release(0)
103+ s.Release(0)
104+ s.Release(99)
105+ if c, _ := s.Acquire(); c != 0 {
106+ t.Fatalf("lowest free slot first, got %d", c)
107+ }
108+}
109+
110+func TestIDs(t *testing.T) {
111+ id, err := NewID()
112+ if err != nil || !ValidID(id) {
113+ t.Fatal(id, err)
114+ }
115+ for _, bad := range []string{"", "er", "er12345", "xx12345678", "er1234567g", "er../../etc", "er123456789"} {
116+ if ValidID(bad) {
117+ t.Errorf("%q valid", bad)
118+ }
119+ }
120+}
121+
122+// fakeDriver is a Driver that records its construction.
123+type fakeDriver struct{ mode Mode }
124+
125+func (f fakeDriver) Mode() Mode { return f.mode }
126+func (fakeDriver) New() (Sandbox, error) { return nil, nil }
127+func (fakeDriver) Reap(context.Context) error { return nil }
128+func (fakeDriver) Close() error { return nil }
129+
130+func facts(built *[]Mode) Factories {
131+ return Factories{
132+ VM: func(context.Context, Config) (Driver, error) { *built = append(*built, VM); return fakeDriver{VM}, nil },
133+ Container: func(context.Context, Config) (Driver, error) {
134+ *built = append(*built, Container)
135+ return fakeDriver{Container}, nil
136+ },
137+ }
138+}
139+
140+// goodProber reports a host where every vm prerequisite exists.
141+func goodProber() Prober {
142+ return Prober{
143+ OpenKVM: func(string) error { return nil },
144+ LookPath: func(n string) (string, error) { return "/usr/bin/" + n, nil },
145+ Stat: func(string) error { return nil },
146+ }
147+}
148+
149+func logCtx(buf *bytes.Buffer) context.Context {
150+ return zerowrap.WithCtx(context.Background(), zerowrap.New(zerowrap.Config{Level: "debug", Format: "json", Output: buf}))
151+}
152+
153+func TestSelectAutoPicksVMWhenEverythingIsThere(t *testing.T) {
154+ var built []Mode
155+ var log bytes.Buffer
156+ cfg, _ := LoadConfig(env(nil))
157+ d, err := Select(logCtx(&log), cfg, facts(&built), goodProber())
158+ if err != nil || d.Mode() != VM || len(built) != 1 {
159+ t.Fatalf("%v %v %v", d, err, built)
160+ }
161+ if !strings.Contains(log.String(), `"mode":"vm"`) || !strings.Contains(log.String(), "reason") {
162+ t.Fatalf("mode and reason must be logged: %s", log.String())
163+ }
164+}
165+
166+// AC4: auto picks container when /dev/kvm is hidden, and says why.
167+func TestSelectAutoFallsBackWithoutKVM(t *testing.T) {
168+ var built []Mode
169+ var log bytes.Buffer
170+ cfg, _ := LoadConfig(env(nil))
171+ cfg.KVMPath = filepath.Join(t.TempDir(), "kvm") // hidden: does not exist
172+ p := goodProber()
173+ p.OpenKVM = Prober{}.withDefaults().OpenKVM // the real open, on the hidden path
174+ d, err := Select(logCtx(&log), cfg, facts(&built), p)
175+ if err != nil || d.Mode() != Container {
176+ t.Fatalf("%v %v", d, err)
177+ }
178+ out := log.String()
179+ if !strings.Contains(out, `"mode":"container"`) || !strings.Contains(out, "not usable read-write") {
180+ t.Fatalf("fallback reason not logged: %s", out)
181+ }
182+}
183+
184+func TestSelectAutoFallbackReasons(t *testing.T) {
185+ cfg, _ := LoadConfig(env(nil))
186+ cases := map[string]Prober{
187+ "firecracker not found": {OpenKVM: func(string) error { return nil }, LookPath: func(n string) (string, error) {
188+ if n == "firecracker" {
189+ return "", os.ErrNotExist
190+ }
191+ return "/x", nil
192+ }, Stat: func(string) error { return nil }},
193+ "jailer not found": {OpenKVM: func(string) error { return nil }, LookPath: func(n string) (string, error) {
194+ if n == "jailer" {
195+ return "", os.ErrNotExist
196+ }
197+ return "/x", nil
198+ }, Stat: func(string) error { return nil }},
199+ "guest kernel": {OpenKVM: func(string) error { return nil }, LookPath: goodProber().LookPath, Stat: func(p string) error {
200+ if strings.HasSuffix(p, "vmlinux") {
201+ return os.ErrNotExist
202+ }
203+ return nil
204+ }},
205+ "base image": {OpenKVM: func(string) error { return nil }, LookPath: goodProber().LookPath, Stat: func(p string) error {
206+ if strings.HasSuffix(p, "base.ext4") {
207+ return os.ErrNotExist
208+ }
209+ return nil
210+ }},
211+ }
212+ for want, p := range cases {
213+ var built []Mode
214+ d, err := Select(context.Background(), cfg, facts(&built), p)
215+ if err != nil || d.Mode() != Container {
216+ t.Errorf("%s: %v %v", want, d, err)
217+ }
218+ if why := VMUnavailable(cfg, p); len(why) != 1 || !strings.Contains(why[0], want) {
219+ t.Errorf("%s: reasons %v", want, why)
220+ }
221+ }
222+}
223+
224+func TestSelectExplicitModes(t *testing.T) {
225+ cfg, _ := LoadConfig(env(map[string]string{EnvIsolation: "vm"}))
226+ var built []Mode
227+ bad := Prober{OpenKVM: func(string) error { return os.ErrPermission }, LookPath: goodProber().LookPath, Stat: goodProber().Stat}
228+ if _, err := Select(context.Background(), cfg, facts(&built), bad); err == nil || !strings.Contains(err.Error(), "kvm") || len(built) != 0 {
229+ t.Fatalf("vm must fail loudly without KVM and build nothing: %v %v", err, built)
230+ }
231+ cfg.Isolation = Container
232+ d, err := Select(context.Background(), cfg, facts(&built), bad)
233+ if err != nil || d.Mode() != Container {
234+ t.Fatalf("container never probes KVM: %v %v", d, err)
235+ }
236+ cfg.Isolation = Mode("x")
237+ if _, err := Select(context.Background(), cfg, facts(&built), bad); err == nil {
238+ t.Fatal("unknown mode accepted")
239+ }
240+}
241+
242+func TestSelectDoesNotMaskBrokenVM(t *testing.T) {
243+ cfg, _ := LoadConfig(env(nil))
244+ boom := errors.New("kernel does not boot")
245+ f := Factories{
246+ VM: func(context.Context, Config) (Driver, error) { return nil, boom },
247+ Container: func(context.Context, Config) (Driver, error) { t.Fatal("must not fall back silently"); return nil, nil },
248+ }
249+ if _, err := Select(context.Background(), cfg, f, goodProber()); !errors.Is(err, boom) {
250+ t.Fatalf("got %v", err)
251+ }
252+ if _, err := Select(context.Background(), cfg, Factories{}, goodProber()); err == nil {
253+ t.Fatal("missing factory accepted")
254+ }
255+}
256+
257+func TestCheckStatic(t *testing.T) {
258+ dir := t.TempDir()
259+ if err := CheckStatic(filepath.Join(dir, "missing")); err == nil {
260+ t.Fatal("missing file accepted")
261+ }
262+ txt := filepath.Join(dir, "script")
263+ _ = os.WriteFile(txt, []byte("#!/bin/sh\n"), 0o755)
264+ if err := CheckStatic(txt); err == nil || !strings.Contains(err.Error(), "CGO_ENABLED=0") {
265+ t.Fatalf("script accepted: %v", err)
266+ }
267+ if err := CheckStatic("/bin/sh"); err == nil {
268+ t.Skip("/bin/sh is static here")
269+ }
270+}
A internal/sandbox/sandboxtest/sandboxtest.go +92 -0
@@ -0,0 +1,92 @@
1+// SPDX-License-Identifier: MIT
2+
3+// Package sandboxtest has helpers shared by the tests of the sandbox drivers.
4+package sandboxtest
5+
6+import (
7+ "context"
8+ "encoding/binary"
9+ "net/netip"
10+ "os"
11+ "sync"
12+ "testing"
13+
14+ "bnema.dev/ermite-runner/internal/netpolicy"
15+)
16+
17+// WriteStaticELF writes a minimal statically linked ELF64 file (one PT_LOAD, no
18+// PT_INTERP): enough for sandbox.CheckStatic, never executed.
19+func WriteStaticELF(t testing.TB, path string) {
20+ t.Helper()
21+ var b [64 + 56]byte
22+ copy(b[:], "\x7fELF")
23+ b[4], b[5], b[6] = 2, 1, 1 // 64-bit, little endian, version
24+ binary.LittleEndian.PutUint16(b[16:], 2) // ET_EXEC
25+ binary.LittleEndian.PutUint16(b[18:], 62) // x86-64
26+ binary.LittleEndian.PutUint32(b[20:], 1)
27+ binary.LittleEndian.PutUint64(b[32:], 64) // e_phoff
28+ binary.LittleEndian.PutUint16(b[52:], 64) // e_ehsize
29+ binary.LittleEndian.PutUint16(b[54:], 56) // e_phentsize
30+ binary.LittleEndian.PutUint16(b[56:], 1) // e_phnum
31+ binary.LittleEndian.PutUint32(b[64:], 1) // PT_LOAD
32+ if err := os.WriteFile(path, b[:], 0o755); err != nil {
33+ t.Fatal(err)
34+ }
35+}
36+
37+// Plumbing is a netsetup.Plumbing that records what is attached.
38+type Plumbing struct {
39+ mu sync.Mutex
40+ Attached map[string]netpolicy.Guest
41+ Events []string
42+ Resets int
43+ FailNext error
44+}
45+
46+// NewPlumbing returns an empty Plumbing.
47+func NewPlumbing() *Plumbing { return &Plumbing{Attached: map[string]netpolicy.Guest{}} }
48+
49+// EnableForwarding implements netsetup.Plumbing.
50+func (p *Plumbing) EnableForwarding() error { return nil }
51+
52+// Attach implements netsetup.Plumbing.
53+func (p *Plumbing) Attach(_ context.Context, g netpolicy.Guest) error {
54+ p.mu.Lock()
55+ defer p.mu.Unlock()
56+ if p.FailNext != nil {
57+ err := p.FailNext
58+ p.FailNext = nil
59+ return err
60+ }
61+ p.Attached[g.Iface] = g
62+ p.Events = append(p.Events, "attach "+g.Iface)
63+ return nil
64+}
65+
66+// Detach implements netsetup.Plumbing.
67+func (p *Plumbing) Detach(_ context.Context, iface string, _ netip.Addr) error {
68+ p.mu.Lock()
69+ defer p.mu.Unlock()
70+ delete(p.Attached, iface)
71+ p.Events = append(p.Events, "detach "+iface)
72+ return nil
73+}
74+
75+// Reset implements netsetup.Plumbing.
76+func (p *Plumbing) Reset(context.Context) error {
77+ p.mu.Lock()
78+ defer p.mu.Unlock()
79+ p.Resets++
80+ p.Attached = map[string]netpolicy.Guest{}
81+ return nil
82+}
83+
84+// Close implements netsetup.Plumbing.
85+func (p *Plumbing) Close() {}
86+
87+// Live returns the number of attached guests.
88+func (p *Plumbing) Live() int {
89+ p.mu.Lock()
90+ defer p.mu.Unlock()
91+ return len(p.Attached)
92+}
@@ -0,0 +1,92 @@
1+// SPDX-License-Identifier: MIT
2+
3+// Package sandboxtest has helpers shared by the tests of the sandbox drivers.
4+package sandboxtest
5+
6+import (
7+ "context"
8+ "encoding/binary"
9+ "net/netip"
10+ "os"
11+ "sync"
12+ "testing"
13+
14+ "bnema.dev/ermite-runner/internal/netpolicy"
15+)
16+
17+// WriteStaticELF writes a minimal statically linked ELF64 file (one PT_LOAD, no
18+// PT_INTERP): enough for sandbox.CheckStatic, never executed.
19+func WriteStaticELF(t testing.TB, path string) {
20+ t.Helper()
21+ var b [64 + 56]byte
22+ copy(b[:], "\x7fELF")
23+ b[4], b[5], b[6] = 2, 1, 1 // 64-bit, little endian, version
24+ binary.LittleEndian.PutUint16(b[16:], 2) // ET_EXEC
25+ binary.LittleEndian.PutUint16(b[18:], 62) // x86-64
26+ binary.LittleEndian.PutUint32(b[20:], 1)
27+ binary.LittleEndian.PutUint64(b[32:], 64) // e_phoff
28+ binary.LittleEndian.PutUint16(b[52:], 64) // e_ehsize
29+ binary.LittleEndian.PutUint16(b[54:], 56) // e_phentsize
30+ binary.LittleEndian.PutUint16(b[56:], 1) // e_phnum
31+ binary.LittleEndian.PutUint32(b[64:], 1) // PT_LOAD
32+ if err := os.WriteFile(path, b[:], 0o755); err != nil {
33+ t.Fatal(err)
34+ }
35+}
36+
37+// Plumbing is a netsetup.Plumbing that records what is attached.
38+type Plumbing struct {
39+ mu sync.Mutex
40+ Attached map[string]netpolicy.Guest
41+ Events []string
42+ Resets int
43+ FailNext error
44+}
45+
46+// NewPlumbing returns an empty Plumbing.
47+func NewPlumbing() *Plumbing { return &Plumbing{Attached: map[string]netpolicy.Guest{}} }
48+
49+// EnableForwarding implements netsetup.Plumbing.
50+func (p *Plumbing) EnableForwarding() error { return nil }
51+
52+// Attach implements netsetup.Plumbing.
53+func (p *Plumbing) Attach(_ context.Context, g netpolicy.Guest) error {
54+ p.mu.Lock()
55+ defer p.mu.Unlock()
56+ if p.FailNext != nil {
57+ err := p.FailNext
58+ p.FailNext = nil
59+ return err
60+ }
61+ p.Attached[g.Iface] = g
62+ p.Events = append(p.Events, "attach "+g.Iface)
63+ return nil
64+}
65+
66+// Detach implements netsetup.Plumbing.
67+func (p *Plumbing) Detach(_ context.Context, iface string, _ netip.Addr) error {
68+ p.mu.Lock()
69+ defer p.mu.Unlock()
70+ delete(p.Attached, iface)
71+ p.Events = append(p.Events, "detach "+iface)
72+ return nil
73+}
74+
75+// Reset implements netsetup.Plumbing.
76+func (p *Plumbing) Reset(context.Context) error {
77+ p.mu.Lock()
78+ defer p.mu.Unlock()
79+ p.Resets++
80+ p.Attached = map[string]netpolicy.Guest{}
81+ return nil
82+}
83+
84+// Close implements netsetup.Plumbing.
85+func (p *Plumbing) Close() {}
86+
87+// Live returns the number of attached guests.
88+func (p *Plumbing) Live() int {
89+ p.mu.Lock()
90+ defer p.mu.Unlock()
91+ return len(p.Attached)
92+}
A internal/sandbox/select.go +156 -0
@@ -0,0 +1,156 @@
1+// SPDX-License-Identifier: MIT
2+
3+package sandbox
4+
5+import (
6+ "context"
7+ "debug/elf"
8+ "errors"
9+ "fmt"
10+ "os"
11+ "os/exec"
12+ "strings"
13+
14+ "github.com/bnema/zerowrap"
15+)
16+
17+// Factory builds the driver of one mode. It is injected so that this package
18+// does not import the drivers (which import it).
19+type Factory func(ctx context.Context, cfg Config) (Driver, error)
20+
21+// Factories are the available drivers.
22+type Factories struct {
23+ VM Factory
24+ Container Factory
25+}
26+
27+// Prober checks the host. The zero value probes the real host; tests replace
28+// fields.
29+type Prober struct {
30+ // OpenKVM opens the KVM device read-write and closes it.
31+ OpenKVM func(path string) error
32+ // LookPath finds a binary in PATH.
33+ LookPath func(name string) (string, error)
34+ // Stat reports whether a path exists.
35+ Stat func(path string) error
36+}
37+
38+func (p Prober) withDefaults() Prober {
39+ if p.OpenKVM == nil {
40+ p.OpenKVM = func(path string) error {
41+ f, err := os.OpenFile(path, os.O_RDWR, 0)
42+ if err != nil {
43+ return err
44+ }
45+ return f.Close()
46+ }
47+ }
48+ if p.LookPath == nil {
49+ p.LookPath = exec.LookPath
50+ }
51+ if p.Stat == nil {
52+ p.Stat = func(path string) error { _, err := os.Stat(path); return err }
53+ }
54+ return p
55+}
56+
57+// VMUnavailable lists why the vm mode cannot run on this host: an empty result
58+// means every prerequisite (readable-writable KVM, Firecracker, jailer unless
59+// disabled, guest kernel, base image) is there.
60+func VMUnavailable(cfg Config, p Prober) []string {
61+ p = p.withDefaults()
62+ var why []string
63+ if err := p.OpenKVM(cfg.KVMPath); err != nil {
64+ why = append(why, fmt.Sprintf("%s not usable read-write: %v", cfg.KVMPath, err))
65+ }
66+ if cfg.Firecracker != "" {
67+ if err := p.Stat(cfg.Firecracker); err != nil {
68+ why = append(why, fmt.Sprintf("firecracker %s: %v", cfg.Firecracker, err))
69+ }
70+ } else if _, err := p.LookPath("firecracker"); err != nil {
71+ why = append(why, "firecracker not found in PATH")
72+ }
73+ if !cfg.JailerOff {
74+ if cfg.Jailer != "" {
75+ if err := p.Stat(cfg.Jailer); err != nil {
76+ why = append(why, fmt.Sprintf("jailer %s: %v", cfg.Jailer, err))
77+ }
78+ } else if _, err := p.LookPath("jailer"); err != nil {
79+ why = append(why, "jailer not found in PATH")
80+ }
81+ }
82+ if err := p.Stat(cfg.KernelPath()); err != nil {
83+ why = append(why, fmt.Sprintf("guest kernel %s: %v", cfg.KernelPath(), err))
84+ }
85+ if err := p.Stat(cfg.BaseImagePath()); err != nil {
86+ why = append(why, fmt.Sprintf("base image %s: %v", cfg.BaseImagePath(), err))
87+ }
88+ return why
89+}
90+
91+// Select picks the isolation mode and builds its driver. For auto it chooses vm
92+// when VMUnavailable is empty, container otherwise. For vm it fails when a
93+// prerequisite is missing; for container it never probes the VM side. The
94+// chosen mode and the reasons are logged with the logger of ctx. A driver
95+// factory error is returned as is (auto does not mask a broken vm setup by
96+// silently running jobs with weaker isolation).
97+func Select(ctx context.Context, cfg Config, f Factories, p Prober) (Driver, error) {
98+ log := zerowrap.FromCtx(ctx)
99+ mode := cfg.Isolation
100+ var reasons []string
101+ switch mode {
102+ case VM, Auto:
103+ reasons = VMUnavailable(cfg, p)
104+ if mode == VM && len(reasons) > 0 {
105+ return nil, fmt.Errorf("isolation vm is not available: %s", strings.Join(reasons, "; "))
106+ }
107+ if len(reasons) == 0 {
108+ mode = VM
109+ } else {
110+ mode = Container
111+ }
112+ case Container:
113+ default:
114+ return nil, fmt.Errorf("unknown isolation %q", mode)
115+ }
116+
117+ build := f.VM
118+ if mode == Container {
119+ build = f.Container
120+ }
121+ if build == nil {
122+ return nil, fmt.Errorf("isolation %s: no driver compiled in", mode)
123+ }
124+ ev := log.Info().Str("configured", string(cfg.Isolation)).Str("mode", string(mode))
125+ switch {
126+ case cfg.Isolation == Auto && mode == VM:
127+ ev = ev.Str("reason", "KVM, firecracker, kernel and base image are available")
128+ case cfg.Isolation == Auto:
129+ ev = ev.Strs("reasons", reasons)
130+ default:
131+ ev = ev.Str("reason", "set by "+EnvIsolation)
132+ }
133+ ev.Msg("isolation mode chosen")
134+ d, err := build(ctx, cfg)
135+ if err != nil {
136+ return nil, fmt.Errorf("isolation %s: %w", mode, err)
137+ }
138+ return d, nil
139+}
140+
141+// CheckStatic fails unless the executable at path is a statically linked ELF
142+// binary: it runs as PID 1 on a root disk that holds nothing else, and in an
143+// image of another distribution.
144+func CheckStatic(path string) error {
145+ f, err := elf.Open(path)
146+ if err != nil {
147+ return fmt.Errorf("%s is not an ELF executable (%w); build the runner with CGO_ENABLED=0", path, err)
148+ }
149+ defer func() { _ = f.Close() }()
150+ for _, p := range f.Progs {
151+ if p.Type == elf.PT_INTERP {
152+ return errors.New(path + " is dynamically linked; build the runner with CGO_ENABLED=0")
153+ }
154+ }
155+ return nil
156+}
@@ -0,0 +1,156 @@
1+// SPDX-License-Identifier: MIT
2+
3+package sandbox
4+
5+import (
6+ "context"
7+ "debug/elf"
8+ "errors"
9+ "fmt"
10+ "os"
11+ "os/exec"
12+ "strings"
13+
14+ "github.com/bnema/zerowrap"
15+)
16+
17+// Factory builds the driver of one mode. It is injected so that this package
18+// does not import the drivers (which import it).
19+type Factory func(ctx context.Context, cfg Config) (Driver, error)
20+
21+// Factories are the available drivers.
22+type Factories struct {
23+ VM Factory
24+ Container Factory
25+}
26+
27+// Prober checks the host. The zero value probes the real host; tests replace
28+// fields.
29+type Prober struct {
30+ // OpenKVM opens the KVM device read-write and closes it.
31+ OpenKVM func(path string) error
32+ // LookPath finds a binary in PATH.
33+ LookPath func(name string) (string, error)
34+ // Stat reports whether a path exists.
35+ Stat func(path string) error
36+}
37+
38+func (p Prober) withDefaults() Prober {
39+ if p.OpenKVM == nil {
40+ p.OpenKVM = func(path string) error {
41+ f, err := os.OpenFile(path, os.O_RDWR, 0)
42+ if err != nil {
43+ return err
44+ }
45+ return f.Close()
46+ }
47+ }
48+ if p.LookPath == nil {
49+ p.LookPath = exec.LookPath
50+ }
51+ if p.Stat == nil {
52+ p.Stat = func(path string) error { _, err := os.Stat(path); return err }
53+ }
54+ return p
55+}
56+
57+// VMUnavailable lists why the vm mode cannot run on this host: an empty result
58+// means every prerequisite (readable-writable KVM, Firecracker, jailer unless
59+// disabled, guest kernel, base image) is there.
60+func VMUnavailable(cfg Config, p Prober) []string {
61+ p = p.withDefaults()
62+ var why []string
63+ if err := p.OpenKVM(cfg.KVMPath); err != nil {
64+ why = append(why, fmt.Sprintf("%s not usable read-write: %v", cfg.KVMPath, err))
65+ }
66+ if cfg.Firecracker != "" {
67+ if err := p.Stat(cfg.Firecracker); err != nil {
68+ why = append(why, fmt.Sprintf("firecracker %s: %v", cfg.Firecracker, err))
69+ }
70+ } else if _, err := p.LookPath("firecracker"); err != nil {
71+ why = append(why, "firecracker not found in PATH")
72+ }
73+ if !cfg.JailerOff {
74+ if cfg.Jailer != "" {
75+ if err := p.Stat(cfg.Jailer); err != nil {
76+ why = append(why, fmt.Sprintf("jailer %s: %v", cfg.Jailer, err))
77+ }
78+ } else if _, err := p.LookPath("jailer"); err != nil {
79+ why = append(why, "jailer not found in PATH")
80+ }
81+ }
82+ if err := p.Stat(cfg.KernelPath()); err != nil {
83+ why = append(why, fmt.Sprintf("guest kernel %s: %v", cfg.KernelPath(), err))
84+ }
85+ if err := p.Stat(cfg.BaseImagePath()); err != nil {
86+ why = append(why, fmt.Sprintf("base image %s: %v", cfg.BaseImagePath(), err))
87+ }
88+ return why
89+}
90+
91+// Select picks the isolation mode and builds its driver. For auto it chooses vm
92+// when VMUnavailable is empty, container otherwise. For vm it fails when a
93+// prerequisite is missing; for container it never probes the VM side. The
94+// chosen mode and the reasons are logged with the logger of ctx. A driver
95+// factory error is returned as is (auto does not mask a broken vm setup by
96+// silently running jobs with weaker isolation).
97+func Select(ctx context.Context, cfg Config, f Factories, p Prober) (Driver, error) {
98+ log := zerowrap.FromCtx(ctx)
99+ mode := cfg.Isolation
100+ var reasons []string
101+ switch mode {
102+ case VM, Auto:
103+ reasons = VMUnavailable(cfg, p)
104+ if mode == VM && len(reasons) > 0 {
105+ return nil, fmt.Errorf("isolation vm is not available: %s", strings.Join(reasons, "; "))
106+ }
107+ if len(reasons) == 0 {
108+ mode = VM
109+ } else {
110+ mode = Container
111+ }
112+ case Container:
113+ default:
114+ return nil, fmt.Errorf("unknown isolation %q", mode)
115+ }
116+
117+ build := f.VM
118+ if mode == Container {
119+ build = f.Container
120+ }
121+ if build == nil {
122+ return nil, fmt.Errorf("isolation %s: no driver compiled in", mode)
123+ }
124+ ev := log.Info().Str("configured", string(cfg.Isolation)).Str("mode", string(mode))
125+ switch {
126+ case cfg.Isolation == Auto && mode == VM:
127+ ev = ev.Str("reason", "KVM, firecracker, kernel and base image are available")
128+ case cfg.Isolation == Auto:
129+ ev = ev.Strs("reasons", reasons)
130+ default:
131+ ev = ev.Str("reason", "set by "+EnvIsolation)
132+ }
133+ ev.Msg("isolation mode chosen")
134+ d, err := build(ctx, cfg)
135+ if err != nil {
136+ return nil, fmt.Errorf("isolation %s: %w", mode, err)
137+ }
138+ return d, nil
139+}
140+
141+// CheckStatic fails unless the executable at path is a statically linked ELF
142+// binary: it runs as PID 1 on a root disk that holds nothing else, and in an
143+// image of another distribution.
144+func CheckStatic(path string) error {
145+ f, err := elf.Open(path)
146+ if err != nil {
147+ return fmt.Errorf("%s is not an ELF executable (%w); build the runner with CGO_ENABLED=0", path, err)
148+ }
149+ defer func() { _ = f.Close() }()
150+ for _, p := range f.Progs {
151+ if p.Type == elf.PT_INTERP {
152+ return errors.New(path + " is dynamically linked; build the runner with CGO_ENABLED=0")
153+ }
154+ }
155+ return nil
156+}

Keyboard shortcuts

Single-key shortcuts are on. ctrl k and esc work whatever this says.

ctrl k :
go to a section, branch, tag, file or repository
?
this list
j k
move the row cursor down / up
enter
open the row under the cursor; on Files, a directory opens or closes in place
↑ ↓ ← → home end
in the file tree (focus on a row): previous / next row, close / open a directory or go to its parent / first entry, first / last row. They never open a file
l h
on Files: open a directory or a file / close a directory or go up to its parent, then to the parent page. Elsewhere: into the side panel (on a merge request, its files) / back to the content; there j k move between its controls
g g G
first / last row
/
filter the list; on Files, find a file anywhere in the repository (esc clears, enter opens the best match)
b
switch branch or tag
g s g f g c g b g m g i g r g a
tabs: summary, files, commits, branches & tags, merge requests, issues, releases, actions (when available)
g h
home
w
wide mode: use the whole width of the screen, or not (remembered)
y
copy a permalink pinned to the commit
d u x
diff: split / unified / collapse or expand all
e E
diff: the whole file in view / every file, with the lines the hunks hide
n
merge request: next note in the diff
esc
close a dialog, leave a field

Branches 4

Tags 11

↑↓ moveenter openesc close