1// SPDX-License-Identifier: MIT
2
3package sandbox
4
5import (
6 "errors"
7 "fmt"
8 "net/netip"
9 "os"
10 "path/filepath"
11 "runtime"
12 "strconv"
13 "strings"
14
15 "bnema.dev/ermite-runner/internal/cachesvc"
16)
17
18// Environment variables read by LoadConfig. The other ERMITE_RUNNER_* variables
19// (URL, NAME, LABELS, ...) belong to the host process, not to the sandboxes.
20const (
21 EnvIsolation = "ERMITE_RUNNER_ISOLATION"
22 EnvStateDir = "ERMITE_RUNNER_STATE_DIR"
23 EnvCapacity = "ERMITE_RUNNER_CAPACITY"
24 EnvImage = "ERMITE_RUNNER_IMAGE"
25 EnvJobCPUs = "ERMITE_RUNNER_JOB_CPUS"
26 EnvJobMemory = "ERMITE_RUNNER_JOB_MEMORY"
27 EnvJobDisk = "ERMITE_RUNNER_JOB_DISK"
28 EnvUIDBase = "ERMITE_RUNNER_UID_BASE"
29 EnvNetwork = "ERMITE_RUNNER_NETWORK"
30 EnvFirecracker = "ERMITE_RUNNER_FIRECRACKER"
31 EnvJailer = "ERMITE_RUNNER_JAILER"
32 EnvJailerMode = "ERMITE_RUNNER_JAILER_MODE"
33 EnvKernel = "ERMITE_RUNNER_KERNEL"
34
35 EnvJobPids = "ERMITE_RUNNER_JOB_PIDS"
36 EnvDenyPrivileged = "ERMITE_RUNNER_DENY_PRIVILEGED"
37 EnvLockFile = "ERMITE_RUNNER_LOCK_FILE"
38 EnvJobDiskBW = "ERMITE_RUNNER_JOB_DISK_BANDWIDTH"
39 EnvJobDiskIOPS = "ERMITE_RUNNER_JOB_DISK_IOPS"
40 EnvJobNetBandwidth = "ERMITE_RUNNER_JOB_NET_BANDWIDTH"
41 // EnvNode=0 turns the Node.js externals off (nothing is downloaded or
42 // attached).
43 EnvNode = "ERMITE_RUNNER_NODE"
44)
45
46// Defaults.
47const (
48 DefaultStateDir = "/var/lib/ermite-runner"
49 DefaultOCIImage = "ermite-runner-ubuntu-24.04:latest"
50 DefaultUIDBase = 900000
51 DefaultNetwork = "10.200.0.0/16"
52 DefaultCPUs = 2
53 DefaultMemoryMiB = 4096
54 DefaultDiskMiB = 64 * 1024
55 DefaultJobPids = 32768
56 // DefaultLockFile is the single-instance lock of the host: reaping removes
57 // everything the runner labels as its own, so two runners (even with
58 // different state directories, or one in a container) must never coexist.
59 DefaultLockFile = "/run/ermite-runner.lock"
60 // Rate limits of a micro-VM, generous so that they only stop a runaway job
61 // from starving the host: 400 MiB/s and 20000 IOPS per disk, 1 Gbit/s per
62 // network direction. Zero disables a limit.
63 DefaultDiskBandwidthMiB = 400
64 DefaultDiskIOPS = 20000
65 DefaultNetMbit = 1000
66)
67
68// Config is the sandbox part of the runner configuration.
69type Config struct {
70 Isolation Mode
71 StateDir string
72 Capacity int
73
74 // Image is ERMITE_RUNNER_IMAGE: a directory (VM: holds the kernel and
75 // base.ext4) or an OCI reference (container). A value that is empty or does
76 // not fit the selected mode falls back to the default for that mode.
77 Image string
78
79 JobCPUs int
80 JobMemoryMiB int
81 JobDiskMiB int
82
83 // UIDBase+slot is the uid and gid the jailer drops to.
84 UIDBase int
85 // Network is the pool of /30 subnets, one per slot (VM TAPs, container
86 // bridges).
87 Network netip.Prefix
88
89 // Firecracker and Jailer are binary paths; empty means look in PATH.
90 Firecracker string
91 Jailer string
92 // Kernel overrides <image dir>/vmlinux (Image on arm64).
93 Kernel string
94 // JailerOff runs Firecracker without the jailer (development and tests as
95 // an unprivileged user). Never use it in production: it only drops the
96 // chroot, uid and namespace confinement of the VMM process.
97 JailerOff bool
98
99 // KVMPath is the KVM device; tests point it elsewhere to hide /dev/kvm.
100 KVMPath string
101 // Executable is attached read-only to every sandbox. Empty means
102 // os.Executable(). It must be a static binary (CGO_ENABLED=0).
103 Executable string
104 // DNSPort is the port of the DNS forwarder; 53 in production. Tests that
105 // cannot bind it use another port with no guest.
106 DNSPort int
107
108 // JobPids is the process limit of a job (container PidsLimit, VM pids.max
109 // is derived from the vCPU count and is not affected).
110 JobPids int
111 // DenyPrivileged refuses the container mode when it would have to run
112 // privileged (no sysbox).
113 DenyPrivileged bool
114 // LockFile is the host-wide single-instance lock.
115 LockFile string
116 // GuestForgeEnabled is ERMITE_RUNNER_GUEST_FORGE=1. The endpoint itself
117 // comes from the server URL, which is not a sandbox setting: the caller
118 // sets GuestForge (ResolveGuestForge) before building the drivers.
119 GuestForgeEnabled bool
120 // GuestForge, when not nil, is the one host endpoint that every sandbox may
121 // reach, and the name the DNS forwarder answers with the sandbox's gateway.
122 GuestForge *GuestForge
123 // Cache, when not nil, is the runner's cache service (package cachesvc),
124 // served to every sandbox on a channel of its own (ServeCache), never on a
125 // network. The caller makes it before building the drivers, like GuestForge.
126 Cache *cachesvc.Service
127 // Rate limits of micro-VMs; zero disables.
128 DiskBandwidthMiB, DiskIOPS, NetMbit int
129
130 // NodeExternals is false when ERMITE_RUNNER_NODE=0: the caller then sets no
131 // Externals.
132 NodeExternals bool
133 // Externals, when not nil, is the host's set of external tools (Node.js),
134 // attached read-only to every sandbox at GuestExternalsDir. The caller sets
135 // it before building the drivers, like GuestForge.
136 Externals ExternalsSource
137}
138
139// LoadConfig reads the sandbox settings from getenv (os.Getenv in production).
140func LoadConfig(getenv func(string) string) (Config, error) {
141 c := Config{
142 Isolation: Auto,
143 StateDir: DefaultStateDir,
144 Capacity: 1,
145 JobCPUs: DefaultCPUs,
146 JobMemoryMiB: DefaultMemoryMiB,
147 JobDiskMiB: DefaultDiskMiB,
148 UIDBase: DefaultUIDBase,
149 KVMPath: "/dev/kvm",
150 DNSPort: 53,
151 JobPids: DefaultJobPids,
152 LockFile: DefaultLockFile,
153
154 NodeExternals: true,
155
156 DiskBandwidthMiB: DefaultDiskBandwidthMiB,
157 DiskIOPS: DefaultDiskIOPS,
158 NetMbit: DefaultNetMbit,
159 }
160 var err error
161 if v := getenv(EnvIsolation); v != "" {
162 switch m := Mode(strings.ToLower(strings.TrimSpace(v))); m {
163 case Auto, VM, Container:
164 c.Isolation = m
165 default:
166 return c, fmt.Errorf("%s: %q is not auto, vm or container", EnvIsolation, v)
167 }
168 }
169 if v := getenv(EnvStateDir); v != "" {
170 c.StateDir = v
171 }
172 if !filepath.IsAbs(c.StateDir) {
173 return c, fmt.Errorf("%s: %q must be an absolute path", EnvStateDir, c.StateDir)
174 }
175 if c.Capacity, err = intEnv(getenv, EnvCapacity, c.Capacity, 1, 1024); err != nil {
176 return c, err
177 }
178 c.Image = getenv(EnvImage)
179 if c.JobCPUs, err = intEnv(getenv, EnvJobCPUs, c.JobCPUs, 1, 128); err != nil {
180 return c, err
181 }
182 if v := getenv(EnvJobMemory); v != "" {
183 if c.JobMemoryMiB, err = ParseSizeMiB(v); err != nil || c.JobMemoryMiB < 256 {
184 return c, fmt.Errorf("%s: %q: need a size of at least 256M", EnvJobMemory, v)
185 }
186 }
187 if v := getenv(EnvJobDisk); v != "" {
188 if c.JobDiskMiB, err = ParseSizeMiB(v); err != nil || c.JobDiskMiB < 1024 {
189 return c, fmt.Errorf("%s: %q: need a size of at least 1G", EnvJobDisk, v)
190 }
191 }
192 if c.UIDBase, err = intEnv(getenv, EnvUIDBase, c.UIDBase, 1000, 4_000_000_000); err != nil {
193 return c, err
194 }
195 if c.UIDBase+c.Capacity > 4_294_967_294 {
196 return c, fmt.Errorf("%s: %d + capacity overflows the uid range", EnvUIDBase, c.UIDBase)
197 }
198 net := DefaultNetwork
199 if v := getenv(EnvNetwork); v != "" {
200 net = v
201 }
202 if c.Network, err = netip.ParsePrefix(net); err != nil || !c.Network.Addr().Is4() || c.Network.Bits() > 30 {
203 return c, fmt.Errorf("%s: %q is not an IPv4 prefix of at most /30", EnvNetwork, net)
204 }
205 if 1<<(30-c.Network.Bits()) < c.Capacity {
206 return c, fmt.Errorf("%s: %s has fewer /30 subnets than the capacity %d", EnvNetwork, c.Network, c.Capacity)
207 }
208 if c.JobPids, err = intEnv(getenv, EnvJobPids, c.JobPids, 64, 4_194_304); err != nil {
209 return c, err
210 }
211 switch strings.ToLower(strings.TrimSpace(getenv(EnvDenyPrivileged))) {
212 case "", "0", "false", "no":
213 case "1", "true", "yes":
214 c.DenyPrivileged = true
215 default:
216 return c, fmt.Errorf("%s: %q is not 0 or 1", EnvDenyPrivileged, getenv(EnvDenyPrivileged))
217 }
218 switch strings.ToLower(strings.TrimSpace(getenv(EnvGuestForge))) {
219 case "", "0", "false", "no":
220 case "1", "true", "yes":
221 c.GuestForgeEnabled = true
222 default:
223 return c, fmt.Errorf("%s: %q is not 0 or 1", EnvGuestForge, getenv(EnvGuestForge))
224 }
225 switch strings.ToLower(strings.TrimSpace(getenv(EnvNode))) {
226 case "", "1", "true", "yes":
227 case "0", "false", "no":
228 c.NodeExternals = false
229 default:
230 return c, fmt.Errorf("%s: %q is not 0 or 1", EnvNode, getenv(EnvNode))
231 }
232 if v := getenv(EnvLockFile); v != "" {
233 if !filepath.IsAbs(v) {
234 return c, fmt.Errorf("%s: %q must be an absolute path", EnvLockFile, v)
235 }
236 c.LockFile = v
237 }
238 if c.DiskBandwidthMiB, err = intEnv(getenv, EnvJobDiskBW, c.DiskBandwidthMiB, 0, 1_000_000); err != nil {
239 return c, err
240 }
241 if c.DiskIOPS, err = intEnv(getenv, EnvJobDiskIOPS, c.DiskIOPS, 0, 100_000_000); err != nil {
242 return c, err
243 }
244 if c.NetMbit, err = intEnv(getenv, EnvJobNetBandwidth, c.NetMbit, 0, 1_000_000); err != nil {
245 return c, err
246 }
247 c.Firecracker = getenv(EnvFirecracker)
248 c.Jailer = getenv(EnvJailer)
249 c.Kernel = getenv(EnvKernel)
250 switch strings.ToLower(getenv(EnvJailerMode)) {
251 case "", "on":
252 case "off":
253 c.JailerOff = true
254 default:
255 return c, fmt.Errorf("%s: %q is not on or off", EnvJailerMode, getenv(EnvJailerMode))
256 }
257 return c, nil
258}
259
260func intEnv(getenv func(string) string, key string, def, lo, hi int) (int, error) {
261 v := getenv(key)
262 if v == "" {
263 return def, nil
264 }
265 n, err := strconv.Atoi(strings.TrimSpace(v))
266 if err != nil || n < lo || n > hi {
267 return 0, fmt.Errorf("%s: %q is not an integer in [%d, %d]", key, v, lo, hi)
268 }
269 return n, nil
270}
271
272// ParseSizeMiB parses "4096", "512M", "4G", "64GiB", "1T" into MiB. A bare
273// number is MiB. Suffixes are binary (1G = 1024 MiB) and case-insensitive.
274func ParseSizeMiB(s string) (int, error) {
275 s = strings.ToLower(strings.TrimSpace(s))
276 s = strings.TrimSuffix(strings.TrimSuffix(s, "b"), "i")
277 mult := 1
278 if s != "" {
279 switch s[len(s)-1] {
280 case 'k':
281 mult, s = 0, s[:len(s)-1]
282 case 'm':
283 mult, s = 1, s[:len(s)-1]
284 case 'g':
285 mult, s = 1024, s[:len(s)-1]
286 case 't':
287 mult, s = 1024*1024, s[:len(s)-1]
288 }
289 }
290 n, err := strconv.Atoi(s)
291 if err != nil || n <= 0 || mult == 0 {
292 return 0, errors.New("invalid size")
293 }
294 if n > (1<<31-1)/mult {
295 return 0, errors.New("size too large")
296 }
297 return n * mult, nil
298}
299
300// ImageDir returns the VM image directory: Image when it is a directory path,
301// else <StateDir>/image.
302func (c Config) ImageDir() string {
303 if c.Image != "" && looksLikePath(c.Image) {
304 return c.Image
305 }
306 return filepath.Join(c.StateDir, "image")
307}
308
309// OCIImage returns the container image reference: Image when it is not a
310// directory path, else DefaultOCIImage.
311func (c Config) OCIImage() string {
312 if c.Image != "" && !looksLikePath(c.Image) {
313 return c.Image
314 }
315 return DefaultOCIImage
316}
317
318func looksLikePath(s string) bool {
319 return strings.HasPrefix(s, "/") || strings.HasPrefix(s, "./") || strings.HasPrefix(s, "../")
320}
321
322// KernelPath returns the guest kernel: Kernel, else <ImageDir>/vmlinux (x86-64)
323// or <ImageDir>/Image (arm64).
324func (c Config) KernelPath() string {
325 if c.Kernel != "" {
326 return c.Kernel
327 }
328 name := "vmlinux"
329 if runtime.GOARCH == "arm64" {
330 name = "Image"
331 }
332 return filepath.Join(c.ImageDir(), name)
333}
334
335// BaseImagePath returns the shared read-only base disk.
336func (c Config) BaseImagePath() string { return filepath.Join(c.ImageDir(), "base.ext4") }
337
338// ExecutablePath returns the binary attached to sandboxes.
339func (c Config) ExecutablePath() (string, error) {
340 if c.Executable != "" {
341 return c.Executable, nil
342 }
343 p, err := os.Executable()
344 if err != nil {
345 return "", fmt.Errorf("sandbox: locate own executable: %w", err)
346 }
347 return filepath.EvalSymlinks(p)
348}
349
350// Resolve fills the zero resource fields of a JobSpec from the config.
351func (c Config) Resolve(s JobSpec) JobSpec {
352 if s.CPUs <= 0 {
353 s.CPUs = c.JobCPUs
354 }
355 if s.MemoryMiB <= 0 {
356 s.MemoryMiB = c.JobMemoryMiB
357 }
358 if s.DiskMiB <= 0 {
359 s.DiskMiB = c.JobDiskMiB
360 }
361 return s
362}