1// SPDX-License-Identifier: MIT
2
3package sandbox
4
5import (
6 "errors"
7 "fmt"
8 "net/netip"
9 "os"
10 "path/filepath"
11 "runtime"
12 "strconv"
13 "strings"
14)
15
16// Environment variables read by LoadConfig. The other ERMITE_RUNNER_* variables
17// (URL, NAME, LABELS, ...) belong to the host process, not to the sandboxes.
18const (
19 EnvIsolation = "ERMITE_RUNNER_ISOLATION"
20 EnvStateDir = "ERMITE_RUNNER_STATE_DIR"
21 EnvCapacity = "ERMITE_RUNNER_CAPACITY"
22 EnvImage = "ERMITE_RUNNER_IMAGE"
23 EnvJobCPUs = "ERMITE_RUNNER_JOB_CPUS"
24 EnvJobMemory = "ERMITE_RUNNER_JOB_MEMORY"
25 EnvJobDisk = "ERMITE_RUNNER_JOB_DISK"
26 EnvUIDBase = "ERMITE_RUNNER_UID_BASE"
27 EnvNetwork = "ERMITE_RUNNER_NETWORK"
28 EnvFirecracker = "ERMITE_RUNNER_FIRECRACKER"
29 EnvJailer = "ERMITE_RUNNER_JAILER"
30 EnvJailerMode = "ERMITE_RUNNER_JAILER_MODE"
31 EnvKernel = "ERMITE_RUNNER_KERNEL"
32
33 EnvJobPids = "ERMITE_RUNNER_JOB_PIDS"
34 EnvDenyPrivileged = "ERMITE_RUNNER_DENY_PRIVILEGED"
35 EnvLockFile = "ERMITE_RUNNER_LOCK_FILE"
36 EnvJobDiskBW = "ERMITE_RUNNER_JOB_DISK_BANDWIDTH"
37 EnvJobDiskIOPS = "ERMITE_RUNNER_JOB_DISK_IOPS"
38 EnvJobNetBandwidth = "ERMITE_RUNNER_JOB_NET_BANDWIDTH"
39 // EnvNode=0 turns the Node.js externals off (nothing is downloaded or
40 // attached).
41 EnvNode = "ERMITE_RUNNER_NODE"
42)
43
44// Defaults.
45const (
46 DefaultStateDir = "/var/lib/ermite-runner"
47 DefaultOCIImage = "ermite-runner-ubuntu-24.04:latest"
48 DefaultUIDBase = 900000
49 DefaultNetwork = "10.200.0.0/16"
50 DefaultCPUs = 2
51 DefaultMemoryMiB = 4096
52 DefaultDiskMiB = 64 * 1024
53 DefaultJobPids = 32768
54 // DefaultLockFile is the single-instance lock of the host: reaping removes
55 // everything the runner labels as its own, so two runners (even with
56 // different state directories, or one in a container) must never coexist.
57 DefaultLockFile = "/run/ermite-runner.lock"
58 // Rate limits of a micro-VM, generous so that they only stop a runaway job
59 // from starving the host: 400 MiB/s and 20000 IOPS per disk, 1 Gbit/s per
60 // network direction. Zero disables a limit.
61 DefaultDiskBandwidthMiB = 400
62 DefaultDiskIOPS = 20000
63 DefaultNetMbit = 1000
64)
65
66// Config is the sandbox part of the runner configuration.
67type Config struct {
68 Isolation Mode
69 StateDir string
70 Capacity int
71
72 // Image is ERMITE_RUNNER_IMAGE: a directory (VM: holds the kernel and
73 // base.ext4) or an OCI reference (container). A value that is empty or does
74 // not fit the selected mode falls back to the default for that mode.
75 Image string
76
77 JobCPUs int
78 JobMemoryMiB int
79 JobDiskMiB int
80
81 // UIDBase+slot is the uid and gid the jailer drops to.
82 UIDBase int
83 // Network is the pool of /30 subnets, one per slot (VM TAPs, container
84 // bridges).
85 Network netip.Prefix
86
87 // Firecracker and Jailer are binary paths; empty means look in PATH.
88 Firecracker string
89 Jailer string
90 // Kernel overrides <image dir>/vmlinux (Image on arm64).
91 Kernel string
92 // JailerOff runs Firecracker without the jailer (development and tests as
93 // an unprivileged user). Never use it in production: it only drops the
94 // chroot, uid and namespace confinement of the VMM process.
95 JailerOff bool
96
97 // KVMPath is the KVM device; tests point it elsewhere to hide /dev/kvm.
98 KVMPath string
99 // Executable is attached read-only to every sandbox. Empty means
100 // os.Executable(). It must be a static binary (CGO_ENABLED=0).
101 Executable string
102 // DNSPort is the port of the DNS forwarder; 53 in production. Tests that
103 // cannot bind it use another port with no guest.
104 DNSPort int
105
106 // JobPids is the process limit of a job (container PidsLimit, VM pids.max
107 // is derived from the vCPU count and is not affected).
108 JobPids int
109 // DenyPrivileged refuses the container mode when it would have to run
110 // privileged (no sysbox).
111 DenyPrivileged bool
112 // LockFile is the host-wide single-instance lock.
113 LockFile string
114 // GuestForgeEnabled is ERMITE_RUNNER_GUEST_FORGE=1. The endpoint itself
115 // comes from the server URL, which is not a sandbox setting: the caller
116 // sets GuestForge (ResolveGuestForge) before building the drivers.
117 GuestForgeEnabled bool
118 // GuestForge, when not nil, is the one host endpoint that every sandbox may
119 // reach, and the name the DNS forwarder answers with the sandbox's gateway.
120 GuestForge *GuestForge
121 // Rate limits of micro-VMs; zero disables.
122 DiskBandwidthMiB, DiskIOPS, NetMbit int
123
124 // NodeExternals is false when ERMITE_RUNNER_NODE=0: the caller then sets no
125 // Externals.
126 NodeExternals bool
127 // Externals, when not nil, is the host's set of external tools (Node.js),
128 // attached read-only to every sandbox at GuestExternalsDir. The caller sets
129 // it before building the drivers, like GuestForge.
130 Externals ExternalsSource
131}
132
133// LoadConfig reads the sandbox settings from getenv (os.Getenv in production).
134func LoadConfig(getenv func(string) string) (Config, error) {
135 c := Config{
136 Isolation: Auto,
137 StateDir: DefaultStateDir,
138 Capacity: 1,
139 JobCPUs: DefaultCPUs,
140 JobMemoryMiB: DefaultMemoryMiB,
141 JobDiskMiB: DefaultDiskMiB,
142 UIDBase: DefaultUIDBase,
143 KVMPath: "/dev/kvm",
144 DNSPort: 53,
145 JobPids: DefaultJobPids,
146 LockFile: DefaultLockFile,
147
148 NodeExternals: true,
149
150 DiskBandwidthMiB: DefaultDiskBandwidthMiB,
151 DiskIOPS: DefaultDiskIOPS,
152 NetMbit: DefaultNetMbit,
153 }
154 var err error
155 if v := getenv(EnvIsolation); v != "" {
156 switch m := Mode(strings.ToLower(strings.TrimSpace(v))); m {
157 case Auto, VM, Container:
158 c.Isolation = m
159 default:
160 return c, fmt.Errorf("%s: %q is not auto, vm or container", EnvIsolation, v)
161 }
162 }
163 if v := getenv(EnvStateDir); v != "" {
164 c.StateDir = v
165 }
166 if !filepath.IsAbs(c.StateDir) {
167 return c, fmt.Errorf("%s: %q must be an absolute path", EnvStateDir, c.StateDir)
168 }
169 if c.Capacity, err = intEnv(getenv, EnvCapacity, c.Capacity, 1, 1024); err != nil {
170 return c, err
171 }
172 c.Image = getenv(EnvImage)
173 if c.JobCPUs, err = intEnv(getenv, EnvJobCPUs, c.JobCPUs, 1, 128); err != nil {
174 return c, err
175 }
176 if v := getenv(EnvJobMemory); v != "" {
177 if c.JobMemoryMiB, err = ParseSizeMiB(v); err != nil || c.JobMemoryMiB < 256 {
178 return c, fmt.Errorf("%s: %q: need a size of at least 256M", EnvJobMemory, v)
179 }
180 }
181 if v := getenv(EnvJobDisk); v != "" {
182 if c.JobDiskMiB, err = ParseSizeMiB(v); err != nil || c.JobDiskMiB < 1024 {
183 return c, fmt.Errorf("%s: %q: need a size of at least 1G", EnvJobDisk, v)
184 }
185 }
186 if c.UIDBase, err = intEnv(getenv, EnvUIDBase, c.UIDBase, 1000, 4_000_000_000); err != nil {
187 return c, err
188 }
189 if c.UIDBase+c.Capacity > 4_294_967_294 {
190 return c, fmt.Errorf("%s: %d + capacity overflows the uid range", EnvUIDBase, c.UIDBase)
191 }
192 net := DefaultNetwork
193 if v := getenv(EnvNetwork); v != "" {
194 net = v
195 }
196 if c.Network, err = netip.ParsePrefix(net); err != nil || !c.Network.Addr().Is4() || c.Network.Bits() > 30 {
197 return c, fmt.Errorf("%s: %q is not an IPv4 prefix of at most /30", EnvNetwork, net)
198 }
199 if 1<<(30-c.Network.Bits()) < c.Capacity {
200 return c, fmt.Errorf("%s: %s has fewer /30 subnets than the capacity %d", EnvNetwork, c.Network, c.Capacity)
201 }
202 if c.JobPids, err = intEnv(getenv, EnvJobPids, c.JobPids, 64, 4_194_304); err != nil {
203 return c, err
204 }
205 switch strings.ToLower(strings.TrimSpace(getenv(EnvDenyPrivileged))) {
206 case "", "0", "false", "no":
207 case "1", "true", "yes":
208 c.DenyPrivileged = true
209 default:
210 return c, fmt.Errorf("%s: %q is not 0 or 1", EnvDenyPrivileged, getenv(EnvDenyPrivileged))
211 }
212 switch strings.ToLower(strings.TrimSpace(getenv(EnvGuestForge))) {
213 case "", "0", "false", "no":
214 case "1", "true", "yes":
215 c.GuestForgeEnabled = true
216 default:
217 return c, fmt.Errorf("%s: %q is not 0 or 1", EnvGuestForge, getenv(EnvGuestForge))
218 }
219 switch strings.ToLower(strings.TrimSpace(getenv(EnvNode))) {
220 case "", "1", "true", "yes":
221 case "0", "false", "no":
222 c.NodeExternals = false
223 default:
224 return c, fmt.Errorf("%s: %q is not 0 or 1", EnvNode, getenv(EnvNode))
225 }
226 if v := getenv(EnvLockFile); v != "" {
227 if !filepath.IsAbs(v) {
228 return c, fmt.Errorf("%s: %q must be an absolute path", EnvLockFile, v)
229 }
230 c.LockFile = v
231 }
232 if c.DiskBandwidthMiB, err = intEnv(getenv, EnvJobDiskBW, c.DiskBandwidthMiB, 0, 1_000_000); err != nil {
233 return c, err
234 }
235 if c.DiskIOPS, err = intEnv(getenv, EnvJobDiskIOPS, c.DiskIOPS, 0, 100_000_000); err != nil {
236 return c, err
237 }
238 if c.NetMbit, err = intEnv(getenv, EnvJobNetBandwidth, c.NetMbit, 0, 1_000_000); err != nil {
239 return c, err
240 }
241 c.Firecracker = getenv(EnvFirecracker)
242 c.Jailer = getenv(EnvJailer)
243 c.Kernel = getenv(EnvKernel)
244 switch strings.ToLower(getenv(EnvJailerMode)) {
245 case "", "on":
246 case "off":
247 c.JailerOff = true
248 default:
249 return c, fmt.Errorf("%s: %q is not on or off", EnvJailerMode, getenv(EnvJailerMode))
250 }
251 return c, nil
252}
253
254func intEnv(getenv func(string) string, key string, def, lo, hi int) (int, error) {
255 v := getenv(key)
256 if v == "" {
257 return def, nil
258 }
259 n, err := strconv.Atoi(strings.TrimSpace(v))
260 if err != nil || n < lo || n > hi {
261 return 0, fmt.Errorf("%s: %q is not an integer in [%d, %d]", key, v, lo, hi)
262 }
263 return n, nil
264}
265
266// ParseSizeMiB parses "4096", "512M", "4G", "64GiB", "1T" into MiB. A bare
267// number is MiB. Suffixes are binary (1G = 1024 MiB) and case-insensitive.
268func ParseSizeMiB(s string) (int, error) {
269 s = strings.ToLower(strings.TrimSpace(s))
270 s = strings.TrimSuffix(strings.TrimSuffix(s, "b"), "i")
271 mult := 1
272 if s != "" {
273 switch s[len(s)-1] {
274 case 'k':
275 mult, s = 0, s[:len(s)-1]
276 case 'm':
277 mult, s = 1, s[:len(s)-1]
278 case 'g':
279 mult, s = 1024, s[:len(s)-1]
280 case 't':
281 mult, s = 1024*1024, s[:len(s)-1]
282 }
283 }
284 n, err := strconv.Atoi(s)
285 if err != nil || n <= 0 || mult == 0 {
286 return 0, errors.New("invalid size")
287 }
288 if n > (1<<31-1)/mult {
289 return 0, errors.New("size too large")
290 }
291 return n * mult, nil
292}
293
294// ImageDir returns the VM image directory: Image when it is a directory path,
295// else <StateDir>/image.
296func (c Config) ImageDir() string {
297 if c.Image != "" && looksLikePath(c.Image) {
298 return c.Image
299 }
300 return filepath.Join(c.StateDir, "image")
301}
302
303// OCIImage returns the container image reference: Image when it is not a
304// directory path, else DefaultOCIImage.
305func (c Config) OCIImage() string {
306 if c.Image != "" && !looksLikePath(c.Image) {
307 return c.Image
308 }
309 return DefaultOCIImage
310}
311
312func looksLikePath(s string) bool {
313 return strings.HasPrefix(s, "/") || strings.HasPrefix(s, "./") || strings.HasPrefix(s, "../")
314}
315
316// KernelPath returns the guest kernel: Kernel, else <ImageDir>/vmlinux (x86-64)
317// or <ImageDir>/Image (arm64).
318func (c Config) KernelPath() string {
319 if c.Kernel != "" {
320 return c.Kernel
321 }
322 name := "vmlinux"
323 if runtime.GOARCH == "arm64" {
324 name = "Image"
325 }
326 return filepath.Join(c.ImageDir(), name)
327}
328
329// BaseImagePath returns the shared read-only base disk.
330func (c Config) BaseImagePath() string { return filepath.Join(c.ImageDir(), "base.ext4") }
331
332// ExecutablePath returns the binary attached to sandboxes.
333func (c Config) ExecutablePath() (string, error) {
334 if c.Executable != "" {
335 return c.Executable, nil
336 }
337 p, err := os.Executable()
338 if err != nil {
339 return "", fmt.Errorf("sandbox: locate own executable: %w", err)
340 }
341 return filepath.EvalSymlinks(p)
342}
343
344// Resolve fills the zero resource fields of a JobSpec from the config.
345func (c Config) Resolve(s JobSpec) JobSpec {
346 if s.CPUs <= 0 {
347 s.CPUs = c.JobCPUs
348 }
349 if s.MemoryMiB <= 0 {
350 s.MemoryMiB = c.JobMemoryMiB
351 }
352 if s.DiskMiB <= 0 {
353 s.DiskMiB = c.JobDiskMiB
354 }
355 return s
356}