1// SPDX-License-Identifier: MIT
2
3package sandbox
4
5import (
6 "errors"
7 "fmt"
8 "net/netip"
9 "os"
10 "path/filepath"
11 "runtime"
12 "strconv"
13 "strings"
14)
15
16// Environment variables read by LoadConfig. The other ERMITE_RUNNER_* variables
17// (URL, NAME, LABELS, ...) belong to the host process, not to the sandboxes.
18const (
19 EnvIsolation = "ERMITE_RUNNER_ISOLATION"
20 EnvStateDir = "ERMITE_RUNNER_STATE_DIR"
21 EnvCapacity = "ERMITE_RUNNER_CAPACITY"
22 EnvImage = "ERMITE_RUNNER_IMAGE"
23 EnvJobCPUs = "ERMITE_RUNNER_JOB_CPUS"
24 EnvJobMemory = "ERMITE_RUNNER_JOB_MEMORY"
25 EnvJobDisk = "ERMITE_RUNNER_JOB_DISK"
26 EnvUIDBase = "ERMITE_RUNNER_UID_BASE"
27 EnvNetwork = "ERMITE_RUNNER_NETWORK"
28 EnvFirecracker = "ERMITE_RUNNER_FIRECRACKER"
29 EnvJailer = "ERMITE_RUNNER_JAILER"
30 EnvJailerMode = "ERMITE_RUNNER_JAILER_MODE"
31 EnvKernel = "ERMITE_RUNNER_KERNEL"
32
33 EnvJobPids = "ERMITE_RUNNER_JOB_PIDS"
34 EnvDenyPrivileged = "ERMITE_RUNNER_DENY_PRIVILEGED"
35 EnvLockFile = "ERMITE_RUNNER_LOCK_FILE"
36 EnvJobDiskBW = "ERMITE_RUNNER_JOB_DISK_BANDWIDTH"
37 EnvJobDiskIOPS = "ERMITE_RUNNER_JOB_DISK_IOPS"
38 EnvJobNetBandwidth = "ERMITE_RUNNER_JOB_NET_BANDWIDTH"
39)
40
41// Defaults.
42const (
43 DefaultStateDir = "/var/lib/ermite-runner"
44 DefaultOCIImage = "ermite-runner-ubuntu-24.04:latest"
45 DefaultUIDBase = 900000
46 DefaultNetwork = "10.200.0.0/16"
47 DefaultCPUs = 2
48 DefaultMemoryMiB = 4096
49 DefaultDiskMiB = 64 * 1024
50 DefaultJobPids = 32768
51 // DefaultLockFile is the single-instance lock of the host: reaping removes
52 // everything the runner labels as its own, so two runners (even with
53 // different state directories, or one in a container) must never coexist.
54 DefaultLockFile = "/run/ermite-runner.lock"
55 // Rate limits of a micro-VM, generous so that they only stop a runaway job
56 // from starving the host: 400 MiB/s and 20000 IOPS per disk, 1 Gbit/s per
57 // network direction. Zero disables a limit.
58 DefaultDiskBandwidthMiB = 400
59 DefaultDiskIOPS = 20000
60 DefaultNetMbit = 1000
61)
62
63// Config is the sandbox part of the runner configuration.
64type Config struct {
65 Isolation Mode
66 StateDir string
67 Capacity int
68
69 // Image is ERMITE_RUNNER_IMAGE: a directory (VM: holds the kernel and
70 // base.ext4) or an OCI reference (container). A value that is empty or does
71 // not fit the selected mode falls back to the default for that mode.
72 Image string
73
74 JobCPUs int
75 JobMemoryMiB int
76 JobDiskMiB int
77
78 // UIDBase+slot is the uid and gid the jailer drops to.
79 UIDBase int
80 // Network is the pool of /30 subnets, one per slot (VM TAPs, container
81 // bridges).
82 Network netip.Prefix
83
84 // Firecracker and Jailer are binary paths; empty means look in PATH.
85 Firecracker string
86 Jailer string
87 // Kernel overrides <image dir>/vmlinux (Image on arm64).
88 Kernel string
89 // JailerOff runs Firecracker without the jailer (development and tests as
90 // an unprivileged user). Never use it in production: it only drops the
91 // chroot, uid and namespace confinement of the VMM process.
92 JailerOff bool
93
94 // KVMPath is the KVM device; tests point it elsewhere to hide /dev/kvm.
95 KVMPath string
96 // Executable is attached read-only to every sandbox. Empty means
97 // os.Executable(). It must be a static binary (CGO_ENABLED=0).
98 Executable string
99 // DNSPort is the port of the DNS forwarder; 53 in production. Tests that
100 // cannot bind it use another port with no guest.
101 DNSPort int
102
103 // JobPids is the process limit of a job (container PidsLimit, VM pids.max
104 // is derived from the vCPU count and is not affected).
105 JobPids int
106 // DenyPrivileged refuses the container mode when it would have to run
107 // privileged (no sysbox).
108 DenyPrivileged bool
109 // LockFile is the host-wide single-instance lock.
110 LockFile string
111 // Rate limits of micro-VMs; zero disables.
112 DiskBandwidthMiB, DiskIOPS, NetMbit int
113}
114
115// LoadConfig reads the sandbox settings from getenv (os.Getenv in production).
116func LoadConfig(getenv func(string) string) (Config, error) {
117 c := Config{
118 Isolation: Auto,
119 StateDir: DefaultStateDir,
120 Capacity: 1,
121 JobCPUs: DefaultCPUs,
122 JobMemoryMiB: DefaultMemoryMiB,
123 JobDiskMiB: DefaultDiskMiB,
124 UIDBase: DefaultUIDBase,
125 KVMPath: "/dev/kvm",
126 DNSPort: 53,
127 JobPids: DefaultJobPids,
128 LockFile: DefaultLockFile,
129
130 DiskBandwidthMiB: DefaultDiskBandwidthMiB,
131 DiskIOPS: DefaultDiskIOPS,
132 NetMbit: DefaultNetMbit,
133 }
134 var err error
135 if v := getenv(EnvIsolation); v != "" {
136 switch m := Mode(strings.ToLower(strings.TrimSpace(v))); m {
137 case Auto, VM, Container:
138 c.Isolation = m
139 default:
140 return c, fmt.Errorf("%s: %q is not auto, vm or container", EnvIsolation, v)
141 }
142 }
143 if v := getenv(EnvStateDir); v != "" {
144 c.StateDir = v
145 }
146 if !filepath.IsAbs(c.StateDir) {
147 return c, fmt.Errorf("%s: %q must be an absolute path", EnvStateDir, c.StateDir)
148 }
149 if c.Capacity, err = intEnv(getenv, EnvCapacity, c.Capacity, 1, 1024); err != nil {
150 return c, err
151 }
152 c.Image = getenv(EnvImage)
153 if c.JobCPUs, err = intEnv(getenv, EnvJobCPUs, c.JobCPUs, 1, 128); err != nil {
154 return c, err
155 }
156 if v := getenv(EnvJobMemory); v != "" {
157 if c.JobMemoryMiB, err = ParseSizeMiB(v); err != nil || c.JobMemoryMiB < 256 {
158 return c, fmt.Errorf("%s: %q: need a size of at least 256M", EnvJobMemory, v)
159 }
160 }
161 if v := getenv(EnvJobDisk); v != "" {
162 if c.JobDiskMiB, err = ParseSizeMiB(v); err != nil || c.JobDiskMiB < 1024 {
163 return c, fmt.Errorf("%s: %q: need a size of at least 1G", EnvJobDisk, v)
164 }
165 }
166 if c.UIDBase, err = intEnv(getenv, EnvUIDBase, c.UIDBase, 1000, 4_000_000_000); err != nil {
167 return c, err
168 }
169 if c.UIDBase+c.Capacity > 4_294_967_294 {
170 return c, fmt.Errorf("%s: %d + capacity overflows the uid range", EnvUIDBase, c.UIDBase)
171 }
172 net := DefaultNetwork
173 if v := getenv(EnvNetwork); v != "" {
174 net = v
175 }
176 if c.Network, err = netip.ParsePrefix(net); err != nil || !c.Network.Addr().Is4() || c.Network.Bits() > 30 {
177 return c, fmt.Errorf("%s: %q is not an IPv4 prefix of at most /30", EnvNetwork, net)
178 }
179 if 1<<(30-c.Network.Bits()) < c.Capacity {
180 return c, fmt.Errorf("%s: %s has fewer /30 subnets than the capacity %d", EnvNetwork, c.Network, c.Capacity)
181 }
182 if c.JobPids, err = intEnv(getenv, EnvJobPids, c.JobPids, 64, 4_194_304); err != nil {
183 return c, err
184 }
185 switch strings.ToLower(strings.TrimSpace(getenv(EnvDenyPrivileged))) {
186 case "", "0", "false", "no":
187 case "1", "true", "yes":
188 c.DenyPrivileged = true
189 default:
190 return c, fmt.Errorf("%s: %q is not 0 or 1", EnvDenyPrivileged, getenv(EnvDenyPrivileged))
191 }
192 if v := getenv(EnvLockFile); v != "" {
193 if !filepath.IsAbs(v) {
194 return c, fmt.Errorf("%s: %q must be an absolute path", EnvLockFile, v)
195 }
196 c.LockFile = v
197 }
198 if c.DiskBandwidthMiB, err = intEnv(getenv, EnvJobDiskBW, c.DiskBandwidthMiB, 0, 1_000_000); err != nil {
199 return c, err
200 }
201 if c.DiskIOPS, err = intEnv(getenv, EnvJobDiskIOPS, c.DiskIOPS, 0, 100_000_000); err != nil {
202 return c, err
203 }
204 if c.NetMbit, err = intEnv(getenv, EnvJobNetBandwidth, c.NetMbit, 0, 1_000_000); err != nil {
205 return c, err
206 }
207 c.Firecracker = getenv(EnvFirecracker)
208 c.Jailer = getenv(EnvJailer)
209 c.Kernel = getenv(EnvKernel)
210 switch strings.ToLower(getenv(EnvJailerMode)) {
211 case "", "on":
212 case "off":
213 c.JailerOff = true
214 default:
215 return c, fmt.Errorf("%s: %q is not on or off", EnvJailerMode, getenv(EnvJailerMode))
216 }
217 return c, nil
218}
219
220func intEnv(getenv func(string) string, key string, def, lo, hi int) (int, error) {
221 v := getenv(key)
222 if v == "" {
223 return def, nil
224 }
225 n, err := strconv.Atoi(strings.TrimSpace(v))
226 if err != nil || n < lo || n > hi {
227 return 0, fmt.Errorf("%s: %q is not an integer in [%d, %d]", key, v, lo, hi)
228 }
229 return n, nil
230}
231
232// ParseSizeMiB parses "4096", "512M", "4G", "64GiB", "1T" into MiB. A bare
233// number is MiB. Suffixes are binary (1G = 1024 MiB) and case-insensitive.
234func ParseSizeMiB(s string) (int, error) {
235 s = strings.ToLower(strings.TrimSpace(s))
236 s = strings.TrimSuffix(strings.TrimSuffix(s, "b"), "i")
237 mult := 1
238 if s != "" {
239 switch s[len(s)-1] {
240 case 'k':
241 mult, s = 0, s[:len(s)-1]
242 case 'm':
243 mult, s = 1, s[:len(s)-1]
244 case 'g':
245 mult, s = 1024, s[:len(s)-1]
246 case 't':
247 mult, s = 1024*1024, s[:len(s)-1]
248 }
249 }
250 n, err := strconv.Atoi(s)
251 if err != nil || n <= 0 || mult == 0 {
252 return 0, errors.New("invalid size")
253 }
254 if n > (1<<31-1)/mult {
255 return 0, errors.New("size too large")
256 }
257 return n * mult, nil
258}
259
260// ImageDir returns the VM image directory: Image when it is a directory path,
261// else <StateDir>/image.
262func (c Config) ImageDir() string {
263 if c.Image != "" && looksLikePath(c.Image) {
264 return c.Image
265 }
266 return filepath.Join(c.StateDir, "image")
267}
268
269// OCIImage returns the container image reference: Image when it is not a
270// directory path, else DefaultOCIImage.
271func (c Config) OCIImage() string {
272 if c.Image != "" && !looksLikePath(c.Image) {
273 return c.Image
274 }
275 return DefaultOCIImage
276}
277
278func looksLikePath(s string) bool {
279 return strings.HasPrefix(s, "/") || strings.HasPrefix(s, "./") || strings.HasPrefix(s, "../")
280}
281
282// KernelPath returns the guest kernel: Kernel, else <ImageDir>/vmlinux (x86-64)
283// or <ImageDir>/Image (arm64).
284func (c Config) KernelPath() string {
285 if c.Kernel != "" {
286 return c.Kernel
287 }
288 name := "vmlinux"
289 if runtime.GOARCH == "arm64" {
290 name = "Image"
291 }
292 return filepath.Join(c.ImageDir(), name)
293}
294
295// BaseImagePath returns the shared read-only base disk.
296func (c Config) BaseImagePath() string { return filepath.Join(c.ImageDir(), "base.ext4") }
297
298// ExecutablePath returns the binary attached to sandboxes.
299func (c Config) ExecutablePath() (string, error) {
300 if c.Executable != "" {
301 return c.Executable, nil
302 }
303 p, err := os.Executable()
304 if err != nil {
305 return "", fmt.Errorf("sandbox: locate own executable: %w", err)
306 }
307 return filepath.EvalSymlinks(p)
308}
309
310// Resolve fills the zero resource fields of a JobSpec from the config.
311func (c Config) Resolve(s JobSpec) JobSpec {
312 if s.CPUs <= 0 {
313 s.CPUs = c.JobCPUs
314 }
315 if s.MemoryMiB <= 0 {
316 s.MemoryMiB = c.JobMemoryMiB
317 }
318 if s.DiskMiB <= 0 {
319 s.DiskMiB = c.JobDiskMiB
320 }
321 return s
322}