1// SPDX-License-Identifier: MIT
2
3// Package sandbox defines the isolation boundary of a CI job: one Sandbox per
4// job, either a Firecracker micro-VM (package vm) or a container with its own
5// Docker daemon (package container). Both run the same executable
6// (`ermite-runner init`, then `ermite-runner exec`) and expose the same framed
7// stream (package execstream) to the host process.
8package sandbox
9
10import (
11 "context"
12 "errors"
13 "io"
14 "net/http"
15)
16
17// Mode is an isolation mode.
18type Mode string
19
20// Isolation modes. Auto is a setting only: a Driver always reports VM or
21// Container.
22const (
23 Auto Mode = "auto"
24 VM Mode = "vm"
25 Container Mode = "container"
26)
27
28// Isolation is the effective isolation of a driver, reported by the host at
29// registration and shown on the job page.
30type Isolation string
31
32// Effective isolations.
33const (
34 IsolationVM Isolation = "vm"
35 IsolationContainerSysbox Isolation = "container_sysbox"
36 IsolationContainerPrivileged Isolation = "container_privileged"
37)
38
39// ErrCapacity is returned by Driver.New when every slot is in use.
40var ErrCapacity = errors.New("sandbox: no free slot")
41
42// ErrLeaked is wrapped by the error of Start or Stop when the cleanup of a
43// sandbox is incomplete: something it created (process, network device,
44// firewall rules, files, containers) may still exist and its slot stays taken.
45// The host should stop assigning jobs to this runner slot, let running jobs
46// finish and call Driver.Reap, which retries the cleanup and frees the slot.
47var ErrLeaked = errors.New("sandbox: cleanup incomplete, resources leaked")
48
49// ErrBusy is returned by Driver.Reap while sandboxes are running: reaping
50// removes everything labelled as ours and would destroy them.
51var ErrBusy = errors.New("sandbox: cannot reap while sandboxes are running")
52
53// JobSpec describes the sandbox of one job. Zero resource fields take the
54// configured defaults.
55type JobSpec struct {
56 // JobID identifies the job in logs, labels and leftover reports. It never
57 // appears in file or interface names.
58 JobID string
59 // CPUs, MemoryMiB and DiskMiB size the sandbox (ERMITE_RUNNER_JOB_*).
60 CPUs int
61 MemoryMiB int
62 DiskMiB int
63 // Results, when not nil, is the results service of the job (artifacts,
64 // package resultssvc), served on the sandbox's cache channel beside the
65 // cache (Config.ServeCache).
66 Results http.Handler
67}
68
69// Conn is the executor stream: bytes in both directions, framed by execstream.
70// Closing it ends the session from the host side.
71type Conn io.ReadWriteCloser
72
73// Sandbox isolates one job. A Sandbox is single use: Start once, then Stop.
74type Sandbox interface {
75 // ID is the sandbox's identifier ("er" + 8 hex digits), assigned by
76 // Driver.New. It names the sandbox's files and Docker objects, so logs and
77 // leftover reports can be matched with them.
78 ID() string
79 // Start creates the sandbox, boots it and waits for the executor inside to
80 // connect back. It aborts when ctx ends or when Stop is called from another
81 // goroutine. On error everything created so far has been removed, or the
82 // error wraps ErrLeaked.
83 Start(ctx context.Context, spec JobSpec) (Conn, error)
84 // Stop destroys the sandbox and everything it created (processes, network
85 // devices, firewall rules, files). It is idempotent, safe after a failed or
86 // never-made Start, runs to completion even when ctx is already done, and
87 // aborts a Start in progress. When something could not be removed the error
88 // wraps ErrLeaked.
89 Stop(ctx context.Context) error
90}
91
92// Driver creates sandboxes of one mode and reaps what a previous run left.
93type Driver interface {
94 // Mode is VM or Container.
95 Mode() Mode
96 // Isolation is the effective isolation (vm, container_sysbox,
97 // container_privileged), for registration with ermite.
98 Isolation() Isolation
99 // New returns a fresh Sandbox bound to a free slot (ErrCapacity if none).
100 // The slot is released by Stop, unless the cleanup leaked (ErrLeaked).
101 New() (Sandbox, error)
102 // Reap removes everything a previous run of the runner left behind
103 // (leftover processes, devices, firewall rules, containers, files) and
104 // frees the slots of leaked sandboxes. Call it once at startup, before any
105 // New, and after ErrLeaked when no sandbox is running; it returns ErrBusy
106 // while one is.
107 Reap(ctx context.Context) error
108 // Close releases the driver's own resources (DNS forwarder, API client,
109 // host lock). Sandboxes still running are not stopped.
110 Close() error
111}
112
113// Slots hands out small integers, lowest first. Slots identify a sandbox's uid,
114// network device and /30 subnet, so two live sandboxes never collide. A slot
115// whose sandbox leaked stays taken (its device name and subnet may still be in
116// use) until ReleaseLeaked.
117type Slots struct {
118 used []bool
119 leaked []bool
120}
121
122// NewSlots returns a pool of n slots.
123func NewSlots(n int) *Slots { return &Slots{used: make([]bool, n), leaked: make([]bool, n)} }
124
125// Acquire returns the lowest free slot, or ErrCapacity. Callers serialise
126// access (the drivers hold their mutex).
127func (s *Slots) Acquire() (int, error) {
128 for i, u := range s.used {
129 if !u {
130 s.used[i] = true
131 return i, nil
132 }
133 }
134 return 0, ErrCapacity
135}
136
137// Release frees a slot; releasing a free or unknown slot is a no-op.
138func (s *Slots) Release(i int) {
139 if i >= 0 && i < len(s.used) {
140 s.used[i], s.leaked[i] = false, false
141 }
142}
143
144// MarkLeaked keeps slot i taken after a failed cleanup.
145func (s *Slots) MarkLeaked(i int) {
146 if i >= 0 && i < len(s.used) {
147 s.used[i], s.leaked[i] = true, true
148 }
149}
150
151// ReleaseLeaked frees every leaked slot: the caller has reaped what they held.
152func (s *Slots) ReleaseLeaked() {
153 for i := range s.used {
154 if s.leaked[i] {
155 s.used[i], s.leaked[i] = false, false
156 }
157 }
158}
159
160// InUse reports how many slots are taken, leaked ones included.
161func (s *Slots) InUse() int {
162 n := 0
163 for _, u := range s.used {
164 if u {
165 n++
166 }
167 }
168 return n
169}
170
171// Busy reports how many slots belong to a sandbox that is not leaked: running,
172// starting, or handed out by New and not yet stopped.
173func (s *Slots) Busy() int {
174 n := 0
175 for i, u := range s.used {
176 if u && !s.leaked[i] {
177 n++
178 }
179 }
180 return n
181}