1// SPDX-License-Identifier: MIT
2
3// Package sandbox defines the isolation boundary of a CI job: one Sandbox per
4// job, either a Firecracker micro-VM (package vm) or a container with its own
5// Docker daemon (package container). Both run the same executable
6// (`ermite-runner init`, then `ermite-runner exec`) and expose the same framed
7// stream (package execstream) to the host process.
8package sandbox
9
10import (
11 "context"
12 "errors"
13 "io"
14 "net/http"
15)
16
17// Mode is an isolation mode.
18type Mode string
19
20// Isolation modes. Auto is a setting only: a Driver always reports VM or
21// Container.
22const (
23 Auto Mode = "auto"
24 VM Mode = "vm"
25 Container Mode = "container"
26)
27
28// Isolation is the effective isolation of a driver, reported by the host at
29// registration and shown on the job page.
30type Isolation string
31
32// Effective isolations.
33const (
34 IsolationVM Isolation = "vm"
35 IsolationContainerSysbox Isolation = "container_sysbox"
36 IsolationContainerPrivileged Isolation = "container_privileged"
37)
38
39// ErrCapacity is returned by Driver.New when every slot is in use.
40var ErrCapacity = errors.New("sandbox: no free slot")
41
42// ErrLeaked is wrapped by the error of Start or Stop when the cleanup of a
43// sandbox is incomplete: something it created (process, network device,
44// firewall rules, files, containers) may still exist and its slot stays taken.
45// The host should stop assigning jobs to this runner slot, let running jobs
46// finish and call Driver.Reap, which retries the cleanup and frees the slot.
47var ErrLeaked = errors.New("sandbox: cleanup incomplete, resources leaked")
48
49// ErrBusy is returned by Driver.Reap while sandboxes are running: reaping
50// removes everything labelled as ours and would destroy them.
51var ErrBusy = errors.New("sandbox: cannot reap while sandboxes are running")
52
53// JobSpec describes the sandbox of one job. Zero resource fields take the
54// configured defaults.
55type JobSpec struct {
56 // JobID identifies the job in logs, labels and leftover reports. It never
57 // appears in file or interface names.
58 JobID string
59 // CPUs, MemoryMiB and DiskMiB size the sandbox (ERMITE_RUNNER_JOB_*).
60 CPUs int
61 MemoryMiB int
62 DiskMiB int
63 // Results, when not nil, is the results service of the job (artifacts,
64 // package resultssvc), served on the sandbox's cache channel beside the
65 // cache (Config.ServeCache).
66 Results http.Handler
67}
68
69// Conn is the executor stream: bytes in both directions, framed by execstream.
70// Closing it ends the session from the host side.
71type Conn io.ReadWriteCloser
72
73// Sandbox isolates one job. A Sandbox is single use: Start once, then Stop.
74type Sandbox interface {
75 // ID is the sandbox's identifier ("er" + 8 hex digits), assigned by
76 // Driver.New. It names the sandbox's files and Docker objects, so logs and
77 // leftover reports can be matched with them.
78 ID() string
79 // Start creates the sandbox, boots it and waits for the executor inside to
80 // connect back. It aborts when ctx ends or when Stop is called from another
81 // goroutine. On error everything created so far has been removed, or the
82 // error wraps ErrLeaked.
83 Start(ctx context.Context, spec JobSpec) (Conn, error)
84 // Stop destroys the sandbox and everything it created (processes, network
85 // devices, firewall rules, files). It is idempotent, safe after a failed or
86 // never-made Start, runs to completion even when ctx is already done, and
87 // aborts a Start in progress. When something could not be removed the error
88 // wraps ErrLeaked.
89 Stop(ctx context.Context) error
90}
91
92// Timed is implemented by sandboxes that measure their start phases.
93type Timed interface {
94 // StartTimings returns a copy of the phase durations (ms) of a successful
95 // Start, nil after a failed one. Call it only once Start has returned.
96 StartTimings() map[string]int64
97}
98
99// Driver creates sandboxes of one mode and reaps what a previous run left.
100type Driver interface {
101 // Mode is VM or Container.
102 Mode() Mode
103 // Isolation is the effective isolation (vm, container_sysbox,
104 // container_privileged), for registration with ermite.
105 Isolation() Isolation
106 // New returns a fresh Sandbox bound to a free slot (ErrCapacity if none).
107 // The slot is released by Stop, unless the cleanup leaked (ErrLeaked).
108 New() (Sandbox, error)
109 // Reap removes everything a previous run of the runner left behind
110 // (leftover processes, devices, firewall rules, containers, files) and
111 // frees the slots of leaked sandboxes. Call it once at startup, before any
112 // New, and after ErrLeaked when no sandbox is running; it returns ErrBusy
113 // while one is.
114 Reap(ctx context.Context) error
115 // Close releases the driver's own resources (DNS forwarder, API client,
116 // host lock). Sandboxes still running are not stopped.
117 Close() error
118}
119
120// Slots hands out small integers, lowest first. Slots identify a sandbox's uid,
121// network device and /30 subnet, so two live sandboxes never collide. A slot
122// whose sandbox leaked stays taken (its device name and subnet may still be in
123// use) until ReleaseLeaked.
124type Slots struct {
125 used []bool
126 leaked []bool
127}
128
129// NewSlots returns a pool of n slots.
130func NewSlots(n int) *Slots { return &Slots{used: make([]bool, n), leaked: make([]bool, n)} }
131
132// Acquire returns the lowest free slot, or ErrCapacity. Callers serialise
133// access (the drivers hold their mutex).
134func (s *Slots) Acquire() (int, error) {
135 for i, u := range s.used {
136 if !u {
137 s.used[i] = true
138 return i, nil
139 }
140 }
141 return 0, ErrCapacity
142}
143
144// Release frees a slot; releasing a free or unknown slot is a no-op.
145func (s *Slots) Release(i int) {
146 if i >= 0 && i < len(s.used) {
147 s.used[i], s.leaked[i] = false, false
148 }
149}
150
151// MarkLeaked keeps slot i taken after a failed cleanup.
152func (s *Slots) MarkLeaked(i int) {
153 if i >= 0 && i < len(s.used) {
154 s.used[i], s.leaked[i] = true, true
155 }
156}
157
158// ReleaseLeaked frees every leaked slot: the caller has reaped what they held.
159func (s *Slots) ReleaseLeaked() {
160 for i := range s.used {
161 if s.leaked[i] {
162 s.used[i], s.leaked[i] = false, false
163 }
164 }
165}
166
167// InUse reports how many slots are taken, leaked ones included.
168func (s *Slots) InUse() int {
169 n := 0
170 for _, u := range s.used {
171 if u {
172 n++
173 }
174 }
175 return n
176}
177
178// Busy reports how many slots belong to a sandbox that is not leaked: running,
179// starting, or handed out by New and not yet stopped.
180func (s *Slots) Busy() int {
181 n := 0
182 for i, u := range s.used {
183 if u && !s.leaked[i] {
184 n++
185 }
186 }
187 return n
188}