1// SPDX-License-Identifier: MIT
2
3// Package sandbox defines the isolation boundary of a CI job: one Sandbox per
4// job, either a Firecracker micro-VM (package vm) or a container with its own
5// Docker daemon (package container). Both run the same executable
6// (`ermite-runner init`, then `ermite-runner exec`) and expose the same framed
7// stream (package execstream) to the host process.
8package sandbox
9
10import (
11 "context"
12 "errors"
13 "io"
14)
15
16// Mode is an isolation mode.
17type Mode string
18
19// Isolation modes. Auto is a setting only: a Driver always reports VM or
20// Container.
21const (
22 Auto Mode = "auto"
23 VM Mode = "vm"
24 Container Mode = "container"
25)
26
27// Isolation is the effective isolation of a driver, reported by the host at
28// registration and shown on the job page.
29type Isolation string
30
31// Effective isolations.
32const (
33 IsolationVM Isolation = "vm"
34 IsolationContainerSysbox Isolation = "container_sysbox"
35 IsolationContainerPrivileged Isolation = "container_privileged"
36)
37
38// ErrCapacity is returned by Driver.New when every slot is in use.
39var ErrCapacity = errors.New("sandbox: no free slot")
40
41// ErrLeaked is wrapped by the error of Start or Stop when the cleanup of a
42// sandbox is incomplete: something it created (process, network device,
43// firewall rules, files, containers) may still exist and its slot stays taken.
44// The host should stop assigning jobs to this runner slot, let running jobs
45// finish and call Driver.Reap, which retries the cleanup and frees the slot.
46var ErrLeaked = errors.New("sandbox: cleanup incomplete, resources leaked")
47
48// ErrBusy is returned by Driver.Reap while sandboxes are running: reaping
49// removes everything labelled as ours and would destroy them.
50var ErrBusy = errors.New("sandbox: cannot reap while sandboxes are running")
51
52// JobSpec describes the sandbox of one job. Zero resource fields take the
53// configured defaults.
54type JobSpec struct {
55 // JobID identifies the job in logs, labels and leftover reports. It never
56 // appears in file or interface names.
57 JobID string
58 // CPUs, MemoryMiB and DiskMiB size the sandbox (ERMITE_RUNNER_JOB_*).
59 CPUs int
60 MemoryMiB int
61 DiskMiB int
62}
63
64// Conn is the executor stream: bytes in both directions, framed by execstream.
65// Closing it ends the session from the host side.
66type Conn io.ReadWriteCloser
67
68// Sandbox isolates one job. A Sandbox is single use: Start once, then Stop.
69type Sandbox interface {
70 // ID is the sandbox's identifier ("er" + 8 hex digits), assigned by
71 // Driver.New. It names the sandbox's files and Docker objects, so logs and
72 // leftover reports can be matched with them.
73 ID() string
74 // Start creates the sandbox, boots it and waits for the executor inside to
75 // connect back. It aborts when ctx ends or when Stop is called from another
76 // goroutine. On error everything created so far has been removed, or the
77 // error wraps ErrLeaked.
78 Start(ctx context.Context, spec JobSpec) (Conn, error)
79 // Stop destroys the sandbox and everything it created (processes, network
80 // devices, firewall rules, files). It is idempotent, safe after a failed or
81 // never-made Start, runs to completion even when ctx is already done, and
82 // aborts a Start in progress. When something could not be removed the error
83 // wraps ErrLeaked.
84 Stop(ctx context.Context) error
85}
86
87// Driver creates sandboxes of one mode and reaps what a previous run left.
88type Driver interface {
89 // Mode is VM or Container.
90 Mode() Mode
91 // Isolation is the effective isolation (vm, container_sysbox,
92 // container_privileged), for registration with ermite.
93 Isolation() Isolation
94 // New returns a fresh Sandbox bound to a free slot (ErrCapacity if none).
95 // The slot is released by Stop, unless the cleanup leaked (ErrLeaked).
96 New() (Sandbox, error)
97 // Reap removes everything a previous run of the runner left behind
98 // (leftover processes, devices, firewall rules, containers, files) and
99 // frees the slots of leaked sandboxes. Call it once at startup, before any
100 // New, and after ErrLeaked when no sandbox is running; it returns ErrBusy
101 // while one is.
102 Reap(ctx context.Context) error
103 // Close releases the driver's own resources (DNS forwarder, API client,
104 // host lock). Sandboxes still running are not stopped.
105 Close() error
106}
107
108// Slots hands out small integers, lowest first. Slots identify a sandbox's uid,
109// network device and /30 subnet, so two live sandboxes never collide. A slot
110// whose sandbox leaked stays taken (its device name and subnet may still be in
111// use) until ReleaseLeaked.
112type Slots struct {
113 used []bool
114 leaked []bool
115}
116
117// NewSlots returns a pool of n slots.
118func NewSlots(n int) *Slots { return &Slots{used: make([]bool, n), leaked: make([]bool, n)} }
119
120// Acquire returns the lowest free slot, or ErrCapacity. Callers serialise
121// access (the drivers hold their mutex).
122func (s *Slots) Acquire() (int, error) {
123 for i, u := range s.used {
124 if !u {
125 s.used[i] = true
126 return i, nil
127 }
128 }
129 return 0, ErrCapacity
130}
131
132// Release frees a slot; releasing a free or unknown slot is a no-op.
133func (s *Slots) Release(i int) {
134 if i >= 0 && i < len(s.used) {
135 s.used[i], s.leaked[i] = false, false
136 }
137}
138
139// MarkLeaked keeps slot i taken after a failed cleanup.
140func (s *Slots) MarkLeaked(i int) {
141 if i >= 0 && i < len(s.used) {
142 s.used[i], s.leaked[i] = true, true
143 }
144}
145
146// ReleaseLeaked frees every leaked slot: the caller has reaped what they held.
147func (s *Slots) ReleaseLeaked() {
148 for i := range s.used {
149 if s.leaked[i] {
150 s.used[i], s.leaked[i] = false, false
151 }
152 }
153}
154
155// InUse reports how many slots are taken, leaked ones included.
156func (s *Slots) InUse() int {
157 n := 0
158 for _, u := range s.used {
159 if u {
160 n++
161 }
162 }
163 return n
164}
165
166// Busy reports how many slots belong to a sandbox that is not leaked: running,
167// starting, or handed out by New and not yet stopped.
168func (s *Slots) Busy() int {
169 n := 0
170 for i, u := range s.used {
171 if u && !s.leaked[i] {
172 n++
173 }
174 }
175 return n
176}