1// SPDX-License-Identifier: MIT
2
3// Package integration holds the end-to-end tests of the sandbox drivers with the
4// real stub executor: output comes back, exit codes are reported, and nothing
5// survives 10 jobs, including killed ones. They need privileges and host
6// software, so they are opt-in:
7//
8// ERMITE_RUNNER_IT_VM=1 Firecracker + KVM + kernel + base image,
9// CAP_NET_ADMIN (TAP, nftables, port 53 on the
10// TAP address)
11// ERMITE_RUNNER_IT_CONTAINER=1 a ROOTFUL Docker daemon (DOCKER_HOST) and the
12// runner image, CAP_NET_ADMIN for nftables
13//
14// See README.md ("Integration tests") for the exact commands, including how to
15// run the VM test without root inside a user and network namespace.
16package integration
17
18import (
19 "context"
20 "fmt"
21 "os"
22 "os/exec"
23 "path/filepath"
24 "regexp"
25 "strings"
26 "testing"
27 "time"
28
29 "github.com/bnema/zerowrap"
30
31 "bnema.dev/ermite-runner/internal/execstream"
32 "bnema.dev/ermite-runner/internal/execstream/execpb"
33 "bnema.dev/ermite-runner/internal/netpolicy"
34 "bnema.dev/ermite-runner/internal/sandbox"
35 "bnema.dev/ermite-runner/internal/sandbox/drivers"
36 "bnema.dev/ermite-runner/internal/sandbox/vm"
37)
38
39// stubScript is the CI-1 stub job of the plan.
40const stubScript = "uname -a; id; cat /etc/os-release; docker info"
41
42func ctxWithLog(t *testing.T) context.Context {
43 return zerowrap.WithCtx(context.Background(), zerowrap.New(zerowrap.Config{Level: "info", Format: "console", Output: testWriter{t}}))
44}
45
46type testWriter struct{ t *testing.T }
47
48func (w testWriter) Write(p []byte) (int, error) {
49 w.t.Log(strings.TrimRight(string(p), "\n"))
50 return len(p), nil
51}
52
53// buildRunner builds the static runner executable the sandboxes attach.
54func buildRunner(t *testing.T) string {
55 t.Helper()
56 out, err := exec.Command("go", "list", "-m", "-f", "{{.Dir}}").Output()
57 if err != nil {
58 t.Fatal(err)
59 }
60 exe := filepath.Join(t.TempDir(), "ermite-runner")
61 cmd := exec.Command("go", "build", "-o", exe, "bnema.dev/ermite-runner")
62 cmd.Dir = strings.TrimSpace(string(out))
63 cmd.Env = append(os.Environ(), "CGO_ENABLED=0", "GOWORK=off")
64 if b, err := cmd.CombinedOutput(); err != nil {
65 t.Fatalf("build runner: %v\n%s", err, b)
66 }
67 return exe
68}
69
70// loadConfig reads the sandbox settings from the environment, like the runner
71// does, over a private state directory.
72func loadConfig(t *testing.T, mode sandbox.Mode) sandbox.Config {
73 t.Helper()
74 getenv := func(k string) string {
75 switch k {
76 case sandbox.EnvIsolation:
77 return string(mode)
78 case sandbox.EnvStateDir:
79 return stateDir(t)
80 case sandbox.EnvCapacity:
81 return "2"
82 case sandbox.EnvLockFile:
83 // A lock of the test's own unless the caller chose one: the default
84 // (/run) is not writable without root, and each test must be able to
85 // take it after the previous one released it.
86 if v := os.Getenv(k); v != "" {
87 return v
88 }
89 return filepath.Join(stateDir(t), "host.lock")
90 case sandbox.EnvJobDisk:
91 if v := os.Getenv(k); v != "" {
92 return v
93 }
94 return "4G"
95 case sandbox.EnvJobMemory:
96 if v := os.Getenv(k); v != "" {
97 return v
98 }
99 return "1024"
100 }
101 return os.Getenv(k)
102 }
103 cfg, err := sandbox.LoadConfig(getenv)
104 if err != nil {
105 t.Fatal(err)
106 }
107 cfg.Executable = buildRunner(t)
108 return cfg
109}
110
111var stateDirs = map[*testing.T]string{}
112
113func stateDir(t *testing.T) string {
114 if d, ok := stateDirs[t]; ok {
115 return d
116 }
117 base := os.Getenv("ERMITE_RUNNER_IT_STATE_DIR")
118 if base == "" {
119 base = t.TempDir()
120 } else {
121 base = filepath.Join(base, fmt.Sprintf("it-%d", time.Now().UnixNano()))
122 _ = os.MkdirAll(base, 0o755)
123 t.Cleanup(func() { _ = os.RemoveAll(base) })
124 }
125 stateDirs[t] = base
126 return base
127}
128
129type result struct {
130 out, errOut string
131 hello *execpb.Hello
132 exit *execpb.Exit
133}
134
135func runJob(ctx context.Context, conn sandbox.Conn, script string) (result, error) {
136 var so, se strings.Builder
137 hello, exit, err := execstream.Drive(conn, &execpb.Run{Script: script}, nil, func(o *execpb.Output) {
138 if o.Stream == execpb.StreamKind_STREAM_KIND_STDERR {
139 se.Write(o.Data)
140 } else {
141 so.Write(o.Data)
142 }
143 })
144 return result{so.String(), se.String(), hello, exit}, err
145}
146
147// need skips the test unless every command is available.
148func need(t *testing.T, cmds ...string) {
149 t.Helper()
150 for _, c := range cmds {
151 if _, err := exec.LookPath(c); err != nil {
152 t.Skipf("%s not found in PATH", c)
153 }
154 }
155}
156
157// needNetAdmin skips unless the process may create TAP devices and nftables
158// tables (root, CAP_NET_ADMIN, or a user+network namespace).
159func needNetAdmin(t *testing.T) {
160 t.Helper()
161 need(t, "ip", "nft")
162 const probe = "ertapprobe"
163 if out, err := exec.Command("ip", "tuntap", "add", "dev", probe, "mode", "tap").CombinedOutput(); err != nil {
164 t.Skipf("cannot create a TAP device (need CAP_NET_ADMIN): %v: %s", err, strings.TrimSpace(string(out)))
165 }
166 _ = exec.Command("ip", "link", "del", "dev", probe).Run()
167 if out, err := exec.Command("nft", "list", "tables").CombinedOutput(); err != nil {
168 t.Skipf("cannot use nftables (need CAP_NET_ADMIN): %v: %s", err, strings.TrimSpace(string(out)))
169 }
170}
171
172// audit fails the test when anything of a job survives: network devices, the
173// nftables table, state files, processes.
174func audit(t *testing.T, stateDir string) {
175 t.Helper()
176 deadline := time.Now().Add(5 * time.Second)
177 var left []string
178 for {
179 left = leftovers(stateDir)
180 if len(left) == 0 || time.Now().After(deadline) {
181 break
182 }
183 time.Sleep(100 * time.Millisecond)
184 }
185 if len(left) != 0 {
186 t.Fatalf("leftovers after the jobs:\n %s", strings.Join(left, "\n "))
187 }
188}
189
190var tapRe = regexp.MustCompile(`(?m)^\d+: (ertap\d+|erbr\d+)[:@]`)
191
192func leftovers(stateDir string) []string {
193 var left []string
194 if out, err := exec.Command("ip", "-o", "link", "show").Output(); err == nil {
195 for _, m := range tapRe.FindAllStringSubmatch(string(out), -1) {
196 left = append(left, "network device "+m[1])
197 }
198 }
199 if out, err := exec.Command("nft", "list", "tables").Output(); err == nil && strings.Contains(string(out), netpolicy.Table) {
200 left = append(left, "nftables table "+netpolicy.Table)
201 }
202 for _, d := range []string{"vm", "vms", "jail", "containers"} {
203 _ = filepath.WalkDir(filepath.Join(stateDir, d), func(p string, e os.DirEntry, err error) error {
204 if err == nil && p != filepath.Join(stateDir, d) {
205 rel, _ := filepath.Rel(stateDir, p)
206 // <state>/jail/<exec file name> is the jailer's static parent of the
207 // per-job chroots (like jail/ itself): not a leftover when empty.
208 if e.IsDir() && d == "jail" && filepath.Dir(rel) == "jail" {
209 return nil
210 }
211 left = append(left, "file "+rel)
212 }
213 return nil
214 })
215 }
216 for pid, id := range vmmProcs() {
217 left = append(left, fmt.Sprintf("process %d %s", pid, id))
218 }
219 return left
220}
221
222// vmmProcs returns the processes whose command line carries one of our VM ids
223// and a VMM marker (vm.IDFromCmdline: the rule the driver's scan is built on),
224// whatever the program is called. Deliberately not narrowed to the state
225// directory: the leak check wants to see any process of ours.
226func vmmProcs() map[int]string {
227 out := map[int]string{}
228 procs, _ := filepath.Glob("/proc/[0-9]*/cmdline")
229 for _, p := range procs {
230 b, err := os.ReadFile(p)
231 if err != nil {
232 continue
233 }
234 id := vm.IDFromCmdline(strings.Split(strings.TrimRight(string(b), "\x00"), "\x00"))
235 if id == "" {
236 continue
237 }
238 var pid int
239 if _, err := fmt.Sscanf(p, "/proc/%d/cmdline", &pid); err == nil {
240 out[pid] = id
241 }
242 }
243 return out
244}
245
246func vmmPIDs() []int {
247 var pids []int
248 for pid := range vmmProcs() {
249 pids = append(pids, pid)
250 }
251 return pids
252}
253
254func checkStubOutput(t *testing.T, r result, wantSandbox string) {
255 t.Helper()
256 if r.hello == nil || r.hello.Sandbox != wantSandbox {
257 t.Errorf("hello: %v", r.hello)
258 }
259 for _, want := range []string{"Linux", "uid=", "Ubuntu", "Server Version"} {
260 if !strings.Contains(r.out, want) {
261 t.Errorf("output lacks %q:\n%s\nstderr:\n%s", want, r.out, r.errOut)
262 }
263 }
264 if r.exit == nil || r.exit.Code != 0 {
265 t.Errorf("stub exit: %v\nstderr: %s", r.exit, r.errOut)
266 }
267}
268
269// jobs runs the common scenario on a driver: 10 jobs, 2 at a time where
270// capacity allows, including a failing job, a job killed from outside, and a
271// job abandoned by a "crashed" runner; then audits.
272func scenario(t *testing.T, ctx context.Context, cfg sandbox.Config, mode sandbox.Mode, killSandbox func(t *testing.T, sb sandbox.Sandbox)) {
273 t.Helper()
274 drv, err := sandbox.Select(ctx, cfg, drivers.Factories(), sandbox.Prober{})
275 if err != nil {
276 t.Fatal(err)
277 }
278 if drv.Mode() != mode {
279 t.Fatalf("driver mode %s, want %s", drv.Mode(), mode)
280 }
281 // Whatever happens below (a Fatal included), leave nothing behind on the host.
282 t.Cleanup(func() {
283 _ = drv.Close()
284 d2, err := sandbox.Select(context.Background(), cfg, drivers.Factories(), sandbox.Prober{})
285 if err != nil {
286 t.Logf("emergency cleanup: %v", err)
287 return
288 }
289 _ = d2.Reap(context.Background())
290 _ = d2.Close()
291 })
292 if err := drv.Reap(ctx); err != nil {
293 t.Fatalf("initial reap: %v", err)
294 }
295
296 start := func(id string) (sandbox.Sandbox, sandbox.Conn) {
297 sb, err := drv.New()
298 if err != nil {
299 t.Fatal(err)
300 }
301 conn, err := sb.Start(ctx, sandbox.JobSpec{JobID: id})
302 if err != nil {
303 _ = sb.Stop(ctx)
304 t.Fatalf("job %s: %v", id, err)
305 }
306 return sb, conn
307 }
308
309 // Jobs 1-7: the stub, in pairs (capacity 2) except the last.
310 for i := 1; i <= 7; i++ {
311 sb, conn := start(fmt.Sprintf("stub-%d", i))
312 r, err := runJob(ctx, conn, stubScript)
313 if err != nil {
314 t.Fatalf("job %d: %v", i, err)
315 }
316 checkStubOutput(t, r, string(mode))
317 if err := sb.Stop(ctx); err != nil {
318 t.Fatalf("job %d stop: %v", i, err)
319 }
320 }
321
322 // Job 8: a failing script: the exit code comes back.
323 {
324 sb, conn := start("failing")
325 r, err := runJob(ctx, conn, "echo before; echo oops >&2; exit 42")
326 if err != nil {
327 t.Fatal(err)
328 }
329 if r.exit.Code != 42 || !strings.Contains(r.out, "before") || !strings.Contains(r.errOut, "oops") {
330 t.Errorf("failing job: %+v", r)
331 }
332 if err := sb.Stop(ctx); err != nil {
333 t.Fatal(err)
334 }
335 }
336
337 // Job 9: killed from outside while running (the VMM or the container dies).
338 {
339 sb, conn := start("killed")
340 done := make(chan error, 1)
341 go func() { _, err := runJob(ctx, conn, "echo started; sleep 600"); done <- err }()
342 time.Sleep(2 * time.Second)
343 killSandbox(t, sb)
344 select {
345 case err := <-done:
346 if err == nil {
347 t.Error("a killed sandbox must make the session fail")
348 }
349 case <-time.After(30 * time.Second):
350 t.Error("session did not end after the sandbox was killed")
351 }
352 if err := sb.Stop(ctx); err != nil {
353 t.Fatalf("stop after kill: %v", err)
354 }
355 }
356
357 // Job 10: the runner itself dies mid-job (no Stop, driver closed); the next
358 // start reaps everything.
359 {
360 _, conn := start("abandoned")
361 go func() { _, _ = runJob(ctx, conn, "sleep 600") }()
362 time.Sleep(2 * time.Second)
363 _ = drv.Close() // what a crash does: locks and listeners vanish, the sandbox stays
364 _ = conn.Close()
365 drv2, err := sandbox.Select(ctx, cfg, drivers.Factories(), sandbox.Prober{})
366 if err != nil {
367 t.Fatal(err)
368 }
369 defer func() { _ = drv2.Close() }()
370 if err := drv2.Reap(ctx); err != nil {
371 t.Fatalf("reap after crash: %v", err)
372 }
373 }
374 audit(t, cfg.StateDir)
375}