1// SPDX-License-Identifier: MIT
2
3package vm
4
5import (
6 "bytes"
7 "context"
8 "encoding/json"
9 "errors"
10 "fmt"
11 "io"
12 "io/fs"
13 "net"
14 "net/http"
15 "net/netip"
16 "os"
17 "os/exec"
18 "path/filepath"
19 "strings"
20 "sync"
21 "syscall"
22 "testing"
23 "time"
24
25 "github.com/bnema/zerowrap"
26 "golang.org/x/sys/unix"
27
28 "bnema.dev/ermite-runner/internal/cmdrun"
29 "bnema.dev/ermite-runner/internal/execstream"
30 "bnema.dev/ermite-runner/internal/execstream/execpb"
31 "bnema.dev/ermite-runner/internal/sandbox"
32 "bnema.dev/ermite-runner/internal/sandbox/disk"
33 "bnema.dev/ermite-runner/internal/sandbox/sandboxtest"
34)
35
36// The tests run a fake Firecracker: this test binary, re-executed through a
37// shell script named "firecracker" with FC_HELPER set. It serves the HTTP API on
38// the unix socket, records every request, and on InstanceStart plays the guest:
39// it connects to the vsock socket and speaks the executor protocol.
40func TestMain(m *testing.M) {
41 if os.Getenv("FC_HELPER") != "" {
42 fakeFirecracker()
43 return
44 }
45 os.Exit(m.Run())
46}
47
48func fakeFirecracker() {
49 var sock, id string
50 args := os.Args[1:]
51 for i := 0; i+1 < len(args); i++ {
52 switch args[i] {
53 case "--api-sock":
54 sock = args[i+1]
55 case "--id":
56 id = args[i+1]
57 }
58 }
59 _ = id
60 mode := os.Getenv("FC_MODE")
61 if mode == "die" {
62 fmt.Fprintln(os.Stderr, "fake firecracker: refusing to start")
63 os.Exit(3)
64 }
65 logf, _ := os.OpenFile(os.Getenv("FC_LOG"), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
66 var vsock string
67 mux := http.NewServeMux()
68 mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
69 body, _ := io.ReadAll(r.Body)
70 if r.Method == http.MethodPut {
71 _, _ = fmt.Fprintf(logf, "%s %s %s\n", r.Method, r.URL.Path, body)
72 }
73 if r.URL.Path == "/vsock" {
74 var v vsockDevice
75 _ = json.Unmarshal(body, &v)
76 vsock = v.UDSPath
77 }
78 w.WriteHeader(http.StatusNoContent)
79 if r.URL.Path == "/actions" && mode != "hang" {
80 go func() {
81 c, err := net.Dial("unix", fmt.Sprintf("%s_%d", vsock, sandbox.VsockPort))
82 if err != nil {
83 _, _ = fmt.Fprintln(logf, "vsock dial:", err)
84 return
85 }
86 defer func() { _ = c.Close() }()
87 g := execstream.NewConn(c)
88 _ = g.Send(&execpb.Frame{Body: &execpb.Frame_Hello{Hello: &execpb.Hello{Sandbox: "vm"}}})
89 if _, err := g.Recv(); err != nil {
90 return
91 }
92 _ = g.Send(&execpb.Frame{Body: &execpb.Frame_Output{Output: &execpb.Output{Data: []byte("hello from fake guest\n")}}})
93 _ = g.Send(&execpb.Frame{Body: &execpb.Frame_Exit{Exit: &execpb.Exit{Code: 0}}})
94 // Like a real VM, stay alive until killed.
95 select {}
96 }()
97 }
98 })
99 ln, err := net.Listen("unix", sock)
100 if err != nil {
101 fmt.Fprintln(os.Stderr, err)
102 os.Exit(2)
103 }
104 _ = http.Serve(ln, mux)
105}
106
107type env struct {
108 cfg sandbox.Config
109 run *cmdrun.Fake
110 plumb *sandboxtest.Plumbing
111 log string
112 taps map[string]bool
113 mu sync.Mutex
114}
115
116// newEnv prepares a state dir, fake binaries, kernel and images.
117func newEnv(t *testing.T, mode string) *env { return newEnvNamed(t, mode, "firecracker") }
118
119// newEnvNamed runs the fake VMM under another program name (argv[0]).
120func newEnvNamed(t *testing.T, mode, argv0 string) *env {
121 t.Helper()
122 bash, err := exec.LookPath("bash")
123 if err != nil {
124 t.Skip("bash needed for the fake firecracker")
125 }
126 dir := t.TempDir()
127 e := &env{taps: map[string]bool{}, plumb: sandboxtest.NewPlumbing(), log: filepath.Join(dir, "fc.log")}
128 self, _ := os.Executable()
129 fc := filepath.Join(dir, "firecracker")
130 script := fmt.Sprintf("#!%s\nexport FC_HELPER=1 FC_MODE=%q FC_LOG=%q\nexec -a %q %q \"$@\"\n", bash, mode, e.log, argv0, self)
131 if err := os.WriteFile(fc, []byte(script), 0o755); err != nil {
132 t.Fatal(err)
133 }
134 img := filepath.Join(dir, "image")
135 _ = os.MkdirAll(img, 0o755)
136 for _, f := range []string{"vmlinux", "base.ext4"} {
137 _ = os.WriteFile(filepath.Join(img, f), []byte("x"), 0o644)
138 }
139 exe := filepath.Join(dir, "runner")
140 sandboxtest.WriteStaticELF(t, exe)
141 e.cfg = sandbox.Config{
142 Isolation: sandbox.VM, StateDir: filepath.Join(dir, "state"), Capacity: 2, Image: img,
143 JobCPUs: 1, JobMemoryMiB: 512, JobDiskMiB: 1024, UIDBase: 900000, KVMPath: "/dev/null",
144 Network: mustPrefix("10.200.0.0/16"), Firecracker: fc, JailerOff: true, Executable: exe, DNSPort: 0,
145 LockFile: filepath.Join(dir, "host.lock"), JobPids: 1000, DiskBandwidthMiB: 400, DiskIOPS: 20000, NetMbit: 1000,
146 }
147 e.run = &cmdrun.Fake{Handle: func(c cmdrun.Call) ([]byte, error) {
148 e.mu.Lock()
149 defer e.mu.Unlock()
150 switch {
151 case c.Name == "mkfs.ext4" && len(c.Args) == 1 && c.Args[0] == "-V":
152 return []byte("mke2fs 1.47.4 (6-Mar-2025)\n"), nil
153 case c.Name == "ip" && len(c.Args) > 1 && c.Args[0] == "tuntap":
154 e.taps[c.Args[3]] = true
155 case c.Name == "ip" && len(c.Args) > 2 && c.Args[0] == "link" && c.Args[1] == "del":
156 if !e.taps[c.Args[3]] {
157 return []byte("Cannot find device"), errors.New("exit status 1")
158 }
159 delete(e.taps, c.Args[3])
160 case c.Name == "ip" && len(c.Args) > 1 && c.Args[0] == "-o":
161 var b strings.Builder
162 i := 1
163 for t := range e.taps {
164 fmt.Fprintf(&b, "%d: %s: <BROADCAST> mtu 1500\n", i, t)
165 i++
166 }
167 return []byte(b.String()), nil
168 }
169 return nil, nil
170 }}
171 return e
172}
173
174func (e *env) driverScope() scope { return newScope(e.cfg.StateDir) }
175
176func (e *env) tapCount() int { e.mu.Lock(); defer e.mu.Unlock(); return len(e.taps) }
177
178func (e *env) driver(t *testing.T) *Driver {
179 t.Helper()
180 d, err := New(context.Background(), e.cfg, Options{Run: e.run, Net: e.plumb, BootTimeout: 5 * time.Second})
181 if err != nil {
182 t.Fatal(err)
183 }
184 t.Cleanup(func() { _ = d.Close() })
185 return d
186}
187
188// leftovers lists everything that must not survive a job.
189func (e *env) leftovers(t *testing.T) []string {
190 t.Helper()
191 var out []string
192 _ = filepath.WalkDir(e.cfg.StateDir, func(p string, d os.DirEntry, err error) error {
193 rel, _ := filepath.Rel(e.cfg.StateDir, p)
194 switch {
195 case err != nil || rel == ".":
196 case strings.HasPrefix(rel, "vm/") || strings.HasPrefix(rel, "vms/") || strings.HasPrefix(rel, "jail/"):
197 out = append(out, rel)
198 }
199 return nil
200 })
201 if n := e.tapCount(); n != 0 {
202 out = append(out, fmt.Sprintf("%d TAP devices", n))
203 }
204 if n := e.plumb.Live(); n != 0 {
205 out = append(out, fmt.Sprintf("%d attached guests", n))
206 }
207 return out
208}
209
210func runStub(t *testing.T, conn sandbox.Conn) (string, int) {
211 t.Helper()
212 var out strings.Builder
213 _, exit, err := execstream.Drive(conn, &execpb.Run{}, nil, func(o *execpb.Output) { out.Write(o.Data) })
214 if err != nil {
215 t.Fatal(err)
216 }
217 return out.String(), int(exit.Code)
218}
219
220func TestStartRunStop(t *testing.T) {
221 e := newEnv(t, "")
222 d := e.driver(t)
223 sb, err := d.New()
224 if err != nil {
225 t.Fatal(err)
226 }
227 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "job-1"})
228 if err != nil {
229 t.Fatal(err)
230 }
231 timings := sb.(sandbox.Timed).StartTimings()
232 for _, key := range []string{"tap_ms", "net_ms", "vmm_ms", "files_ms", "scratch_ms", "vsock_ms", "configure_ms", "connect_ms"} {
233 if value, ok := timings[key]; !ok || value < 0 {
234 t.Errorf("%s = %d, present=%v", key, value, ok)
235 }
236 }
237 timings["tap_ms"] = -1
238 if sb.(sandbox.Timed).StartTimings()["tap_ms"] < 0 {
239 t.Error("shared timing map")
240 }
241 out, code := runStub(t, conn)
242 if out != "hello from fake guest\n" || code != 0 {
243 t.Fatalf("out=%q code=%d", out, code)
244 }
245 // While running: record, TAP, attachment, process.
246 if e.tapCount() != 1 || e.plumb.Live() != 1 {
247 t.Fatalf("tap=%d attached=%d", e.tapCount(), e.plumb.Live())
248 }
249 if recs, _ := d.store.list(); len(recs) != 1 || recs[0].Job != "job-1" || recs[0].PID == 0 || recs[0].Tap != "ertap0" {
250 t.Fatalf("records: %+v", recs)
251 }
252 pid := sb.(*vmSandbox).rec.PID
253
254 if err := sb.Stop(context.Background()); err != nil {
255 t.Fatal(err)
256 }
257 if left := e.leftovers(t); len(left) != 0 {
258 t.Fatalf("leftovers after Stop: %v", left)
259 }
260 if err := syscall.Kill(pid, 0); err == nil {
261 t.Fatalf("firecracker pid %d still alive", pid)
262 }
263 if err := sb.Stop(context.Background()); err != nil {
264 t.Fatalf("Stop must be idempotent: %v", err)
265 }
266 if d.slots.InUse() != 0 {
267 t.Fatal("slot not released")
268 }
269}
270
271// The API calls must follow the order Firecracker needs and carry the
272// plan's disk layout: exec disk (boot root, RO), base (RO), scratch (RW).
273func TestAPICallsAndDiskLayout(t *testing.T) {
274 e := newEnv(t, "")
275 d := e.driver(t)
276 sb, _ := d.New()
277 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j", CPUs: 3, MemoryMiB: 768, DiskMiB: 2048})
278 if err != nil {
279 t.Fatal(err)
280 }
281 runStub(t, conn)
282 defer func() { _ = sb.Stop(context.Background()) }()
283
284 b, _ := os.ReadFile(e.log)
285 lines := strings.Split(strings.TrimSpace(string(b)), "\n")
286 var paths []string
287 for _, l := range lines {
288 f := strings.Fields(l)
289 if len(f) >= 2 && f[0] == "PUT" {
290 paths = append(paths, f[1])
291 }
292 }
293 want := []string{"/machine-config", "/boot-source", "/drives/rootfs", "/drives/base", "/drives/scratch", "/network-interfaces/eth0", "/vsock", "/entropy", "/actions"}
294 if strings.Join(paths, ",") != strings.Join(want, ",") {
295 t.Fatalf("call order:\n got %v\nwant %v", paths, want)
296 }
297 log := string(b)
298 for _, frag := range []string{
299 `"vcpu_count":3`, `"mem_size_mib":768`, `"smt":false`,
300 `"drive_id":"rootfs"`, `"is_root_device":true,"is_read_only":true`,
301 `"drive_id":"base"`, `"is_root_device":false,"is_read_only":true`,
302 `"drive_id":"scratch"`, `"is_root_device":false,"is_read_only":false`,
303 `"host_dev_name":"ertap0"`, `"guest_cid":3`, `"action_type":"InstanceStart"`,
304 "ipv6.disable=1", "ermite.connect=vsock:1024", "ermite.gw=10.200.0.1", "ip=10.200.0.2::10.200.0.1:255.255.255.252",
305 } {
306 if !strings.Contains(log, frag) {
307 t.Errorf("API log lacks %q", frag)
308 }
309 }
310 // The scratch disk was created sparse at the requested size, and formatted.
311 var mkfs []string
312 for _, l := range e.run.Lines() {
313 if strings.HasPrefix(l, "mkfs.ext4") {
314 mkfs = append(mkfs, l)
315 }
316 }
317 if len(mkfs) == 0 {
318 t.Fatalf("no mkfs call: %v", e.run.Lines())
319 }
320}
321
322func TestJobsDoNotShareSlotsOrTaps(t *testing.T) {
323 e := newEnv(t, "")
324 d := e.driver(t)
325 var sbs []sandbox.Sandbox
326 for i := range 2 {
327 sb, err := d.New()
328 if err != nil {
329 t.Fatal(err)
330 }
331 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: fmt.Sprint(i)})
332 if err != nil {
333 t.Fatal(err)
334 }
335 runStub(t, conn)
336 sbs = append(sbs, sb)
337 }
338 if _, err := d.New(); !errors.Is(err, sandbox.ErrCapacity) {
339 t.Fatalf("capacity not enforced: %v", err)
340 }
341 if e.tapCount() != 2 {
342 t.Fatalf("taps: %v", e.taps)
343 }
344 for _, sb := range sbs {
345 _ = sb.Stop(context.Background())
346 }
347 if left := e.leftovers(t); len(left) != 0 {
348 t.Fatalf("leftovers: %v", left)
349 }
350}
351
352func TestStartFailureCleansEverything(t *testing.T) {
353 e := newEnv(t, "die")
354 d := e.driver(t)
355 sb, _ := d.New()
356 _, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
357 if err == nil || !strings.Contains(err.Error(), "refusing to start") {
358 t.Fatalf("want the VMM console in the error, got %v", err)
359 }
360 if left := e.leftovers(t); len(left) != 0 {
361 t.Fatalf("leftovers after failed Start: %v", left)
362 }
363 if d.slots.InUse() != 0 {
364 t.Fatal("slot leaked by a failed Start")
365 }
366}
367
368func TestBootTimeoutCleansEverything(t *testing.T) {
369 e := newEnv(t, "hang")
370 d, err := New(context.Background(), e.cfg, Options{Run: e.run, Net: e.plumb, BootTimeout: 300 * time.Millisecond})
371 if err != nil {
372 t.Fatal(err)
373 }
374 defer func() { _ = d.Close() }()
375 sb, _ := d.New()
376 _, err = sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
377 if err == nil || !strings.Contains(err.Error(), "did not connect") {
378 t.Fatalf("got %v", err)
379 }
380 if left := e.leftovers(t); len(left) != 0 {
381 t.Fatalf("leftovers: %v", left)
382 }
383}
384
385func TestStartCancelledByContext(t *testing.T) {
386 e := newEnv(t, "hang")
387 d := e.driver(t)
388 sb, _ := d.New()
389 ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
390 defer cancel()
391 if _, err := sb.Start(ctx, sandbox.JobSpec{JobID: "j"}); !errors.Is(err, context.DeadlineExceeded) {
392 t.Fatalf("got %v", err)
393 }
394 if left := e.leftovers(t); len(left) != 0 {
395 t.Fatalf("leftovers: %v", left)
396 }
397}
398
399func TestNetworkAttachFailureCleans(t *testing.T) {
400 e := newEnv(t, "")
401 d := e.driver(t)
402 e.plumb.FailNext = errors.New("nft broke")
403 sb, _ := d.New()
404 if _, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"}); err == nil || !strings.Contains(err.Error(), "nft broke") {
405 t.Fatalf("got %v", err)
406 }
407 if left := e.leftovers(t); len(left) != 0 {
408 t.Fatalf("leftovers: %v", left)
409 }
410}
411
412func TestStopWithoutStartReleasesSlot(t *testing.T) {
413 e := newEnv(t, "")
414 d := e.driver(t)
415 sb, _ := d.New()
416 if err := sb.Stop(context.Background()); err != nil {
417 t.Fatal(err)
418 }
419 if d.slots.InUse() != 0 {
420 t.Fatal("slot leaked")
421 }
422 if _, err := sb.Start(context.Background(), sandbox.JobSpec{}); err == nil {
423 t.Fatal("Start after Stop accepted")
424 }
425}
426
427// Stop must finish the cleanup even when the caller's context is already done
428// (a cancelled job).
429func TestStopRunsWithCancelledContext(t *testing.T) {
430 e := newEnv(t, "")
431 d := e.driver(t)
432 sb, _ := d.New()
433 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
434 if err != nil {
435 t.Fatal(err)
436 }
437 runStub(t, conn)
438 ctx, cancel := context.WithCancel(context.Background())
439 cancel()
440 if err := sb.Stop(ctx); err != nil {
441 t.Fatal(err)
442 }
443 if left := e.leftovers(t); len(left) != 0 {
444 t.Fatalf("leftovers: %v", left)
445 }
446}
447
448// A killed runner leaves a running Firecracker, a TAP, nft rules and files. The
449// next start must reap all of it (AC4: nothing survives, incl. killed jobs).
450func TestReapAfterRunnerCrash(t *testing.T) {
451 e := newEnv(t, "")
452 d1 := e.driver(t)
453 sb, _ := d1.New()
454 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
455 if err != nil {
456 t.Fatal(err)
457 }
458 runStub(t, conn)
459 pid := sb.(*vmSandbox).rec.PID
460 if err := syscall.Kill(pid, 0); err != nil {
461 t.Fatal("fake VM not running")
462 }
463 // The runner dies: its lock goes away, nothing else is cleaned.
464 _ = d1.lock.Release()
465 _ = conn.Close()
466 if e.tapCount() != 1 {
467 t.Fatal("setup: no TAP")
468 }
469
470 d2 := e.driver(t)
471 if err := d2.Reap(context.Background()); err != nil {
472 t.Fatal(err)
473 }
474 if left := e.leftovers(t); len(left) != 0 {
475 t.Fatalf("leftovers after Reap: %v", left)
476 }
477 deadline := time.Now().Add(3 * time.Second)
478 for syscall.Kill(pid, 0) == nil && time.Now().Before(deadline) {
479 // The fake is a child of this test process: it is a zombie until the
480 // original driver's Wait goroutine collects it.
481 if b, err := os.ReadFile(fmt.Sprintf("/proc/%d/stat", pid)); err == nil && strings.Contains(string(b), ") Z") {
482 break
483 }
484 time.Sleep(20 * time.Millisecond)
485 }
486 if b, _ := os.ReadFile(fmt.Sprintf("/proc/%d/stat", pid)); len(b) > 0 && !strings.Contains(string(b), ") Z") {
487 t.Fatalf("firecracker %d survived Reap: %s", pid, b)
488 }
489 if e.plumb.Resets == 0 {
490 t.Fatal("the nftables table was not reset")
491 }
492}
493
494func TestReapRemovesCorruptRecordAndTempFiles(t *testing.T) {
495 e := newEnv(t, "")
496 d := e.driver(t)
497 _ = os.MkdirAll(d.store.dir, 0o700)
498 _ = os.WriteFile(filepath.Join(d.store.dir, "er00112233.json"), []byte("{garbage"), 0o600)
499 _ = os.WriteFile(filepath.Join(d.store.dir, "er44556677.json.tmp"), []byte("{"), 0o600)
500 _ = os.MkdirAll(filepath.Join(e.cfg.StateDir, "vm", "er00112233"), 0o700)
501 _ = os.MkdirAll(filepath.Join(e.cfg.StateDir, "vm", "not-ours"), 0o700) // must be left alone
502 if err := d.Reap(context.Background()); err != nil {
503 t.Fatal(err)
504 }
505 if _, err := os.Stat(filepath.Join(e.cfg.StateDir, "vm", "not-ours")); err != nil {
506 t.Fatal("Reap removed a directory that is not ours")
507 }
508 _ = os.Remove(filepath.Join(e.cfg.StateDir, "vm", "not-ours"))
509 if left := e.leftovers(t); len(left) != 0 {
510 t.Fatalf("leftovers: %v", left)
511 }
512}
513
514func TestReapLeavesForeignTapsAlone(t *testing.T) {
515 e := newEnv(t, "")
516 e.taps["tap0"] = true
517 e.taps["ertapx"] = true // not ertap<digits>
518 e.taps["ertap7"] = true
519 d := e.driver(t)
520 if err := d.Reap(context.Background()); err != nil {
521 t.Fatal(err)
522 }
523 if !e.taps["tap0"] || !e.taps["ertapx"] || e.taps["ertap7"] {
524 t.Fatalf("taps after reap: %v", e.taps)
525 }
526}
527
528func TestSecondRunnerOnSameStateDirRefused(t *testing.T) {
529 e := newEnv(t, "")
530 _ = e.driver(t)
531 _, err := New(context.Background(), e.cfg, Options{Run: e.run, Net: e.plumb})
532 if err == nil || !strings.Contains(err.Error(), "another ermite-runner is running on this host") {
533 t.Fatalf("got %v", err)
534 }
535}
536
537func TestNewRejectsDynamicExecutable(t *testing.T) {
538 e := newEnv(t, "")
539 e.cfg.Executable = "/bin/sh"
540 if _, err := New(context.Background(), e.cfg, Options{Run: e.run, Net: e.plumb}); err == nil {
541 t.Skip("/bin/sh is static here")
542 }
543}
544
545func TestExecDiskIsKeyedByExecutableHash(t *testing.T) {
546 e := newEnv(t, "")
547 d1 := e.driver(t)
548 first := d1.execDisk
549 _ = d1.Close()
550 // Same executable: reused, no second mkfs of an exec disk.
551 n := len(e.run.Calls)
552 d2 := e.driver(t)
553 if d2.execDisk != first {
554 t.Fatal("exec disk rebuilt for the same executable")
555 }
556 for _, c := range e.run.Calls[n:] {
557 if c.Name == "mkfs.ext4" && (len(c.Args) != 1 || c.Args[0] != "-V") {
558 t.Fatal("mkfs run again for an unchanged executable")
559 }
560 }
561 _ = d2.Close()
562 // A runner upgrade: new disk, old one removed.
563 if err := os.WriteFile(e.cfg.Executable, append(readFile(t, e.cfg.Executable), 0), 0o755); err != nil {
564 t.Fatal(err)
565 }
566 d3 := e.driver(t)
567 if d3.execDisk == first {
568 t.Fatal("exec disk not rebuilt after the executable changed")
569 }
570 if _, err := os.Stat(first); err == nil {
571 t.Fatal("stale exec disk kept")
572 }
573}
574
575func readFile(t *testing.T, p string) []byte {
576 t.Helper()
577 b, err := os.ReadFile(p)
578 if err != nil {
579 t.Fatal(err)
580 }
581 return b
582}
583
584func TestJailedLayout(t *testing.T) {
585 l := newLayout("/var/lib/er", "firecracker", "er00112233", true, 900003)
586 if l.root != "/var/lib/er/jail/firecracker/er00112233/root" || l.dir != "/var/lib/er/jail/firecracker/er00112233" {
587 t.Fatalf("%+v", l)
588 }
589 if l.fc("scratch.ext4") != "/scratch.ext4" || l.host("scratch.ext4") != l.root+"/scratch.ext4" {
590 t.Fatal("jailed paths")
591 }
592 if jailBase("/var/lib/er") != "/var/lib/er/jail" {
593 t.Fatal("jail base")
594 }
595 if l2 := newLayout("/var/lib/er", "firecracker-v1.17", "er00112233", true, 1); l2.dir != "/var/lib/er/jail/firecracker-v1.17/er00112233" {
596 t.Fatalf("the jailer names its chroot after the exec file: %s", l2.dir)
597 }
598 u := newLayout("/var/lib/er", "firecracker", "er00112233", false, 1000)
599 if u.fc("api.sock") != "/var/lib/er/vm/er00112233/api.sock" {
600 t.Fatalf("unjailed path %s", u.fc("api.sock"))
601 }
602 if p, _ := u.share("/img/base.ext4", "base.ext4"); p != "/img/base.ext4" {
603 t.Fatal("unjailed share must use the file in place")
604 }
605}
606
607func TestJailedShareHardLinks(t *testing.T) {
608 dir := t.TempDir()
609 src := filepath.Join(dir, "base.ext4")
610 _ = os.WriteFile(src, []byte("data"), 0o644)
611 l := newLayout(dir, "firecracker", "er00112233", true, os.Getuid())
612 if err := os.MkdirAll(l.root, 0o700); err != nil {
613 t.Fatal(err)
614 }
615 p, err := l.share(src, "base.ext4")
616 if err != nil || p != "/base.ext4" {
617 t.Fatalf("%q %v", p, err)
618 }
619 a, _ := os.Stat(src)
620 b, _ := os.Stat(l.host("base.ext4"))
621 if !os.SameFile(a, b) {
622 t.Fatal("shared file must be a hard link, never a copy")
623 }
624 if err := checkWorldReadable(src); err != nil {
625 t.Fatal(err)
626 }
627 _ = os.Chmod(src, 0o600)
628 if err := checkWorldReadable(src); err == nil {
629 t.Fatal("unreadable-by-others file accepted for the jail")
630 }
631}
632
633func TestBootArgs(t *testing.T) {
634 a := bootArgs(mustAddr("10.200.0.6"), mustAddr("10.200.0.5"), 1791818522, false)
635 for _, want := range []string{
636 "ipv6.disable=1", "root=/dev/vda ro", "init=/ermite-runner", "reboot=k", "panic=1",
637 "ip=10.200.0.6::10.200.0.5:255.255.255.252:ermite-job:eth0:off:10.200.0.5",
638 "ermite.sandbox=vm", "ermite.connect=vsock:1024", "ermite.gw=10.200.0.5", "ermite.time=1791818522",
639 } {
640 if !strings.Contains(a, want) {
641 t.Errorf("boot args lack %q: %s", want, a)
642 }
643 }
644 if !strings.HasSuffix(a, "-- init") {
645 t.Errorf("init subcommand missing: %s", a)
646 }
647 if strings.Contains(a, "ermite.externals") {
648 t.Errorf("externals announced without a disk: %s", a)
649 }
650 if b := bootArgs(mustAddr("10.200.0.6"), mustAddr("10.200.0.5"), 1791818522, true); !strings.Contains(b, " ermite.externals=1 -- init") {
651 t.Errorf("externals not announced: %s", b)
652 }
653 if macFor(0) != "06:00:ac:10:00:00" || macFor(257) != "06:00:ac:10:01:01" {
654 t.Fatal(macFor(257))
655 }
656}
657
658func TestParseTapNames(t *testing.T) {
659 in := "1: lo: <LOOPBACK> mtu 65536\n2: eth0: <UP> mtu 1500\n5: ertap0: <BROADCAST> mtu 1500\n9: ertap12@NONE: <X>\n10: ertapx: <X>\n11: tap0: <X>\n12: ertap: <X>\n"
660 got := strings.Join(parseTapNames(in), ",")
661 if got != "ertap0,ertap12" {
662 t.Fatalf("got %q", got)
663 }
664}
665
666func TestFirecrackerProcsScan(t *testing.T) {
667 proc := t.TempDir()
668 mk := func(pid, cmdline string) {
669 _ = os.MkdirAll(filepath.Join(proc, pid), 0o755)
670 _ = os.WriteFile(filepath.Join(proc, pid, "cmdline"), []byte(cmdline), 0o644)
671 }
672 mk("100", "firecracker\x00--id\x00er00112233\x00--api-sock\x00/var/lib/er/vm/er00112233/api.sock\x00")
673 mk("101", "/usr/bin/firecracker\x00--api-sock\x00/var/lib/er/vm/er44556677/api.sock\x00--id\x00er44556677\x00")
674 mk("102", "firecracker\x00--id\x00other-tool-vm\x00") // someone else's VM
675 mk("103", "bash\x00--id\x00er00112233\x00") // not firecracker
676 mk("1", "firecracker\x00--id\x00er99999999\x00") // pid 1 is never touched
677 mk("abc", "")
678 got := vmmProcs(proc, newScope("/var/lib/er"))
679 if len(got) != 2 || got[100] != "er00112233" || got[101] != "er44556677" {
680 t.Fatalf("%v", got)
681 }
682}
683
684func TestRingKeepsTail(t *testing.T) {
685 r := newRing(8)
686 _, _ = r.Write([]byte("0123456789"))
687 _, _ = r.Write([]byte("ab"))
688 if r.String() != "23456789ab"[2:] {
689 t.Fatalf("%q", r.String())
690 }
691}
692
693func TestAPIErrorsSurfaceFirecrackerFault(t *testing.T) {
694 sock := filepath.Join(t.TempDir(), "s")
695 ln, err := net.Listen("unix", sock)
696 if err != nil {
697 t.Fatal(err)
698 }
699 defer func() { _ = ln.Close() }()
700 go func() {
701 _ = http.Serve(ln, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
702 w.WriteHeader(400)
703 _, _ = w.Write([]byte(`{"fault_message":"Invalid request"}`))
704 }))
705 }()
706 err = newAPIClient(sock).put(context.Background(), "/boot-source", bootSource{})
707 if err == nil || !strings.Contains(err.Error(), "Invalid request") || !strings.Contains(err.Error(), "/boot-source") {
708 t.Fatalf("got %v", err)
709 }
710}
711
712func mustPrefix(s string) netip.Prefix { return netip.MustParsePrefix(s) }
713func mustAddr(s string) netip.Addr { return netip.MustParseAddr(s) }
714
715// C3: processes are matched by "--id <our id>" in their command line, not by the
716// program name: a renamed or versioned Firecracker binary is still found and
717// reaped, and a recycled pid is never killed on the strength of a record alone.
718func TestReapFindsRenamedVMMByID(t *testing.T) {
719 e := newEnvNamed(t, "", "fc-renamed-1.17")
720 d1 := e.driver(t)
721 sb, _ := d1.New()
722 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
723 if err != nil {
724 t.Fatal(err)
725 }
726 runStub(t, conn)
727 pid := sb.(*vmSandbox).rec.PID
728 if got := cmdlineID("/proc", pid, e.driverScope()); got != sb.ID() {
729 t.Fatalf("cmdlineID=%q want %q (argv[0] is %q)", got, sb.ID(), "fc-renamed-1.17")
730 }
731 _ = d1.lock.Release()
732 _ = conn.Close()
733 d2 := e.driver(t)
734 if err := d2.Reap(context.Background()); err != nil {
735 t.Fatal(err)
736 }
737 if pidAlive(pid) {
738 t.Fatalf("a VMM with a renamed binary survived Reap (pid %d)", pid)
739 }
740}
741
742func TestIDFromCmdline(t *testing.T) {
743 for name, tc := range map[string]struct {
744 args []string
745 want string
746 }{
747 "firecracker": {[]string{"firecracker", "--id", "er00112233", "--api-sock", "/x"}, "er00112233"},
748 "jailer": {[]string{"jailer", "--id", "er00112233", "--exec-file", "/fc", "--uid", "1"}, "er00112233"},
749 "renamed": {[]string{"anything", "--api-sock", "/x", "--id", "er44556677"}, "er44556677"},
750 // What a real jailer 1.17 leaves: /firecracker --id <id> --start-time-us ...
751 "jailed firecracker": {[]string{"/firecracker", "--id", "er00112233", "--start-time-us", "150801025577", "--start-time-cpu-us", "60", "--parent-cpu-time-us", "1332"}, "er00112233"},
752 "foreign id": {[]string{"firecracker", "--id", "my-vm", "--api-sock", "/x"}, ""},
753 "no marker": {[]string{"python", "--id", "er00112233"}, ""},
754 "id without value": {[]string{"firecracker", "--api-sock", "/x", "--id"}, ""},
755 "path-like id": {[]string{"firecracker", "--id", "er../../../x", "--api-sock", "/x"}, ""},
756 } {
757 if got := IDFromCmdline(tc.args); got != tc.want {
758 t.Errorf("%s: got %q want %q", name, got, tc.want)
759 }
760 }
761}
762
763func TestReapDoesNotTrustARecycledPid(t *testing.T) {
764 e := newEnv(t, "")
765 d := e.driver(t)
766 // A record pointing at this very test process: its command line carries no
767 // VM id, so it must not be killed.
768 _ = d.store.save(record{ID: "er00112233", Slot: 0, PID: os.Getpid()})
769 if err := d.Reap(context.Background()); err != nil {
770 t.Fatal(err)
771 }
772 if !pidAlive(os.Getpid()) {
773 t.Fatal("Reap killed a process on the strength of a stale record")
774 }
775}
776
777// C5 + C8: when the VMM cannot be killed, the TAP, the firewall policy and the
778// files are left alone (the VM may still be running), the error wraps ErrLeaked,
779// the slot stays taken, and Reap (with a working kill) finishes the cleanup and
780// frees the slot.
781func TestStopKeepsNetworkWhenTheVMMSurvives(t *testing.T) {
782 e := newEnv(t, "")
783 stubborn := true
784 d, err := New(context.Background(), e.cfg, Options{Run: e.run, Net: e.plumb, BootTimeout: 5 * time.Second,
785 kill: func(pid int, still func() bool, to time.Duration) bool {
786 if stubborn {
787 return false
788 }
789 return killAndWait(pid, still, to)
790 }})
791 if err != nil {
792 t.Fatal(err)
793 }
794 defer func() { _ = d.Close() }()
795 sb, _ := d.New()
796 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
797 if err != nil {
798 t.Fatal(err)
799 }
800 runStub(t, conn)
801
802 err = sb.Stop(context.Background())
803 if !errors.Is(err, sandbox.ErrLeaked) {
804 t.Fatalf("Stop must report ErrLeaked, got %v", err)
805 }
806 if e.tapCount() != 1 || e.plumb.Live() != 1 {
807 t.Fatalf("the TAP and the policy must stay while the VMM may run: taps=%d attached=%d", e.tapCount(), e.plumb.Live())
808 }
809 if recs, _ := d.store.list(); len(recs) != 1 {
810 t.Fatalf("the state record must stay for Reap: %v", recs)
811 }
812 if err := sb.Stop(context.Background()); !errors.Is(err, sandbox.ErrLeaked) {
813 t.Fatalf("a second Stop must keep reporting the leak, got %v", err)
814 }
815 if d.slots.InUse() != 1 {
816 t.Fatal("the slot of a leaked sandbox stays taken")
817 }
818 // Other jobs keep running on the free slot; the leaked one is never reused.
819 other, err := d.New()
820 if err != nil {
821 t.Fatal(err)
822 }
823 if other.(*vmSandbox).slot == sb.(*vmSandbox).slot {
824 t.Fatal("a leaked slot was handed out again")
825 }
826 // Reap refuses while a sandbox is running...
827 if err := d.Reap(context.Background()); !errors.Is(err, sandbox.ErrBusy) {
828 t.Fatalf("Reap with a handed-out sandbox: %v", err)
829 }
830 _ = other.Stop(context.Background())
831 // ...and finishes the job once idle.
832 stubborn = false
833 if err := d.Reap(context.Background()); err != nil {
834 t.Fatal(err)
835 }
836 if left := e.leftovers(t); len(left) != 0 {
837 t.Fatalf("leftovers after Reap: %v", left)
838 }
839 if d.slots.InUse() != 0 {
840 t.Fatal("Reap must free the slot of the leaked sandbox")
841 }
842}
843
844// C7: Stop from another goroutine aborts a boot in progress at once, instead of
845// waiting for the boot timeout, and leaves nothing behind.
846func TestStopAbortsABootInProgress(t *testing.T) {
847 e := newEnv(t, "hang")
848 d, err := New(context.Background(), e.cfg, Options{Run: e.run, Net: e.plumb, BootTimeout: time.Minute})
849 if err != nil {
850 t.Fatal(err)
851 }
852 defer func() { _ = d.Close() }()
853 sb, _ := d.New()
854 started := make(chan error, 1)
855 go func() {
856 _, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
857 started <- err
858 }()
859 time.Sleep(700 * time.Millisecond) // the fake is now waiting for the executor
860 t0 := time.Now()
861 if err := sb.Stop(context.Background()); err != nil {
862 t.Fatal(err)
863 }
864 select {
865 case err := <-started:
866 if !errors.Is(err, sandbox.ErrAborted) {
867 t.Fatalf("Start must report that Stop aborted it, got %v", err)
868 }
869 case <-time.After(10 * time.Second):
870 t.Fatal("Start was not aborted by Stop")
871 }
872 if time.Since(t0) > 8*time.Second {
873 t.Fatalf("Stop took %v", time.Since(t0))
874 }
875 if left := e.leftovers(t); len(left) != 0 {
876 t.Fatalf("leftovers: %v", left)
877 }
878 // Stop before Start: Start must refuse.
879 sb2, _ := d.New()
880 _ = sb2.Stop(context.Background())
881 if _, err := sb2.Start(context.Background(), sandbox.JobSpec{}); err == nil {
882 t.Fatal("Start after Stop accepted")
883 }
884}
885
886// C16: the sandbox exposes its id, which names its files.
887func TestSandboxIDNamesItsFiles(t *testing.T) {
888 e := newEnv(t, "")
889 d := e.driver(t)
890 sb, _ := d.New()
891 if !sandbox.ValidID(sb.ID()) {
892 t.Fatalf("id %q", sb.ID())
893 }
894 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
895 if err != nil {
896 t.Fatal(err)
897 }
898 runStub(t, conn)
899 defer func() { _ = sb.Stop(context.Background()) }()
900 if _, err := os.Stat(filepath.Join(e.cfg.StateDir, "vm", sb.ID())); err != nil {
901 t.Fatalf("the VM directory must be named after the sandbox id: %v", err)
902 }
903 if d.Isolation() != sandbox.IsolationVM || d.Mode() != sandbox.VM {
904 t.Fatal("isolation")
905 }
906}
907
908// S7: cgroup v2 limits, a new PID namespace, and the jailer's own options.
909func TestJailerArgs(t *testing.T) {
910 args := jailerArgs("/usr/bin/jailer", "/usr/bin/firecracker", "er00112233", 900003, "/var/lib/er/jail",
911 sandbox.JobSpec{CPUs: 4, MemoryMiB: 2048}, nil)
912 line := " " + strings.Join(args, " ") + " "
913 for _, want := range []string{
914 " --id er00112233 ", " --exec-file /usr/bin/firecracker ", " --uid 900003 ", " --gid 900003 ",
915 " --chroot-base-dir /var/lib/er/jail ", " --cgroup-version 2 ", " --new-pid-ns ",
916 " --cgroup memory.max=" + fmt.Sprint((2048+vmmOverheadMiB)<<20) + " ",
917 " --cgroup pids.max=64 ", " --resource-limit no-file=4096 ", " -- ",
918 } {
919 if !strings.Contains(line, want) {
920 t.Errorf("jailer args lack %q: %s", want, line)
921 }
922 }
923 if args[len(args)-1] != "--" {
924 t.Errorf("the jailer separator must come last when there are no extra args: %v", args)
925 }
926 more := jailerArgs("j", "f", "er00112233", 1, "/b", sandbox.JobSpec{CPUs: 1, MemoryMiB: 512}, []string{"--x"})
927 if more[len(more)-2] != "--" || more[len(more)-1] != "--x" {
928 t.Errorf("extra args follow the separator: %v", more)
929 }
930}
931
932// S7: sane rate limiters on the scratch disk and both directions of eth0, off
933// when configured to zero, and only on the writable disk.
934func TestRateLimitersInTheVMConfig(t *testing.T) {
935 e := newEnv(t, "")
936 d := e.driver(t)
937 sb, _ := d.New()
938 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
939 if err != nil {
940 t.Fatal(err)
941 }
942 runStub(t, conn)
943 defer func() { _ = sb.Stop(context.Background()) }()
944 b, _ := os.ReadFile(e.log)
945 log := string(b)
946 for _, want := range []string{
947 `"rate_limiter":{"bandwidth":{"size":419430400,"one_time_burst":419430400,"refill_time":1000},"ops":{"size":20000,"one_time_burst":20000,"refill_time":1000}}`,
948 `"rx_rate_limiter":{"bandwidth":{"size":125000000,"one_time_burst":125000000,"refill_time":1000}}`,
949 `"tx_rate_limiter":{"bandwidth":{"size":125000000,"one_time_burst":125000000,"refill_time":1000}}`,
950 } {
951 if !strings.Contains(log, want) {
952 t.Errorf("API log lacks %s\n%s", want, log)
953 }
954 }
955 if n := strings.Count(log, `"rate_limiter"`); n != 1 {
956 t.Errorf("only the scratch drive is rate limited, got %d rate_limiter fields", n)
957 }
958 if diskLimiter(0, 0) != nil || netLimiter(0) != nil {
959 t.Error("zero must disable the limiters")
960 }
961 if l := diskLimiter(0, 100); l.Bandwidth != nil || l.Ops == nil {
962 t.Errorf("%+v", l)
963 }
964}
965
966// S8: the free-space check warns, and only warns.
967func TestNewWarnsOnLowFreeSpace(t *testing.T) {
968 e := newEnv(t, "")
969 var buf bytes.Buffer
970 ctx := zerowrap.WithCtx(context.Background(), zerowrap.New(zerowrap.Config{Level: "debug", Format: "json", Output: &buf}))
971 d, err := New(ctx, e.cfg, Options{Run: e.run, Net: e.plumb, FreeSpace: func(string) (int64, error) { return 1, nil }})
972 if err != nil {
973 t.Fatalf("low space must not fail New: %v", err)
974 }
975 defer func() { _ = d.Close() }()
976 if !strings.Contains(buf.String(), "less free space") {
977 t.Fatalf("no warning: %s", buf.String())
978 }
979}
980
981// S11: the scratch disk is created without following symlinks and chowned
982// through its descriptor.
983func TestMakeScratchRefusesSymlinksAndExistingFiles(t *testing.T) {
984 dir := t.TempDir()
985 target := filepath.Join(dir, "victim")
986 _ = os.WriteFile(target, []byte("precious"), 0o600)
987 link := filepath.Join(dir, "scratch.ext4")
988 if err := os.Symlink(target, link); err != nil {
989 t.Fatal(err)
990 }
991 run := &cmdrun.Fake{}
992 if err := disk.CreateScratch(context.Background(), run, link, 8, -1); err == nil {
993 t.Fatal("a planted symlink must be refused")
994 }
995 if b, _ := os.ReadFile(target); string(b) != "precious" {
996 t.Fatalf("the symlink target was modified: %q", b)
997 }
998 if len(run.Calls) != 0 {
999 t.Fatalf("mkfs must not run: %v", run.Lines())
1000 }
1001 ok := filepath.Join(dir, "fresh.ext4")
1002 if err := disk.CreateScratch(context.Background(), run, ok, 8, os.Getuid()); err != nil {
1003 t.Fatal(err)
1004 }
1005 if st, err := os.Stat(ok); err != nil || st.Size() != 8<<20 || st.Mode().Perm() != 0o600 {
1006 t.Fatalf("%v %v", st, err)
1007 }
1008 if err := disk.CreateScratch(context.Background(), run, ok, 8, -1); err == nil {
1009 t.Fatal("an existing file must be refused (O_EXCL)")
1010 }
1011}
1012
1013func TestOwnUsesLchown(t *testing.T) {
1014 dir := t.TempDir()
1015 l := newLayout(dir, "firecracker", "er00112233", true, os.Getuid())
1016 _ = os.MkdirAll(l.root, 0o700)
1017 target := filepath.Join(dir, "victim")
1018 _ = os.WriteFile(target, nil, 0o600)
1019 if err := os.Symlink(target, l.host("link")); err != nil {
1020 t.Fatal(err)
1021 }
1022 if err := l.own("link"); err != nil {
1023 t.Fatalf("Lchown on a link must act on the link itself: %v", err)
1024 }
1025 if err := l.own("missing"); err == nil {
1026 t.Fatal("a missing file is an error")
1027 }
1028 if u := newLayout(dir, "f", "er00112233", false, 1); u.own("anything") != nil {
1029 t.Fatal("no-op without jailer")
1030 }
1031}
1032
1033func TestRemoveCgroup(t *testing.T) {
1034 root := t.TempDir()
1035 _ = os.MkdirAll(filepath.Join(root, "firecracker", "er00112233"), 0o755)
1036 if err := removeCgroup(root, "firecracker", "er00112233"); err != nil {
1037 t.Fatal(err)
1038 }
1039 if err := removeCgroup(root, "firecracker", "er00112233"); err != nil {
1040 t.Fatalf("a missing cgroup is fine: %v", err)
1041 }
1042 _ = os.MkdirAll(filepath.Join(root, "firecracker", "er44556677", "child"), 0o755)
1043 if err := removeCgroup(root, "firecracker", "er44556677"); err == nil {
1044 t.Fatal("a non-empty cgroup cannot be removed and must be reported")
1045 }
1046}
1047
1048// Jailer 1.17 canonicalizes --chroot-base-dir and exits when it does not exist:
1049// the driver must create it (seen on a real host, invisible with the jailer off).
1050func TestNewCreatesTheJailerChrootBase(t *testing.T) {
1051 e := newEnv(t, "ok")
1052 e.cfg.JailerOff = false
1053 e.cfg.Jailer = e.cfg.Firecracker // any executable: it is only resolved here
1054 short, err := os.MkdirTemp("", "er")
1055 if err != nil {
1056 t.Fatal(err)
1057 }
1058 defer func() { _ = os.RemoveAll(short) }()
1059 e.cfg.StateDir = filepath.Join(short, "s") // the jailed API socket path must fit in sun_path
1060 d, err := New(context.Background(), e.cfg, Options{Run: e.run, Net: e.plumb, BootTimeout: 5 * time.Second})
1061 if err != nil {
1062 t.Fatal(err)
1063 }
1064 defer func() { _ = d.Close() }()
1065 st, err := os.Stat(jailBase(e.cfg.StateDir))
1066 if err != nil || !st.IsDir() {
1067 t.Fatalf("jail base not created: %v", err)
1068 }
1069}
1070
1071// GO-004: the longest socket path of a VM, in both jailer modes, 107 bytes ok /
1072// 108 rejected.
1073func TestSocketPathBoundaryInBothModes(t *testing.T) {
1074 for _, jailed := range []bool{true, false} {
1075 name := map[bool]string{true: "jailed", false: "unjailed"}[jailed]
1076 // Find the longest socket path with an empty-ish state dir, then pad the
1077 // state dir so that it is exactly maxSockPath, then one more.
1078 longest := func(state string) int {
1079 n := 0
1080 for _, p := range socketPaths(state, "firecracker", jailed) {
1081 n = max(n, len(p))
1082 }
1083 return n
1084 }
1085 base := "/s"
1086 pad := maxSockPath - longest(base)
1087 if pad < 0 {
1088 t.Fatalf("%s: base already too long", name)
1089 }
1090 ok := base + strings.Repeat("d", pad)
1091 if longest(ok) != maxSockPath {
1092 t.Fatalf("%s: longest=%d want %d", name, longest(ok), maxSockPath)
1093 }
1094 if err := checkSocketPaths(ok, "firecracker", jailed); err != nil {
1095 t.Errorf("%s: %d bytes rejected: %v", name, maxSockPath, err)
1096 }
1097 err := checkSocketPaths(ok+"d", "firecracker", jailed)
1098 if err == nil || !strings.Contains(err.Error(), "limit of a Unix socket") {
1099 t.Errorf("%s: %d bytes accepted: %v", name, maxSockPath+1, err)
1100 }
1101 }
1102}
1103
1104func TestNewRejectsAStateDirThatMakesASocketPathTooLong(t *testing.T) {
1105 for _, jailerOff := range []bool{false, true} {
1106 e := newEnv(t, "ok")
1107 e.cfg.JailerOff = jailerOff
1108 e.cfg.Jailer = e.cfg.Firecracker
1109 e.cfg.StateDir = filepath.Join(e.cfg.StateDir, strings.Repeat("d", 60))
1110 _, err := New(context.Background(), e.cfg, Options{Run: e.run, Net: e.plumb, BootTimeout: 5 * time.Second})
1111 if err == nil || !strings.Contains(err.Error(), "limit of a Unix socket") {
1112 t.Fatalf("jailerOff=%v: want a clear socket path error, got %v", jailerOff, err)
1113 }
1114 }
1115}
1116
1117// GO-003: the chroot base must be a real directory, owned by root (or the
1118// runner's own user), not writable by group or others; an existing one is never
1119// chmod-ed into compliance.
1120func TestEnsureJailBase(t *testing.T) {
1121 root := t.TempDir()
1122 good := filepath.Join(root, "good")
1123 if err := ensureJailBase(good); err != nil {
1124 t.Fatal(err)
1125 }
1126 if fi, _ := os.Stat(good); fi.Mode().Perm() != 0o700 {
1127 t.Fatalf("created with %v", fi.Mode().Perm())
1128 }
1129 if err := ensureJailBase(good); err != nil { // idempotent
1130 t.Fatal(err)
1131 }
1132 for name, mode := range map[string]os.FileMode{"group-writable": 0o770, "other-writable": 0o707, "sticky-world": 0o777} {
1133 d := filepath.Join(root, name)
1134 _ = os.Mkdir(d, 0o700)
1135 _ = os.Chmod(d, mode)
1136 if err := ensureJailBase(d); err == nil || !strings.Contains(err.Error(), "writable by group or others") {
1137 t.Errorf("%s: %v", name, err)
1138 }
1139 if fi, _ := os.Stat(d); fi.Mode().Perm() != mode {
1140 t.Errorf("%s: mode was changed to %v", name, fi.Mode().Perm())
1141 }
1142 }
1143 rx := filepath.Join(root, "rx")
1144 _ = os.Mkdir(rx, 0o755) // readable by others is fine
1145 _ = os.Chmod(rx, 0o755)
1146 if err := ensureJailBase(rx); err != nil {
1147 t.Errorf("0755 refused: %v", err)
1148 }
1149 file := filepath.Join(root, "file")
1150 _ = os.WriteFile(file, nil, 0o600)
1151 if err := ensureJailBase(file); err == nil {
1152 t.Error("a file accepted")
1153 }
1154 link := filepath.Join(root, "link")
1155 _ = os.Symlink(good, link)
1156 if err := ensureJailBase(link); err == nil || !strings.Contains(err.Error(), "not a directory") {
1157 t.Errorf("a symlink accepted: %v", err)
1158 }
1159}
1160
1161type fakeInfo struct {
1162 fs.FileInfo
1163 uid uint32
1164}
1165
1166func (f fakeInfo) Sys() any { return &syscall.Stat_t{Uid: f.uid} }
1167
1168func TestJailBaseOwner(t *testing.T) {
1169 fi, err := os.Stat(t.TempDir())
1170 if err != nil {
1171 t.Fatal(err)
1172 }
1173 d := "/x/jail"
1174 mine := os.Geteuid()
1175 if err := checkJailBaseInfo(d, fakeInfo{fi, 0}, mine); err != nil {
1176 t.Errorf("root-owned refused: %v", err)
1177 }
1178 if err := checkJailBaseInfo(d, fakeInfo{fi, uint32(mine)}, mine); err != nil {
1179 t.Errorf("own user refused: %v", err)
1180 }
1181 if err := checkJailBaseInfo(d, fakeInfo{fi, uint32(mine) + 4242}, mine); err == nil || !strings.Contains(err.Error(), "owned by uid") {
1182 t.Errorf("foreign owner accepted: %v", err)
1183 }
1184}
1185
1186// GO-001: a Firecracker with only the jailer's --start-time-us is ours only when
1187// it lives in our jail base; a jailer only when it was given our chroot base.
1188func TestScanIsScopedToOurJailBase(t *testing.T) {
1189 state := t.TempDir()
1190 sc := newScope(state)
1191 foreign := t.TempDir()
1192 proc := t.TempDir()
1193 mk := func(pid, root string, args ...string) {
1194 dir := filepath.Join(proc, pid)
1195 _ = os.MkdirAll(dir, 0o755)
1196 _ = os.WriteFile(filepath.Join(dir, "cmdline"), []byte(strings.Join(args, "\x00")+"\x00"), 0o644)
1197 if root != "" {
1198 if err := os.Symlink(root, filepath.Join(dir, "root")); err != nil {
1199 t.Fatal(err)
1200 }
1201 }
1202 }
1203 ours := filepath.Join(jailBase(state), "firecracker", "er00112233", "root")
1204 if err := os.MkdirAll(ours, 0o755); err != nil {
1205 t.Fatal(err)
1206 }
1207 // What a pivot_root'ed jailed VMM looks like: its root is our chroot directory
1208 // but readlink does not say so (it says "/"); a symlinked alias reproduces
1209 // that: another path, same inode.
1210 alias := filepath.Join(t.TempDir(), "alias")
1211 if err := os.Symlink(ours, alias); err != nil {
1212 t.Fatal(err)
1213 }
1214 // Our jail base has a chroot for another id: a foreign VMM must not match it.
1215 if err := os.MkdirAll(filepath.Join(jailBase(state), "firecracker", "er44556677", "root"), 0o755); err != nil {
1216 t.Fatal(err)
1217 }
1218 foreignRoot := filepath.Join(foreign, "jail", "firecracker", "er44556677", "root")
1219 _ = os.MkdirAll(foreignRoot, 0o755)
1220 mk("100", ours, "/firecracker", "--id", "er00112233", "--start-time-us", "1", "--start-time-cpu-us", "2")
1221 mk("101", foreignRoot, "/firecracker", "--id", "er44556677", "--start-time-us", "1")
1222 mk("109", alias, "/firecracker", "--id", "er00112233", "--start-time-us", "1") // same inode, other path
1223 mk("110", alias, "/firecracker", "--id", "er77777777", "--start-time-us", "1") // our dir, but not that id's chroot
1224 mk("102", "/", "/firecracker", "--id", "er8899aabb", "--start-time-us", "1") // not chrooted at all
1225 mk("103", "", "/firecracker", "--id", "er0000cccc", "--start-time-us", "1") // root unreadable
1226 mk("104", "/", "jailer", "--id", "er11111111", "--exec-file", "/fc", "--chroot-base-dir", jailBase(state))
1227 mk("105", "/", "jailer", "--id", "er22222222", "--exec-file", "/fc", "--chroot-base-dir", filepath.Join(foreign, "jail"))
1228 mk("106", "/", "jailer", "--id", "er33333333", "--exec-file", "/fc")
1229 mk("107", ours+"/../../../../x", "/firecracker", "--id", "er55555555", "--start-time-us", "1") // lexical escape
1230 vmDir := filepath.Join(state, "vm", "er66666666")
1231 mk("108", "/", "firecracker", "--id", "er66666666", "--api-sock", filepath.Join(vmDir, "api.sock")) // unjailed, in our dir
1232 mk("111", "/", "firecracker", "--id", "er99999999", "--api-sock", "/x") // socket elsewhere
1233 mk("112", "/", "firecracker", "--id", "erabababab", "--api-sock", filepath.Join(vmDir, "api.sock")) // our dir, another id
1234 mk("113", "/", "firecracker", "--id", "er66666666", "--api-sock", vmDir+"/../../x/api.sock") // escapes by ..
1235 mk("114", "/", "firecracker", "--id", "er66666666", "--api-sock", "vm/er66666666/api.sock") // relative
1236 mk("115", "/", "firecracker", "--id", "er66666666", "--api-sock") // no value
1237 mk("116", "/", "firecracker", "--id", "er66666666", "--api-sock", vmDir+"x/api.sock") // prefix sibling
1238 got := vmmProcs(proc, sc)
1239 want := map[int]string{100: "er00112233", 104: "er11111111", 108: "er66666666", 109: "er00112233"}
1240 if len(got) != len(want) {
1241 t.Fatalf("got %v want %v", got, want)
1242 }
1243 for pid, id := range want {
1244 if got[pid] != id {
1245 t.Fatalf("got %v want %v", got, want)
1246 }
1247 }
1248 if cmdlineID(proc, 101, sc) != "" || cmdlineID(proc, 100, sc) != "er00112233" {
1249 t.Fatal("cmdlineID must apply the same scope")
1250 }
1251}
1252
1253// GO-001 (pid reuse): the kill goes through a pidfd, after the identity check.
1254func startSleeper(t *testing.T) *exec.Cmd {
1255 t.Helper()
1256 c := exec.Command("sleep", "60")
1257 if err := c.Start(); err != nil {
1258 t.Skipf("cannot start sleep: %v", err)
1259 }
1260 t.Cleanup(func() { _ = c.Process.Kill(); _ = c.Wait() })
1261 return c
1262}
1263
1264func needPidfd(t *testing.T) {
1265 t.Helper()
1266 fd, err := unix.PidfdOpen(os.Getpid(), 0)
1267 if err != nil {
1268 t.Skipf("no pidfd on this kernel: %v", err)
1269 }
1270 _ = unix.Close(fd)
1271}
1272
1273func TestKillAndWaitKillsThroughThePidfd(t *testing.T) {
1274 needPidfd(t)
1275 c := startSleeper(t)
1276 asked := false
1277 if !killAndWait(c.Process.Pid, func() bool { asked = true; return true }, 5*time.Second) {
1278 t.Fatal("the process is still there")
1279 }
1280 if !asked {
1281 t.Fatal("the identity check did not run")
1282 }
1283 // The child is a zombie now (we have not waited for it): it must count as gone.
1284 if _, err := c.Process.Wait(); err != nil {
1285 t.Fatal(err)
1286 }
1287}
1288
1289func TestKillAndWaitDoesNotKillAMismatchAfterTheOpen(t *testing.T) {
1290 needPidfd(t)
1291 c := startSleeper(t)
1292 // The pid now belongs to something that is not the VMM we meant (what a
1293 // recycled pid looks like): no signal, and "gone" (the VMM we meant is).
1294 if !killAndWait(c.Process.Pid, func() bool { return false }, 2*time.Second) {
1295 t.Fatal("a mismatch means the intended process is gone: true expected")
1296 }
1297 if !pidAlive(c.Process.Pid) {
1298 t.Fatal("a process that failed the identity check was killed")
1299 }
1300}
1301
1302func TestKillAndWaitOnAProcessThatIsAlreadyGone(t *testing.T) {
1303 needPidfd(t)
1304 c := exec.Command("true")
1305 if err := c.Run(); err != nil {
1306 t.Skip(err)
1307 }
1308 called := false
1309 if !killAndWait(c.Process.Pid, func() bool { called = true; return true }, time.Second) {
1310 t.Fatal("a reaped process must count as gone")
1311 }
1312 _ = called
1313}
1314
1315func TestKillAndWaitReportsAProcessThatStays(t *testing.T) {
1316 needPidfd(t)
1317 // pid 1 cannot be killed by us in a test (and must never be: the identity
1318 // check says it is ours here only to show the failure path would not hang).
1319 if os.Geteuid() == 0 {
1320 t.Skip("root could signal pid 1")
1321 }
1322 if killAndWait(1, func() bool { return true }, 200*time.Millisecond) {
1323 t.Fatal("an unkillable process was reported gone")
1324 }
1325}
1326
1327// Guest forge access: the exception reaches the policy of the guest, and the
1328// warning naming the origin is logged at the driver start and at every job.
1329func TestGuestForgeIsAttachedAndWarnedAtEveryJob(t *testing.T) {
1330 e := newEnv(t, "")
1331 var err error
1332 if e.cfg.GuestForge, err = sandbox.ParseGuestForge("https://ermite.example.org:8443", false); err != nil {
1333 t.Fatal(err)
1334 }
1335 var buf bytes.Buffer
1336 ctx := zerowrap.WithCtx(context.Background(), zerowrap.New(zerowrap.Config{Level: "debug", Format: "json", Output: &buf}))
1337 d, err := New(ctx, e.cfg, Options{Run: e.run, Net: e.plumb, BootTimeout: 5 * time.Second})
1338 if err != nil {
1339 t.Fatal(err)
1340 }
1341 defer func() { _ = d.Close() }()
1342 msg := "guest forge access: sandboxes may reach https://ermite.example.org:8443 on the host"
1343 if n := strings.Count(buf.String(), msg); n != 1 {
1344 t.Fatalf("one warning at driver start, got %d", n)
1345 }
1346 for i := range 2 {
1347 sb, _ := d.New()
1348 conn, err := sb.Start(ctx, sandbox.JobSpec{JobID: fmt.Sprint("job", i)})
1349 if err != nil {
1350 t.Fatal(err)
1351 }
1352 if len(e.plumb.Attached) != 1 {
1353 t.Fatalf("%d attached", len(e.plumb.Attached))
1354 }
1355 for _, g := range e.plumb.Attached {
1356 if g.ForgePort != 8443 {
1357 t.Errorf("guest attached with ForgePort %d", g.ForgePort)
1358 }
1359 }
1360 runStub(t, conn)
1361 _ = sb.Stop(ctx)
1362 }
1363 if n := strings.Count(buf.String(), msg); n != 3 {
1364 t.Fatalf("the warning must be logged at every job start (1 + 2), got %d", n)
1365 }
1366}
1367
1368// The host's external tools ride on a fourth, read-only drive, keyed by their
1369// content: a new Node.js builds a new disk and leaves the exec disk alone.
1370func TestExternalsDiskIsAttachedReadOnlyAndKeyedByContent(t *testing.T) {
1371 e := newEnv(t, "")
1372 src := &sandboxtest.Externals{}
1373 n24 := sandboxtest.MakeExternal(t, "node24", "node24-24.21.0-x64-aaaa", "v24.21.0")
1374 n20 := sandboxtest.MakeExternal(t, "node20", "node20-20.20.2-x64-bbbb", "v20.20.2")
1375 src.Set(n24, n20)
1376 e.cfg.Externals = src
1377 d := e.driver(t)
1378 execDisk := d.execDisk
1379
1380 boot := func() (drives []string, log string) {
1381 t.Helper()
1382 _ = os.Remove(e.log)
1383 sb, _ := d.New()
1384 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
1385 if err != nil {
1386 t.Fatal(err)
1387 }
1388 runStub(t, conn)
1389 defer func() { _ = sb.Stop(context.Background()) }()
1390 b, _ := os.ReadFile(e.log)
1391 log = string(b)
1392 for _, l := range strings.Split(log, "\n") {
1393 if f := strings.Fields(l); len(f) >= 2 && f[0] == "PUT" && strings.HasPrefix(f[1], "/drives/") {
1394 drives = append(drives, f[1])
1395 }
1396 }
1397 return drives, log
1398 }
1399 drives, log := boot()
1400 if strings.Join(drives, ",") != "/drives/rootfs,/drives/base,/drives/scratch,/drives/externals" {
1401 t.Fatalf("drives: %v", drives)
1402 }
1403 if !strings.Contains(log, `"drive_id":"externals"`) || !strings.Contains(log, `"is_root_device":false,"is_read_only":true`) || !strings.Contains(log, "ermite.externals=1") {
1404 t.Fatalf("API log:\n%s", log)
1405 }
1406 // The scratch drive is still the third (vdc), and the only writable one.
1407 if strings.Count(log, `"is_read_only":false`) != 1 {
1408 t.Errorf("only the scratch drive is writable:\n%s", log)
1409 }
1410 first, _ := filepath.Glob(filepath.Join(e.cfg.StateDir, "cache", "extdisk-*"))
1411 if len(first) != 1 {
1412 t.Fatalf("externals disks: %v", first)
1413 }
1414 mkfs := func() (n int) {
1415 for _, c := range e.run.Calls {
1416 if c.Name == "mkfs.ext4" && len(c.Args) > 1 {
1417 for _, a := range c.Args {
1418 if a == "ermite-ext" {
1419 n++
1420 }
1421 }
1422 }
1423 }
1424 return
1425 }
1426 if mkfs() != 1 {
1427 t.Fatalf("externals disk built %d times", mkfs())
1428 }
1429
1430 // A second job with the same tools reuses the disk.
1431 boot()
1432 if mkfs() != 1 {
1433 t.Fatal("externals disk rebuilt for the same content")
1434 }
1435
1436 // A new Node.js: another disk, and the exec disk is untouched.
1437 src.Set(sandboxtest.MakeExternal(t, "node24", "node24-24.22.0-x64-cccc", "v24.22.0"), n20)
1438 boot()
1439 if mkfs() != 2 {
1440 t.Fatal("externals disk not rebuilt for a new version")
1441 }
1442 if d.execDisk != execDisk {
1443 t.Fatal("exec disk changed")
1444 }
1445 if _, err := os.Stat(execDisk); err != nil {
1446 t.Fatal("exec disk removed")
1447 }
1448 if now, _ := filepath.Glob(filepath.Join(e.cfg.StateDir, "cache", "extdisk-*")); len(now) != 2 || (now[0] != first[0] && now[1] != first[0]) {
1449 // The previous disk may still be linked by a starting VM: kept for an hour.
1450 t.Fatalf("externals disks: %v", now)
1451 }
1452}
1453
1454func TestNoExternalsMeansThreeDrives(t *testing.T) {
1455 for _, src := range []sandbox.ExternalsSource{nil, &sandboxtest.Externals{}} {
1456 e := newEnv(t, "")
1457 e.cfg.Externals = src
1458 d := e.driver(t)
1459 sb, _ := d.New()
1460 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
1461 if err != nil {
1462 t.Fatal(err)
1463 }
1464 runStub(t, conn)
1465 b, _ := os.ReadFile(e.log)
1466 if strings.Contains(string(b), "externals") {
1467 t.Errorf("externals attached without any:\n%s", b)
1468 }
1469 _ = sb.Stop(context.Background())
1470 if left, _ := filepath.Glob(filepath.Join(e.cfg.StateDir, "cache", "extdisk-*")); len(left) != 0 {
1471 t.Errorf("disk built for nothing: %v", left)
1472 }
1473 }
1474}
1475
1476func TestExternalsDiskFailureDoesNotStopTheJob(t *testing.T) {
1477 e := newEnv(t, "")
1478 src := &sandboxtest.Externals{}
1479 src.Set(sandbox.External{Name: "node24", ID: "x", Dir: filepath.Join(t.TempDir(), "missing")})
1480 e.cfg.Externals = src
1481 var buf bytes.Buffer
1482 ctx := zerowrap.WithCtx(context.Background(), zerowrap.New(zerowrap.Config{Level: "debug", Format: "json", Output: &buf}))
1483 d := e.driver(t)
1484 sb, _ := d.New()
1485 conn, err := sb.Start(ctx, sandbox.JobSpec{JobID: "j"})
1486 if err != nil {
1487 t.Fatalf("a broken externals set must not stop the job: %v", err)
1488 }
1489 runStub(t, conn)
1490 _ = sb.Stop(ctx)
1491 if !strings.Contains(buf.String(), "external tools (Node.js) not attached") {
1492 t.Fatalf("failure not logged: %s", buf.String())
1493 }
1494 b, _ := os.ReadFile(e.log)
1495 if strings.Contains(string(b), `"externals"`) {
1496 t.Fatal("drive attached after a failure")
1497 }
1498}
1499
1500// With the real mkfs.ext4 the disk holds <name>/bin/node at its root, which the
1501// guest mounts at /home/runner/externals.
1502func TestExternalsDiskContent(t *testing.T) {
1503 if _, err := exec.LookPath("mkfs.ext4"); err != nil {
1504 t.Skip("mkfs.ext4 needed")
1505 }
1506 dbg, err := exec.LookPath("debugfs")
1507 if err != nil {
1508 t.Skip("debugfs needed")
1509 }
1510 set := []sandbox.External{
1511 sandboxtest.MakeExternal(t, "node24", "a", "v24"),
1512 sandboxtest.MakeExternal(t, "node20", "b", "v20"),
1513 }
1514 path, err := ensureExternalsDisk(context.Background(), cmdrun.Exec{}, set, t.TempDir())
1515 if err != nil {
1516 t.Fatal(err)
1517 }
1518 out := func(cmd string) string {
1519 b, err := exec.Command(dbg, "-R", cmd, path).CombinedOutput()
1520 if err != nil {
1521 t.Fatalf("debugfs %s: %v\n%s", cmd, err, b)
1522 }
1523 _, body, _ := strings.Cut(string(b), "\n") // the banner line
1524 return body
1525 }
1526 if ls := out("ls -l /"); !strings.Contains(ls, "node24") || !strings.Contains(ls, "node20") {
1527 t.Fatalf("root:\n%s", ls)
1528 }
1529 if got := out("cat /node24/bin/node"); got != "v24" {
1530 t.Fatalf("node24/bin/node: %q", got)
1531 }
1532 if ls := out("stat /node20/bin/node"); !strings.Contains(ls, "Mode: 0755") {
1533 t.Fatalf("mode:\n%s", ls)
1534 }
1535 if ls := out("stat /node20/nodejs"); !strings.Contains(ls, "Type: symlink") || !strings.Contains(ls, `"bin/node"`) {
1536 t.Fatalf("symlink:\n%s", ls)
1537 }
1538 if st, _ := os.Stat(path); st.Size() > 64<<20 {
1539 t.Fatalf("a tiny tree gave a %d byte disk", st.Size())
1540 }
1541}