1// SPDX-License-Identifier: MIT
2
3package vm
4
5import (
6 "bytes"
7 "context"
8 "encoding/json"
9 "errors"
10 "fmt"
11 "io"
12 "io/fs"
13 "net"
14 "net/http"
15 "net/netip"
16 "os"
17 "os/exec"
18 "path/filepath"
19 "strings"
20 "sync"
21 "syscall"
22 "testing"
23 "time"
24
25 "github.com/bnema/zerowrap"
26 "golang.org/x/sys/unix"
27
28 "bnema.dev/ermite-runner/internal/cmdrun"
29 "bnema.dev/ermite-runner/internal/execstream"
30 "bnema.dev/ermite-runner/internal/execstream/execpb"
31 "bnema.dev/ermite-runner/internal/sandbox"
32 "bnema.dev/ermite-runner/internal/sandbox/disk"
33 "bnema.dev/ermite-runner/internal/sandbox/sandboxtest"
34)
35
36// The tests run a fake Firecracker: this test binary, re-executed through a
37// shell script named "firecracker" with FC_HELPER set. It serves the HTTP API on
38// the unix socket, records every request, and on InstanceStart plays the guest:
39// it connects to the vsock socket and speaks the executor protocol.
40func TestMain(m *testing.M) {
41 if os.Getenv("FC_HELPER") != "" {
42 fakeFirecracker()
43 return
44 }
45 os.Exit(m.Run())
46}
47
48func fakeFirecracker() {
49 var sock, id string
50 args := os.Args[1:]
51 for i := 0; i+1 < len(args); i++ {
52 switch args[i] {
53 case "--api-sock":
54 sock = args[i+1]
55 case "--id":
56 id = args[i+1]
57 }
58 }
59 _ = id
60 mode := os.Getenv("FC_MODE")
61 if mode == "die" {
62 fmt.Fprintln(os.Stderr, "fake firecracker: refusing to start")
63 os.Exit(3)
64 }
65 logf, _ := os.OpenFile(os.Getenv("FC_LOG"), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
66 var vsock string
67 mux := http.NewServeMux()
68 mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
69 body, _ := io.ReadAll(r.Body)
70 if r.Method == http.MethodPut {
71 _, _ = fmt.Fprintf(logf, "%s %s %s\n", r.Method, r.URL.Path, body)
72 }
73 if r.URL.Path == "/vsock" {
74 var v vsockDevice
75 _ = json.Unmarshal(body, &v)
76 vsock = v.UDSPath
77 }
78 w.WriteHeader(http.StatusNoContent)
79 if r.URL.Path == "/actions" && mode != "hang" {
80 go func() {
81 c, err := net.Dial("unix", fmt.Sprintf("%s_%d", vsock, sandbox.VsockPort))
82 if err != nil {
83 _, _ = fmt.Fprintln(logf, "vsock dial:", err)
84 return
85 }
86 defer func() { _ = c.Close() }()
87 g := execstream.NewConn(c)
88 _ = g.Send(&execpb.Frame{Body: &execpb.Frame_Hello{Hello: &execpb.Hello{Sandbox: "vm"}}})
89 if _, err := g.Recv(); err != nil {
90 return
91 }
92 _ = g.Send(&execpb.Frame{Body: &execpb.Frame_Output{Output: &execpb.Output{Data: []byte("hello from fake guest\n")}}})
93 _ = g.Send(&execpb.Frame{Body: &execpb.Frame_Exit{Exit: &execpb.Exit{Code: 0}}})
94 // Like a real VM, stay alive until killed.
95 select {}
96 }()
97 }
98 })
99 ln, err := net.Listen("unix", sock)
100 if err != nil {
101 fmt.Fprintln(os.Stderr, err)
102 os.Exit(2)
103 }
104 _ = http.Serve(ln, mux)
105}
106
107type env struct {
108 cfg sandbox.Config
109 run *cmdrun.Fake
110 plumb *sandboxtest.Plumbing
111 log string
112 taps map[string]bool
113 mu sync.Mutex
114}
115
116// newEnv prepares a state dir, fake binaries, kernel and images.
117func newEnv(t *testing.T, mode string) *env { return newEnvNamed(t, mode, "firecracker") }
118
119// newEnvNamed runs the fake VMM under another program name (argv[0]).
120func newEnvNamed(t *testing.T, mode, argv0 string) *env {
121 t.Helper()
122 bash, err := exec.LookPath("bash")
123 if err != nil {
124 t.Skip("bash needed for the fake firecracker")
125 }
126 dir := t.TempDir()
127 e := &env{taps: map[string]bool{}, plumb: sandboxtest.NewPlumbing(), log: filepath.Join(dir, "fc.log")}
128 self, _ := os.Executable()
129 fc := filepath.Join(dir, "firecracker")
130 script := fmt.Sprintf("#!%s\nexport FC_HELPER=1 FC_MODE=%q FC_LOG=%q\nexec -a %q %q \"$@\"\n", bash, mode, e.log, argv0, self)
131 if err := os.WriteFile(fc, []byte(script), 0o755); err != nil {
132 t.Fatal(err)
133 }
134 img := filepath.Join(dir, "image")
135 _ = os.MkdirAll(img, 0o755)
136 for _, f := range []string{"vmlinux", "base.ext4"} {
137 _ = os.WriteFile(filepath.Join(img, f), []byte("x"), 0o644)
138 }
139 exe := filepath.Join(dir, "runner")
140 sandboxtest.WriteStaticELF(t, exe)
141 e.cfg = sandbox.Config{
142 Isolation: sandbox.VM, StateDir: filepath.Join(dir, "state"), Capacity: 2, Image: img,
143 JobCPUs: 1, JobMemoryMiB: 512, JobDiskMiB: 1024, UIDBase: 900000, KVMPath: "/dev/null",
144 Network: mustPrefix("10.200.0.0/16"), Firecracker: fc, JailerOff: true, Executable: exe, DNSPort: 0,
145 LockFile: filepath.Join(dir, "host.lock"), JobPids: 1000, DiskBandwidthMiB: 400, DiskIOPS: 20000, NetMbit: 1000,
146 }
147 e.run = &cmdrun.Fake{Handle: func(c cmdrun.Call) ([]byte, error) {
148 e.mu.Lock()
149 defer e.mu.Unlock()
150 switch {
151 case c.Name == "mkfs.ext4" && len(c.Args) == 1 && c.Args[0] == "-V":
152 return []byte("mke2fs 1.47.4 (6-Mar-2025)\n"), nil
153 case c.Name == "ip" && len(c.Args) > 1 && c.Args[0] == "tuntap":
154 e.taps[c.Args[3]] = true
155 case c.Name == "ip" && len(c.Args) > 2 && c.Args[0] == "link" && c.Args[1] == "del":
156 if !e.taps[c.Args[3]] {
157 return []byte("Cannot find device"), errors.New("exit status 1")
158 }
159 delete(e.taps, c.Args[3])
160 case c.Name == "ip" && len(c.Args) > 1 && c.Args[0] == "-o":
161 var b strings.Builder
162 i := 1
163 for t := range e.taps {
164 fmt.Fprintf(&b, "%d: %s: <BROADCAST> mtu 1500\n", i, t)
165 i++
166 }
167 return []byte(b.String()), nil
168 }
169 return nil, nil
170 }}
171 return e
172}
173
174func (e *env) driverScope() scope { return newScope(e.cfg.StateDir) }
175
176func (e *env) tapCount() int { e.mu.Lock(); defer e.mu.Unlock(); return len(e.taps) }
177
178func (e *env) driver(t *testing.T) *Driver {
179 t.Helper()
180 d, err := New(context.Background(), e.cfg, Options{Run: e.run, Net: e.plumb, BootTimeout: 5 * time.Second})
181 if err != nil {
182 t.Fatal(err)
183 }
184 t.Cleanup(func() { _ = d.Close() })
185 return d
186}
187
188// leftovers lists everything that must not survive a job.
189func (e *env) leftovers(t *testing.T) []string {
190 t.Helper()
191 var out []string
192 _ = filepath.WalkDir(e.cfg.StateDir, func(p string, d os.DirEntry, err error) error {
193 rel, _ := filepath.Rel(e.cfg.StateDir, p)
194 switch {
195 case err != nil || rel == ".":
196 case strings.HasPrefix(rel, "vm/") || strings.HasPrefix(rel, "vms/") || strings.HasPrefix(rel, "jail/"):
197 out = append(out, rel)
198 }
199 return nil
200 })
201 if n := e.tapCount(); n != 0 {
202 out = append(out, fmt.Sprintf("%d TAP devices", n))
203 }
204 if n := e.plumb.Live(); n != 0 {
205 out = append(out, fmt.Sprintf("%d attached guests", n))
206 }
207 return out
208}
209
210func runStub(t *testing.T, conn sandbox.Conn) (string, int) {
211 t.Helper()
212 var out strings.Builder
213 _, exit, err := execstream.Drive(conn, &execpb.Run{}, nil, func(o *execpb.Output) { out.Write(o.Data) })
214 if err != nil {
215 t.Fatal(err)
216 }
217 return out.String(), int(exit.Code)
218}
219
220func TestStartRunStop(t *testing.T) {
221 e := newEnv(t, "")
222 d := e.driver(t)
223 sb, err := d.New()
224 if err != nil {
225 t.Fatal(err)
226 }
227 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "job-1"})
228 if err != nil {
229 t.Fatal(err)
230 }
231 out, code := runStub(t, conn)
232 if out != "hello from fake guest\n" || code != 0 {
233 t.Fatalf("out=%q code=%d", out, code)
234 }
235 // While running: record, TAP, attachment, process.
236 if e.tapCount() != 1 || e.plumb.Live() != 1 {
237 t.Fatalf("tap=%d attached=%d", e.tapCount(), e.plumb.Live())
238 }
239 if recs, _ := d.store.list(); len(recs) != 1 || recs[0].Job != "job-1" || recs[0].PID == 0 || recs[0].Tap != "ertap0" {
240 t.Fatalf("records: %+v", recs)
241 }
242 pid := sb.(*vmSandbox).rec.PID
243
244 if err := sb.Stop(context.Background()); err != nil {
245 t.Fatal(err)
246 }
247 if left := e.leftovers(t); len(left) != 0 {
248 t.Fatalf("leftovers after Stop: %v", left)
249 }
250 if err := syscall.Kill(pid, 0); err == nil {
251 t.Fatalf("firecracker pid %d still alive", pid)
252 }
253 if err := sb.Stop(context.Background()); err != nil {
254 t.Fatalf("Stop must be idempotent: %v", err)
255 }
256 if d.slots.InUse() != 0 {
257 t.Fatal("slot not released")
258 }
259}
260
261// The API calls must follow the order Firecracker needs and carry the
262// plan's disk layout: exec disk (boot root, RO), base (RO), scratch (RW).
263func TestAPICallsAndDiskLayout(t *testing.T) {
264 e := newEnv(t, "")
265 d := e.driver(t)
266 sb, _ := d.New()
267 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j", CPUs: 3, MemoryMiB: 768, DiskMiB: 2048})
268 if err != nil {
269 t.Fatal(err)
270 }
271 runStub(t, conn)
272 defer func() { _ = sb.Stop(context.Background()) }()
273
274 b, _ := os.ReadFile(e.log)
275 lines := strings.Split(strings.TrimSpace(string(b)), "\n")
276 var paths []string
277 for _, l := range lines {
278 f := strings.Fields(l)
279 if len(f) >= 2 && f[0] == "PUT" {
280 paths = append(paths, f[1])
281 }
282 }
283 want := []string{"/machine-config", "/boot-source", "/drives/rootfs", "/drives/base", "/drives/scratch", "/network-interfaces/eth0", "/vsock", "/entropy", "/actions"}
284 if strings.Join(paths, ",") != strings.Join(want, ",") {
285 t.Fatalf("call order:\n got %v\nwant %v", paths, want)
286 }
287 log := string(b)
288 for _, frag := range []string{
289 `"vcpu_count":3`, `"mem_size_mib":768`, `"smt":false`,
290 `"drive_id":"rootfs"`, `"is_root_device":true,"is_read_only":true`,
291 `"drive_id":"base"`, `"is_root_device":false,"is_read_only":true`,
292 `"drive_id":"scratch"`, `"is_root_device":false,"is_read_only":false`,
293 `"host_dev_name":"ertap0"`, `"guest_cid":3`, `"action_type":"InstanceStart"`,
294 "ipv6.disable=1", "ermite.connect=vsock:1024", "ermite.gw=10.200.0.1", "ip=10.200.0.2::10.200.0.1:255.255.255.252",
295 } {
296 if !strings.Contains(log, frag) {
297 t.Errorf("API log lacks %q", frag)
298 }
299 }
300 // The scratch disk was created sparse at the requested size, and formatted.
301 var mkfs []string
302 for _, l := range e.run.Lines() {
303 if strings.HasPrefix(l, "mkfs.ext4") {
304 mkfs = append(mkfs, l)
305 }
306 }
307 if len(mkfs) == 0 {
308 t.Fatalf("no mkfs call: %v", e.run.Lines())
309 }
310}
311
312func TestJobsDoNotShareSlotsOrTaps(t *testing.T) {
313 e := newEnv(t, "")
314 d := e.driver(t)
315 var sbs []sandbox.Sandbox
316 for i := range 2 {
317 sb, err := d.New()
318 if err != nil {
319 t.Fatal(err)
320 }
321 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: fmt.Sprint(i)})
322 if err != nil {
323 t.Fatal(err)
324 }
325 runStub(t, conn)
326 sbs = append(sbs, sb)
327 }
328 if _, err := d.New(); !errors.Is(err, sandbox.ErrCapacity) {
329 t.Fatalf("capacity not enforced: %v", err)
330 }
331 if e.tapCount() != 2 {
332 t.Fatalf("taps: %v", e.taps)
333 }
334 for _, sb := range sbs {
335 _ = sb.Stop(context.Background())
336 }
337 if left := e.leftovers(t); len(left) != 0 {
338 t.Fatalf("leftovers: %v", left)
339 }
340}
341
342func TestStartFailureCleansEverything(t *testing.T) {
343 e := newEnv(t, "die")
344 d := e.driver(t)
345 sb, _ := d.New()
346 _, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
347 if err == nil || !strings.Contains(err.Error(), "refusing to start") {
348 t.Fatalf("want the VMM console in the error, got %v", err)
349 }
350 if left := e.leftovers(t); len(left) != 0 {
351 t.Fatalf("leftovers after failed Start: %v", left)
352 }
353 if d.slots.InUse() != 0 {
354 t.Fatal("slot leaked by a failed Start")
355 }
356}
357
358func TestBootTimeoutCleansEverything(t *testing.T) {
359 e := newEnv(t, "hang")
360 d, err := New(context.Background(), e.cfg, Options{Run: e.run, Net: e.plumb, BootTimeout: 300 * time.Millisecond})
361 if err != nil {
362 t.Fatal(err)
363 }
364 defer func() { _ = d.Close() }()
365 sb, _ := d.New()
366 _, err = sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
367 if err == nil || !strings.Contains(err.Error(), "did not connect") {
368 t.Fatalf("got %v", err)
369 }
370 if left := e.leftovers(t); len(left) != 0 {
371 t.Fatalf("leftovers: %v", left)
372 }
373}
374
375func TestStartCancelledByContext(t *testing.T) {
376 e := newEnv(t, "hang")
377 d := e.driver(t)
378 sb, _ := d.New()
379 ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
380 defer cancel()
381 if _, err := sb.Start(ctx, sandbox.JobSpec{JobID: "j"}); !errors.Is(err, context.DeadlineExceeded) {
382 t.Fatalf("got %v", err)
383 }
384 if left := e.leftovers(t); len(left) != 0 {
385 t.Fatalf("leftovers: %v", left)
386 }
387}
388
389func TestNetworkAttachFailureCleans(t *testing.T) {
390 e := newEnv(t, "")
391 d := e.driver(t)
392 e.plumb.FailNext = errors.New("nft broke")
393 sb, _ := d.New()
394 if _, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"}); err == nil || !strings.Contains(err.Error(), "nft broke") {
395 t.Fatalf("got %v", err)
396 }
397 if left := e.leftovers(t); len(left) != 0 {
398 t.Fatalf("leftovers: %v", left)
399 }
400}
401
402func TestStopWithoutStartReleasesSlot(t *testing.T) {
403 e := newEnv(t, "")
404 d := e.driver(t)
405 sb, _ := d.New()
406 if err := sb.Stop(context.Background()); err != nil {
407 t.Fatal(err)
408 }
409 if d.slots.InUse() != 0 {
410 t.Fatal("slot leaked")
411 }
412 if _, err := sb.Start(context.Background(), sandbox.JobSpec{}); err == nil {
413 t.Fatal("Start after Stop accepted")
414 }
415}
416
417// Stop must finish the cleanup even when the caller's context is already done
418// (a cancelled job).
419func TestStopRunsWithCancelledContext(t *testing.T) {
420 e := newEnv(t, "")
421 d := e.driver(t)
422 sb, _ := d.New()
423 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
424 if err != nil {
425 t.Fatal(err)
426 }
427 runStub(t, conn)
428 ctx, cancel := context.WithCancel(context.Background())
429 cancel()
430 if err := sb.Stop(ctx); err != nil {
431 t.Fatal(err)
432 }
433 if left := e.leftovers(t); len(left) != 0 {
434 t.Fatalf("leftovers: %v", left)
435 }
436}
437
438// A killed runner leaves a running Firecracker, a TAP, nft rules and files. The
439// next start must reap all of it (AC4: nothing survives, incl. killed jobs).
440func TestReapAfterRunnerCrash(t *testing.T) {
441 e := newEnv(t, "")
442 d1 := e.driver(t)
443 sb, _ := d1.New()
444 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
445 if err != nil {
446 t.Fatal(err)
447 }
448 runStub(t, conn)
449 pid := sb.(*vmSandbox).rec.PID
450 if err := syscall.Kill(pid, 0); err != nil {
451 t.Fatal("fake VM not running")
452 }
453 // The runner dies: its lock goes away, nothing else is cleaned.
454 _ = d1.lock.Release()
455 _ = conn.Close()
456 if e.tapCount() != 1 {
457 t.Fatal("setup: no TAP")
458 }
459
460 d2 := e.driver(t)
461 if err := d2.Reap(context.Background()); err != nil {
462 t.Fatal(err)
463 }
464 if left := e.leftovers(t); len(left) != 0 {
465 t.Fatalf("leftovers after Reap: %v", left)
466 }
467 deadline := time.Now().Add(3 * time.Second)
468 for syscall.Kill(pid, 0) == nil && time.Now().Before(deadline) {
469 // The fake is a child of this test process: it is a zombie until the
470 // original driver's Wait goroutine collects it.
471 if b, err := os.ReadFile(fmt.Sprintf("/proc/%d/stat", pid)); err == nil && strings.Contains(string(b), ") Z") {
472 break
473 }
474 time.Sleep(20 * time.Millisecond)
475 }
476 if b, _ := os.ReadFile(fmt.Sprintf("/proc/%d/stat", pid)); len(b) > 0 && !strings.Contains(string(b), ") Z") {
477 t.Fatalf("firecracker %d survived Reap: %s", pid, b)
478 }
479 if e.plumb.Resets == 0 {
480 t.Fatal("the nftables table was not reset")
481 }
482}
483
484func TestReapRemovesCorruptRecordAndTempFiles(t *testing.T) {
485 e := newEnv(t, "")
486 d := e.driver(t)
487 _ = os.MkdirAll(d.store.dir, 0o700)
488 _ = os.WriteFile(filepath.Join(d.store.dir, "er00112233.json"), []byte("{garbage"), 0o600)
489 _ = os.WriteFile(filepath.Join(d.store.dir, "er44556677.json.tmp"), []byte("{"), 0o600)
490 _ = os.MkdirAll(filepath.Join(e.cfg.StateDir, "vm", "er00112233"), 0o700)
491 _ = os.MkdirAll(filepath.Join(e.cfg.StateDir, "vm", "not-ours"), 0o700) // must be left alone
492 if err := d.Reap(context.Background()); err != nil {
493 t.Fatal(err)
494 }
495 if _, err := os.Stat(filepath.Join(e.cfg.StateDir, "vm", "not-ours")); err != nil {
496 t.Fatal("Reap removed a directory that is not ours")
497 }
498 _ = os.Remove(filepath.Join(e.cfg.StateDir, "vm", "not-ours"))
499 if left := e.leftovers(t); len(left) != 0 {
500 t.Fatalf("leftovers: %v", left)
501 }
502}
503
504func TestReapLeavesForeignTapsAlone(t *testing.T) {
505 e := newEnv(t, "")
506 e.taps["tap0"] = true
507 e.taps["ertapx"] = true // not ertap<digits>
508 e.taps["ertap7"] = true
509 d := e.driver(t)
510 if err := d.Reap(context.Background()); err != nil {
511 t.Fatal(err)
512 }
513 if !e.taps["tap0"] || !e.taps["ertapx"] || e.taps["ertap7"] {
514 t.Fatalf("taps after reap: %v", e.taps)
515 }
516}
517
518func TestSecondRunnerOnSameStateDirRefused(t *testing.T) {
519 e := newEnv(t, "")
520 _ = e.driver(t)
521 _, err := New(context.Background(), e.cfg, Options{Run: e.run, Net: e.plumb})
522 if err == nil || !strings.Contains(err.Error(), "another ermite-runner is running on this host") {
523 t.Fatalf("got %v", err)
524 }
525}
526
527func TestNewRejectsDynamicExecutable(t *testing.T) {
528 e := newEnv(t, "")
529 e.cfg.Executable = "/bin/sh"
530 if _, err := New(context.Background(), e.cfg, Options{Run: e.run, Net: e.plumb}); err == nil {
531 t.Skip("/bin/sh is static here")
532 }
533}
534
535func TestExecDiskIsKeyedByExecutableHash(t *testing.T) {
536 e := newEnv(t, "")
537 d1 := e.driver(t)
538 first := d1.execDisk
539 _ = d1.Close()
540 // Same executable: reused, no second mkfs of an exec disk.
541 n := len(e.run.Calls)
542 d2 := e.driver(t)
543 if d2.execDisk != first {
544 t.Fatal("exec disk rebuilt for the same executable")
545 }
546 for _, c := range e.run.Calls[n:] {
547 if c.Name == "mkfs.ext4" && (len(c.Args) != 1 || c.Args[0] != "-V") {
548 t.Fatal("mkfs run again for an unchanged executable")
549 }
550 }
551 _ = d2.Close()
552 // A runner upgrade: new disk, old one removed.
553 if err := os.WriteFile(e.cfg.Executable, append(readFile(t, e.cfg.Executable), 0), 0o755); err != nil {
554 t.Fatal(err)
555 }
556 d3 := e.driver(t)
557 if d3.execDisk == first {
558 t.Fatal("exec disk not rebuilt after the executable changed")
559 }
560 if _, err := os.Stat(first); err == nil {
561 t.Fatal("stale exec disk kept")
562 }
563}
564
565func readFile(t *testing.T, p string) []byte {
566 t.Helper()
567 b, err := os.ReadFile(p)
568 if err != nil {
569 t.Fatal(err)
570 }
571 return b
572}
573
574func TestJailedLayout(t *testing.T) {
575 l := newLayout("/var/lib/er", "firecracker", "er00112233", true, 900003)
576 if l.root != "/var/lib/er/jail/firecracker/er00112233/root" || l.dir != "/var/lib/er/jail/firecracker/er00112233" {
577 t.Fatalf("%+v", l)
578 }
579 if l.fc("scratch.ext4") != "/scratch.ext4" || l.host("scratch.ext4") != l.root+"/scratch.ext4" {
580 t.Fatal("jailed paths")
581 }
582 if jailBase("/var/lib/er") != "/var/lib/er/jail" {
583 t.Fatal("jail base")
584 }
585 if l2 := newLayout("/var/lib/er", "firecracker-v1.17", "er00112233", true, 1); l2.dir != "/var/lib/er/jail/firecracker-v1.17/er00112233" {
586 t.Fatalf("the jailer names its chroot after the exec file: %s", l2.dir)
587 }
588 u := newLayout("/var/lib/er", "firecracker", "er00112233", false, 1000)
589 if u.fc("api.sock") != "/var/lib/er/vm/er00112233/api.sock" {
590 t.Fatalf("unjailed path %s", u.fc("api.sock"))
591 }
592 if p, _ := u.share("/img/base.ext4", "base.ext4"); p != "/img/base.ext4" {
593 t.Fatal("unjailed share must use the file in place")
594 }
595}
596
597func TestJailedShareHardLinks(t *testing.T) {
598 dir := t.TempDir()
599 src := filepath.Join(dir, "base.ext4")
600 _ = os.WriteFile(src, []byte("data"), 0o644)
601 l := newLayout(dir, "firecracker", "er00112233", true, os.Getuid())
602 if err := os.MkdirAll(l.root, 0o700); err != nil {
603 t.Fatal(err)
604 }
605 p, err := l.share(src, "base.ext4")
606 if err != nil || p != "/base.ext4" {
607 t.Fatalf("%q %v", p, err)
608 }
609 a, _ := os.Stat(src)
610 b, _ := os.Stat(l.host("base.ext4"))
611 if !os.SameFile(a, b) {
612 t.Fatal("shared file must be a hard link, never a copy")
613 }
614 if err := checkWorldReadable(src); err != nil {
615 t.Fatal(err)
616 }
617 _ = os.Chmod(src, 0o600)
618 if err := checkWorldReadable(src); err == nil {
619 t.Fatal("unreadable-by-others file accepted for the jail")
620 }
621}
622
623func TestBootArgs(t *testing.T) {
624 a := bootArgs(mustAddr("10.200.0.6"), mustAddr("10.200.0.5"), 1791818522, false)
625 for _, want := range []string{
626 "ipv6.disable=1", "root=/dev/vda ro", "init=/ermite-runner", "reboot=k", "panic=1",
627 "ip=10.200.0.6::10.200.0.5:255.255.255.252:ermite-job:eth0:off:10.200.0.5",
628 "ermite.sandbox=vm", "ermite.connect=vsock:1024", "ermite.gw=10.200.0.5", "ermite.time=1791818522",
629 } {
630 if !strings.Contains(a, want) {
631 t.Errorf("boot args lack %q: %s", want, a)
632 }
633 }
634 if !strings.HasSuffix(a, "-- init") {
635 t.Errorf("init subcommand missing: %s", a)
636 }
637 if strings.Contains(a, "ermite.externals") {
638 t.Errorf("externals announced without a disk: %s", a)
639 }
640 if b := bootArgs(mustAddr("10.200.0.6"), mustAddr("10.200.0.5"), 1791818522, true); !strings.Contains(b, " ermite.externals=1 -- init") {
641 t.Errorf("externals not announced: %s", b)
642 }
643 if macFor(0) != "06:00:ac:10:00:00" || macFor(257) != "06:00:ac:10:01:01" {
644 t.Fatal(macFor(257))
645 }
646}
647
648func TestParseTapNames(t *testing.T) {
649 in := "1: lo: <LOOPBACK> mtu 65536\n2: eth0: <UP> mtu 1500\n5: ertap0: <BROADCAST> mtu 1500\n9: ertap12@NONE: <X>\n10: ertapx: <X>\n11: tap0: <X>\n12: ertap: <X>\n"
650 got := strings.Join(parseTapNames(in), ",")
651 if got != "ertap0,ertap12" {
652 t.Fatalf("got %q", got)
653 }
654}
655
656func TestFirecrackerProcsScan(t *testing.T) {
657 proc := t.TempDir()
658 mk := func(pid, cmdline string) {
659 _ = os.MkdirAll(filepath.Join(proc, pid), 0o755)
660 _ = os.WriteFile(filepath.Join(proc, pid, "cmdline"), []byte(cmdline), 0o644)
661 }
662 mk("100", "firecracker\x00--id\x00er00112233\x00--api-sock\x00/var/lib/er/vm/er00112233/api.sock\x00")
663 mk("101", "/usr/bin/firecracker\x00--api-sock\x00/var/lib/er/vm/er44556677/api.sock\x00--id\x00er44556677\x00")
664 mk("102", "firecracker\x00--id\x00other-tool-vm\x00") // someone else's VM
665 mk("103", "bash\x00--id\x00er00112233\x00") // not firecracker
666 mk("1", "firecracker\x00--id\x00er99999999\x00") // pid 1 is never touched
667 mk("abc", "")
668 got := vmmProcs(proc, newScope("/var/lib/er"))
669 if len(got) != 2 || got[100] != "er00112233" || got[101] != "er44556677" {
670 t.Fatalf("%v", got)
671 }
672}
673
674func TestRingKeepsTail(t *testing.T) {
675 r := newRing(8)
676 _, _ = r.Write([]byte("0123456789"))
677 _, _ = r.Write([]byte("ab"))
678 if r.String() != "23456789ab"[2:] {
679 t.Fatalf("%q", r.String())
680 }
681}
682
683func TestAPIErrorsSurfaceFirecrackerFault(t *testing.T) {
684 sock := filepath.Join(t.TempDir(), "s")
685 ln, err := net.Listen("unix", sock)
686 if err != nil {
687 t.Fatal(err)
688 }
689 defer func() { _ = ln.Close() }()
690 go func() {
691 _ = http.Serve(ln, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
692 w.WriteHeader(400)
693 _, _ = w.Write([]byte(`{"fault_message":"Invalid request"}`))
694 }))
695 }()
696 err = newAPIClient(sock).put(context.Background(), "/boot-source", bootSource{})
697 if err == nil || !strings.Contains(err.Error(), "Invalid request") || !strings.Contains(err.Error(), "/boot-source") {
698 t.Fatalf("got %v", err)
699 }
700}
701
702func mustPrefix(s string) netip.Prefix { return netip.MustParsePrefix(s) }
703func mustAddr(s string) netip.Addr { return netip.MustParseAddr(s) }
704
705// C3: processes are matched by "--id <our id>" in their command line, not by the
706// program name: a renamed or versioned Firecracker binary is still found and
707// reaped, and a recycled pid is never killed on the strength of a record alone.
708func TestReapFindsRenamedVMMByID(t *testing.T) {
709 e := newEnvNamed(t, "", "fc-renamed-1.17")
710 d1 := e.driver(t)
711 sb, _ := d1.New()
712 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
713 if err != nil {
714 t.Fatal(err)
715 }
716 runStub(t, conn)
717 pid := sb.(*vmSandbox).rec.PID
718 if got := cmdlineID("/proc", pid, e.driverScope()); got != sb.ID() {
719 t.Fatalf("cmdlineID=%q want %q (argv[0] is %q)", got, sb.ID(), "fc-renamed-1.17")
720 }
721 _ = d1.lock.Release()
722 _ = conn.Close()
723 d2 := e.driver(t)
724 if err := d2.Reap(context.Background()); err != nil {
725 t.Fatal(err)
726 }
727 if pidAlive(pid) {
728 t.Fatalf("a VMM with a renamed binary survived Reap (pid %d)", pid)
729 }
730}
731
732func TestIDFromCmdline(t *testing.T) {
733 for name, tc := range map[string]struct {
734 args []string
735 want string
736 }{
737 "firecracker": {[]string{"firecracker", "--id", "er00112233", "--api-sock", "/x"}, "er00112233"},
738 "jailer": {[]string{"jailer", "--id", "er00112233", "--exec-file", "/fc", "--uid", "1"}, "er00112233"},
739 "renamed": {[]string{"anything", "--api-sock", "/x", "--id", "er44556677"}, "er44556677"},
740 // What a real jailer 1.17 leaves: /firecracker --id <id> --start-time-us ...
741 "jailed firecracker": {[]string{"/firecracker", "--id", "er00112233", "--start-time-us", "150801025577", "--start-time-cpu-us", "60", "--parent-cpu-time-us", "1332"}, "er00112233"},
742 "foreign id": {[]string{"firecracker", "--id", "my-vm", "--api-sock", "/x"}, ""},
743 "no marker": {[]string{"python", "--id", "er00112233"}, ""},
744 "id without value": {[]string{"firecracker", "--api-sock", "/x", "--id"}, ""},
745 "path-like id": {[]string{"firecracker", "--id", "er../../../x", "--api-sock", "/x"}, ""},
746 } {
747 if got := IDFromCmdline(tc.args); got != tc.want {
748 t.Errorf("%s: got %q want %q", name, got, tc.want)
749 }
750 }
751}
752
753func TestReapDoesNotTrustARecycledPid(t *testing.T) {
754 e := newEnv(t, "")
755 d := e.driver(t)
756 // A record pointing at this very test process: its command line carries no
757 // VM id, so it must not be killed.
758 _ = d.store.save(record{ID: "er00112233", Slot: 0, PID: os.Getpid()})
759 if err := d.Reap(context.Background()); err != nil {
760 t.Fatal(err)
761 }
762 if !pidAlive(os.Getpid()) {
763 t.Fatal("Reap killed a process on the strength of a stale record")
764 }
765}
766
767// C5 + C8: when the VMM cannot be killed, the TAP, the firewall policy and the
768// files are left alone (the VM may still be running), the error wraps ErrLeaked,
769// the slot stays taken, and Reap (with a working kill) finishes the cleanup and
770// frees the slot.
771func TestStopKeepsNetworkWhenTheVMMSurvives(t *testing.T) {
772 e := newEnv(t, "")
773 stubborn := true
774 d, err := New(context.Background(), e.cfg, Options{Run: e.run, Net: e.plumb, BootTimeout: 5 * time.Second,
775 kill: func(pid int, still func() bool, to time.Duration) bool {
776 if stubborn {
777 return false
778 }
779 return killAndWait(pid, still, to)
780 }})
781 if err != nil {
782 t.Fatal(err)
783 }
784 defer func() { _ = d.Close() }()
785 sb, _ := d.New()
786 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
787 if err != nil {
788 t.Fatal(err)
789 }
790 runStub(t, conn)
791
792 err = sb.Stop(context.Background())
793 if !errors.Is(err, sandbox.ErrLeaked) {
794 t.Fatalf("Stop must report ErrLeaked, got %v", err)
795 }
796 if e.tapCount() != 1 || e.plumb.Live() != 1 {
797 t.Fatalf("the TAP and the policy must stay while the VMM may run: taps=%d attached=%d", e.tapCount(), e.plumb.Live())
798 }
799 if recs, _ := d.store.list(); len(recs) != 1 {
800 t.Fatalf("the state record must stay for Reap: %v", recs)
801 }
802 if err := sb.Stop(context.Background()); !errors.Is(err, sandbox.ErrLeaked) {
803 t.Fatalf("a second Stop must keep reporting the leak, got %v", err)
804 }
805 if d.slots.InUse() != 1 {
806 t.Fatal("the slot of a leaked sandbox stays taken")
807 }
808 // Other jobs keep running on the free slot; the leaked one is never reused.
809 other, err := d.New()
810 if err != nil {
811 t.Fatal(err)
812 }
813 if other.(*vmSandbox).slot == sb.(*vmSandbox).slot {
814 t.Fatal("a leaked slot was handed out again")
815 }
816 // Reap refuses while a sandbox is running...
817 if err := d.Reap(context.Background()); !errors.Is(err, sandbox.ErrBusy) {
818 t.Fatalf("Reap with a handed-out sandbox: %v", err)
819 }
820 _ = other.Stop(context.Background())
821 // ...and finishes the job once idle.
822 stubborn = false
823 if err := d.Reap(context.Background()); err != nil {
824 t.Fatal(err)
825 }
826 if left := e.leftovers(t); len(left) != 0 {
827 t.Fatalf("leftovers after Reap: %v", left)
828 }
829 if d.slots.InUse() != 0 {
830 t.Fatal("Reap must free the slot of the leaked sandbox")
831 }
832}
833
834// C7: Stop from another goroutine aborts a boot in progress at once, instead of
835// waiting for the boot timeout, and leaves nothing behind.
836func TestStopAbortsABootInProgress(t *testing.T) {
837 e := newEnv(t, "hang")
838 d, err := New(context.Background(), e.cfg, Options{Run: e.run, Net: e.plumb, BootTimeout: time.Minute})
839 if err != nil {
840 t.Fatal(err)
841 }
842 defer func() { _ = d.Close() }()
843 sb, _ := d.New()
844 started := make(chan error, 1)
845 go func() {
846 _, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
847 started <- err
848 }()
849 time.Sleep(700 * time.Millisecond) // the fake is now waiting for the executor
850 t0 := time.Now()
851 if err := sb.Stop(context.Background()); err != nil {
852 t.Fatal(err)
853 }
854 select {
855 case err := <-started:
856 if !errors.Is(err, sandbox.ErrAborted) {
857 t.Fatalf("Start must report that Stop aborted it, got %v", err)
858 }
859 case <-time.After(10 * time.Second):
860 t.Fatal("Start was not aborted by Stop")
861 }
862 if time.Since(t0) > 8*time.Second {
863 t.Fatalf("Stop took %v", time.Since(t0))
864 }
865 if left := e.leftovers(t); len(left) != 0 {
866 t.Fatalf("leftovers: %v", left)
867 }
868 // Stop before Start: Start must refuse.
869 sb2, _ := d.New()
870 _ = sb2.Stop(context.Background())
871 if _, err := sb2.Start(context.Background(), sandbox.JobSpec{}); err == nil {
872 t.Fatal("Start after Stop accepted")
873 }
874}
875
876// C16: the sandbox exposes its id, which names its files.
877func TestSandboxIDNamesItsFiles(t *testing.T) {
878 e := newEnv(t, "")
879 d := e.driver(t)
880 sb, _ := d.New()
881 if !sandbox.ValidID(sb.ID()) {
882 t.Fatalf("id %q", sb.ID())
883 }
884 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
885 if err != nil {
886 t.Fatal(err)
887 }
888 runStub(t, conn)
889 defer func() { _ = sb.Stop(context.Background()) }()
890 if _, err := os.Stat(filepath.Join(e.cfg.StateDir, "vm", sb.ID())); err != nil {
891 t.Fatalf("the VM directory must be named after the sandbox id: %v", err)
892 }
893 if d.Isolation() != sandbox.IsolationVM || d.Mode() != sandbox.VM {
894 t.Fatal("isolation")
895 }
896}
897
898// S7: cgroup v2 limits, a new PID namespace, and the jailer's own options.
899func TestJailerArgs(t *testing.T) {
900 args := jailerArgs("/usr/bin/jailer", "/usr/bin/firecracker", "er00112233", 900003, "/var/lib/er/jail",
901 sandbox.JobSpec{CPUs: 4, MemoryMiB: 2048}, nil)
902 line := " " + strings.Join(args, " ") + " "
903 for _, want := range []string{
904 " --id er00112233 ", " --exec-file /usr/bin/firecracker ", " --uid 900003 ", " --gid 900003 ",
905 " --chroot-base-dir /var/lib/er/jail ", " --cgroup-version 2 ", " --new-pid-ns ",
906 " --cgroup memory.max=" + fmt.Sprint((2048+vmmOverheadMiB)<<20) + " ",
907 " --cgroup pids.max=64 ", " --resource-limit no-file=4096 ", " -- ",
908 } {
909 if !strings.Contains(line, want) {
910 t.Errorf("jailer args lack %q: %s", want, line)
911 }
912 }
913 if args[len(args)-1] != "--" {
914 t.Errorf("the jailer separator must come last when there are no extra args: %v", args)
915 }
916 more := jailerArgs("j", "f", "er00112233", 1, "/b", sandbox.JobSpec{CPUs: 1, MemoryMiB: 512}, []string{"--x"})
917 if more[len(more)-2] != "--" || more[len(more)-1] != "--x" {
918 t.Errorf("extra args follow the separator: %v", more)
919 }
920}
921
922// S7: sane rate limiters on the scratch disk and both directions of eth0, off
923// when configured to zero, and only on the writable disk.
924func TestRateLimitersInTheVMConfig(t *testing.T) {
925 e := newEnv(t, "")
926 d := e.driver(t)
927 sb, _ := d.New()
928 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
929 if err != nil {
930 t.Fatal(err)
931 }
932 runStub(t, conn)
933 defer func() { _ = sb.Stop(context.Background()) }()
934 b, _ := os.ReadFile(e.log)
935 log := string(b)
936 for _, want := range []string{
937 `"rate_limiter":{"bandwidth":{"size":419430400,"one_time_burst":419430400,"refill_time":1000},"ops":{"size":20000,"one_time_burst":20000,"refill_time":1000}}`,
938 `"rx_rate_limiter":{"bandwidth":{"size":125000000,"one_time_burst":125000000,"refill_time":1000}}`,
939 `"tx_rate_limiter":{"bandwidth":{"size":125000000,"one_time_burst":125000000,"refill_time":1000}}`,
940 } {
941 if !strings.Contains(log, want) {
942 t.Errorf("API log lacks %s\n%s", want, log)
943 }
944 }
945 if n := strings.Count(log, `"rate_limiter"`); n != 1 {
946 t.Errorf("only the scratch drive is rate limited, got %d rate_limiter fields", n)
947 }
948 if diskLimiter(0, 0) != nil || netLimiter(0) != nil {
949 t.Error("zero must disable the limiters")
950 }
951 if l := diskLimiter(0, 100); l.Bandwidth != nil || l.Ops == nil {
952 t.Errorf("%+v", l)
953 }
954}
955
956// S8: the free-space check warns, and only warns.
957func TestNewWarnsOnLowFreeSpace(t *testing.T) {
958 e := newEnv(t, "")
959 var buf bytes.Buffer
960 ctx := zerowrap.WithCtx(context.Background(), zerowrap.New(zerowrap.Config{Level: "debug", Format: "json", Output: &buf}))
961 d, err := New(ctx, e.cfg, Options{Run: e.run, Net: e.plumb, FreeSpace: func(string) (int64, error) { return 1, nil }})
962 if err != nil {
963 t.Fatalf("low space must not fail New: %v", err)
964 }
965 defer func() { _ = d.Close() }()
966 if !strings.Contains(buf.String(), "less free space") {
967 t.Fatalf("no warning: %s", buf.String())
968 }
969}
970
971// S11: the scratch disk is created without following symlinks and chowned
972// through its descriptor.
973func TestMakeScratchRefusesSymlinksAndExistingFiles(t *testing.T) {
974 dir := t.TempDir()
975 target := filepath.Join(dir, "victim")
976 _ = os.WriteFile(target, []byte("precious"), 0o600)
977 link := filepath.Join(dir, "scratch.ext4")
978 if err := os.Symlink(target, link); err != nil {
979 t.Fatal(err)
980 }
981 run := &cmdrun.Fake{}
982 if err := disk.CreateScratch(context.Background(), run, link, 8, -1); err == nil {
983 t.Fatal("a planted symlink must be refused")
984 }
985 if b, _ := os.ReadFile(target); string(b) != "precious" {
986 t.Fatalf("the symlink target was modified: %q", b)
987 }
988 if len(run.Calls) != 0 {
989 t.Fatalf("mkfs must not run: %v", run.Lines())
990 }
991 ok := filepath.Join(dir, "fresh.ext4")
992 if err := disk.CreateScratch(context.Background(), run, ok, 8, os.Getuid()); err != nil {
993 t.Fatal(err)
994 }
995 if st, err := os.Stat(ok); err != nil || st.Size() != 8<<20 || st.Mode().Perm() != 0o600 {
996 t.Fatalf("%v %v", st, err)
997 }
998 if err := disk.CreateScratch(context.Background(), run, ok, 8, -1); err == nil {
999 t.Fatal("an existing file must be refused (O_EXCL)")
1000 }
1001}
1002
1003func TestOwnUsesLchown(t *testing.T) {
1004 dir := t.TempDir()
1005 l := newLayout(dir, "firecracker", "er00112233", true, os.Getuid())
1006 _ = os.MkdirAll(l.root, 0o700)
1007 target := filepath.Join(dir, "victim")
1008 _ = os.WriteFile(target, nil, 0o600)
1009 if err := os.Symlink(target, l.host("link")); err != nil {
1010 t.Fatal(err)
1011 }
1012 if err := l.own("link"); err != nil {
1013 t.Fatalf("Lchown on a link must act on the link itself: %v", err)
1014 }
1015 if err := l.own("missing"); err == nil {
1016 t.Fatal("a missing file is an error")
1017 }
1018 if u := newLayout(dir, "f", "er00112233", false, 1); u.own("anything") != nil {
1019 t.Fatal("no-op without jailer")
1020 }
1021}
1022
1023func TestRemoveCgroup(t *testing.T) {
1024 root := t.TempDir()
1025 _ = os.MkdirAll(filepath.Join(root, "firecracker", "er00112233"), 0o755)
1026 if err := removeCgroup(root, "firecracker", "er00112233"); err != nil {
1027 t.Fatal(err)
1028 }
1029 if err := removeCgroup(root, "firecracker", "er00112233"); err != nil {
1030 t.Fatalf("a missing cgroup is fine: %v", err)
1031 }
1032 _ = os.MkdirAll(filepath.Join(root, "firecracker", "er44556677", "child"), 0o755)
1033 if err := removeCgroup(root, "firecracker", "er44556677"); err == nil {
1034 t.Fatal("a non-empty cgroup cannot be removed and must be reported")
1035 }
1036}
1037
1038// Jailer 1.17 canonicalizes --chroot-base-dir and exits when it does not exist:
1039// the driver must create it (seen on a real host, invisible with the jailer off).
1040func TestNewCreatesTheJailerChrootBase(t *testing.T) {
1041 e := newEnv(t, "ok")
1042 e.cfg.JailerOff = false
1043 e.cfg.Jailer = e.cfg.Firecracker // any executable: it is only resolved here
1044 short, err := os.MkdirTemp("", "er")
1045 if err != nil {
1046 t.Fatal(err)
1047 }
1048 defer func() { _ = os.RemoveAll(short) }()
1049 e.cfg.StateDir = filepath.Join(short, "s") // the jailed API socket path must fit in sun_path
1050 d, err := New(context.Background(), e.cfg, Options{Run: e.run, Net: e.plumb, BootTimeout: 5 * time.Second})
1051 if err != nil {
1052 t.Fatal(err)
1053 }
1054 defer func() { _ = d.Close() }()
1055 st, err := os.Stat(jailBase(e.cfg.StateDir))
1056 if err != nil || !st.IsDir() {
1057 t.Fatalf("jail base not created: %v", err)
1058 }
1059}
1060
1061// GO-004: the longest socket path of a VM, in both jailer modes, 107 bytes ok /
1062// 108 rejected.
1063func TestSocketPathBoundaryInBothModes(t *testing.T) {
1064 for _, jailed := range []bool{true, false} {
1065 name := map[bool]string{true: "jailed", false: "unjailed"}[jailed]
1066 // Find the longest socket path with an empty-ish state dir, then pad the
1067 // state dir so that it is exactly maxSockPath, then one more.
1068 longest := func(state string) int {
1069 n := 0
1070 for _, p := range socketPaths(state, "firecracker", jailed) {
1071 n = max(n, len(p))
1072 }
1073 return n
1074 }
1075 base := "/s"
1076 pad := maxSockPath - longest(base)
1077 if pad < 0 {
1078 t.Fatalf("%s: base already too long", name)
1079 }
1080 ok := base + strings.Repeat("d", pad)
1081 if longest(ok) != maxSockPath {
1082 t.Fatalf("%s: longest=%d want %d", name, longest(ok), maxSockPath)
1083 }
1084 if err := checkSocketPaths(ok, "firecracker", jailed); err != nil {
1085 t.Errorf("%s: %d bytes rejected: %v", name, maxSockPath, err)
1086 }
1087 err := checkSocketPaths(ok+"d", "firecracker", jailed)
1088 if err == nil || !strings.Contains(err.Error(), "limit of a Unix socket") {
1089 t.Errorf("%s: %d bytes accepted: %v", name, maxSockPath+1, err)
1090 }
1091 }
1092}
1093
1094func TestNewRejectsAStateDirThatMakesASocketPathTooLong(t *testing.T) {
1095 for _, jailerOff := range []bool{false, true} {
1096 e := newEnv(t, "ok")
1097 e.cfg.JailerOff = jailerOff
1098 e.cfg.Jailer = e.cfg.Firecracker
1099 e.cfg.StateDir = filepath.Join(e.cfg.StateDir, strings.Repeat("d", 60))
1100 _, err := New(context.Background(), e.cfg, Options{Run: e.run, Net: e.plumb, BootTimeout: 5 * time.Second})
1101 if err == nil || !strings.Contains(err.Error(), "limit of a Unix socket") {
1102 t.Fatalf("jailerOff=%v: want a clear socket path error, got %v", jailerOff, err)
1103 }
1104 }
1105}
1106
1107// GO-003: the chroot base must be a real directory, owned by root (or the
1108// runner's own user), not writable by group or others; an existing one is never
1109// chmod-ed into compliance.
1110func TestEnsureJailBase(t *testing.T) {
1111 root := t.TempDir()
1112 good := filepath.Join(root, "good")
1113 if err := ensureJailBase(good); err != nil {
1114 t.Fatal(err)
1115 }
1116 if fi, _ := os.Stat(good); fi.Mode().Perm() != 0o700 {
1117 t.Fatalf("created with %v", fi.Mode().Perm())
1118 }
1119 if err := ensureJailBase(good); err != nil { // idempotent
1120 t.Fatal(err)
1121 }
1122 for name, mode := range map[string]os.FileMode{"group-writable": 0o770, "other-writable": 0o707, "sticky-world": 0o777} {
1123 d := filepath.Join(root, name)
1124 _ = os.Mkdir(d, 0o700)
1125 _ = os.Chmod(d, mode)
1126 if err := ensureJailBase(d); err == nil || !strings.Contains(err.Error(), "writable by group or others") {
1127 t.Errorf("%s: %v", name, err)
1128 }
1129 if fi, _ := os.Stat(d); fi.Mode().Perm() != mode {
1130 t.Errorf("%s: mode was changed to %v", name, fi.Mode().Perm())
1131 }
1132 }
1133 rx := filepath.Join(root, "rx")
1134 _ = os.Mkdir(rx, 0o755) // readable by others is fine
1135 _ = os.Chmod(rx, 0o755)
1136 if err := ensureJailBase(rx); err != nil {
1137 t.Errorf("0755 refused: %v", err)
1138 }
1139 file := filepath.Join(root, "file")
1140 _ = os.WriteFile(file, nil, 0o600)
1141 if err := ensureJailBase(file); err == nil {
1142 t.Error("a file accepted")
1143 }
1144 link := filepath.Join(root, "link")
1145 _ = os.Symlink(good, link)
1146 if err := ensureJailBase(link); err == nil || !strings.Contains(err.Error(), "not a directory") {
1147 t.Errorf("a symlink accepted: %v", err)
1148 }
1149}
1150
1151type fakeInfo struct {
1152 fs.FileInfo
1153 uid uint32
1154}
1155
1156func (f fakeInfo) Sys() any { return &syscall.Stat_t{Uid: f.uid} }
1157
1158func TestJailBaseOwner(t *testing.T) {
1159 fi, err := os.Stat(t.TempDir())
1160 if err != nil {
1161 t.Fatal(err)
1162 }
1163 d := "/x/jail"
1164 mine := os.Geteuid()
1165 if err := checkJailBaseInfo(d, fakeInfo{fi, 0}, mine); err != nil {
1166 t.Errorf("root-owned refused: %v", err)
1167 }
1168 if err := checkJailBaseInfo(d, fakeInfo{fi, uint32(mine)}, mine); err != nil {
1169 t.Errorf("own user refused: %v", err)
1170 }
1171 if err := checkJailBaseInfo(d, fakeInfo{fi, uint32(mine) + 4242}, mine); err == nil || !strings.Contains(err.Error(), "owned by uid") {
1172 t.Errorf("foreign owner accepted: %v", err)
1173 }
1174}
1175
1176// GO-001: a Firecracker with only the jailer's --start-time-us is ours only when
1177// it lives in our jail base; a jailer only when it was given our chroot base.
1178func TestScanIsScopedToOurJailBase(t *testing.T) {
1179 state := t.TempDir()
1180 sc := newScope(state)
1181 foreign := t.TempDir()
1182 proc := t.TempDir()
1183 mk := func(pid, root string, args ...string) {
1184 dir := filepath.Join(proc, pid)
1185 _ = os.MkdirAll(dir, 0o755)
1186 _ = os.WriteFile(filepath.Join(dir, "cmdline"), []byte(strings.Join(args, "\x00")+"\x00"), 0o644)
1187 if root != "" {
1188 if err := os.Symlink(root, filepath.Join(dir, "root")); err != nil {
1189 t.Fatal(err)
1190 }
1191 }
1192 }
1193 ours := filepath.Join(jailBase(state), "firecracker", "er00112233", "root")
1194 if err := os.MkdirAll(ours, 0o755); err != nil {
1195 t.Fatal(err)
1196 }
1197 // What a pivot_root'ed jailed VMM looks like: its root is our chroot directory
1198 // but readlink does not say so (it says "/"); a symlinked alias reproduces
1199 // that: another path, same inode.
1200 alias := filepath.Join(t.TempDir(), "alias")
1201 if err := os.Symlink(ours, alias); err != nil {
1202 t.Fatal(err)
1203 }
1204 // Our jail base has a chroot for another id: a foreign VMM must not match it.
1205 if err := os.MkdirAll(filepath.Join(jailBase(state), "firecracker", "er44556677", "root"), 0o755); err != nil {
1206 t.Fatal(err)
1207 }
1208 foreignRoot := filepath.Join(foreign, "jail", "firecracker", "er44556677", "root")
1209 _ = os.MkdirAll(foreignRoot, 0o755)
1210 mk("100", ours, "/firecracker", "--id", "er00112233", "--start-time-us", "1", "--start-time-cpu-us", "2")
1211 mk("101", foreignRoot, "/firecracker", "--id", "er44556677", "--start-time-us", "1")
1212 mk("109", alias, "/firecracker", "--id", "er00112233", "--start-time-us", "1") // same inode, other path
1213 mk("110", alias, "/firecracker", "--id", "er77777777", "--start-time-us", "1") // our dir, but not that id's chroot
1214 mk("102", "/", "/firecracker", "--id", "er8899aabb", "--start-time-us", "1") // not chrooted at all
1215 mk("103", "", "/firecracker", "--id", "er0000cccc", "--start-time-us", "1") // root unreadable
1216 mk("104", "/", "jailer", "--id", "er11111111", "--exec-file", "/fc", "--chroot-base-dir", jailBase(state))
1217 mk("105", "/", "jailer", "--id", "er22222222", "--exec-file", "/fc", "--chroot-base-dir", filepath.Join(foreign, "jail"))
1218 mk("106", "/", "jailer", "--id", "er33333333", "--exec-file", "/fc")
1219 mk("107", ours+"/../../../../x", "/firecracker", "--id", "er55555555", "--start-time-us", "1") // lexical escape
1220 vmDir := filepath.Join(state, "vm", "er66666666")
1221 mk("108", "/", "firecracker", "--id", "er66666666", "--api-sock", filepath.Join(vmDir, "api.sock")) // unjailed, in our dir
1222 mk("111", "/", "firecracker", "--id", "er99999999", "--api-sock", "/x") // socket elsewhere
1223 mk("112", "/", "firecracker", "--id", "erabababab", "--api-sock", filepath.Join(vmDir, "api.sock")) // our dir, another id
1224 mk("113", "/", "firecracker", "--id", "er66666666", "--api-sock", vmDir+"/../../x/api.sock") // escapes by ..
1225 mk("114", "/", "firecracker", "--id", "er66666666", "--api-sock", "vm/er66666666/api.sock") // relative
1226 mk("115", "/", "firecracker", "--id", "er66666666", "--api-sock") // no value
1227 mk("116", "/", "firecracker", "--id", "er66666666", "--api-sock", vmDir+"x/api.sock") // prefix sibling
1228 got := vmmProcs(proc, sc)
1229 want := map[int]string{100: "er00112233", 104: "er11111111", 108: "er66666666", 109: "er00112233"}
1230 if len(got) != len(want) {
1231 t.Fatalf("got %v want %v", got, want)
1232 }
1233 for pid, id := range want {
1234 if got[pid] != id {
1235 t.Fatalf("got %v want %v", got, want)
1236 }
1237 }
1238 if cmdlineID(proc, 101, sc) != "" || cmdlineID(proc, 100, sc) != "er00112233" {
1239 t.Fatal("cmdlineID must apply the same scope")
1240 }
1241}
1242
1243// GO-001 (pid reuse): the kill goes through a pidfd, after the identity check.
1244func startSleeper(t *testing.T) *exec.Cmd {
1245 t.Helper()
1246 c := exec.Command("sleep", "60")
1247 if err := c.Start(); err != nil {
1248 t.Skipf("cannot start sleep: %v", err)
1249 }
1250 t.Cleanup(func() { _ = c.Process.Kill(); _ = c.Wait() })
1251 return c
1252}
1253
1254func needPidfd(t *testing.T) {
1255 t.Helper()
1256 fd, err := unix.PidfdOpen(os.Getpid(), 0)
1257 if err != nil {
1258 t.Skipf("no pidfd on this kernel: %v", err)
1259 }
1260 _ = unix.Close(fd)
1261}
1262
1263func TestKillAndWaitKillsThroughThePidfd(t *testing.T) {
1264 needPidfd(t)
1265 c := startSleeper(t)
1266 asked := false
1267 if !killAndWait(c.Process.Pid, func() bool { asked = true; return true }, 5*time.Second) {
1268 t.Fatal("the process is still there")
1269 }
1270 if !asked {
1271 t.Fatal("the identity check did not run")
1272 }
1273 // The child is a zombie now (we have not waited for it): it must count as gone.
1274 if _, err := c.Process.Wait(); err != nil {
1275 t.Fatal(err)
1276 }
1277}
1278
1279func TestKillAndWaitDoesNotKillAMismatchAfterTheOpen(t *testing.T) {
1280 needPidfd(t)
1281 c := startSleeper(t)
1282 // The pid now belongs to something that is not the VMM we meant (what a
1283 // recycled pid looks like): no signal, and "gone" (the VMM we meant is).
1284 if !killAndWait(c.Process.Pid, func() bool { return false }, 2*time.Second) {
1285 t.Fatal("a mismatch means the intended process is gone: true expected")
1286 }
1287 if !pidAlive(c.Process.Pid) {
1288 t.Fatal("a process that failed the identity check was killed")
1289 }
1290}
1291
1292func TestKillAndWaitOnAProcessThatIsAlreadyGone(t *testing.T) {
1293 needPidfd(t)
1294 c := exec.Command("true")
1295 if err := c.Run(); err != nil {
1296 t.Skip(err)
1297 }
1298 called := false
1299 if !killAndWait(c.Process.Pid, func() bool { called = true; return true }, time.Second) {
1300 t.Fatal("a reaped process must count as gone")
1301 }
1302 _ = called
1303}
1304
1305func TestKillAndWaitReportsAProcessThatStays(t *testing.T) {
1306 needPidfd(t)
1307 // pid 1 cannot be killed by us in a test (and must never be: the identity
1308 // check says it is ours here only to show the failure path would not hang).
1309 if os.Geteuid() == 0 {
1310 t.Skip("root could signal pid 1")
1311 }
1312 if killAndWait(1, func() bool { return true }, 200*time.Millisecond) {
1313 t.Fatal("an unkillable process was reported gone")
1314 }
1315}
1316
1317// Guest forge access: the exception reaches the policy of the guest, and the
1318// warning naming the origin is logged at the driver start and at every job.
1319func TestGuestForgeIsAttachedAndWarnedAtEveryJob(t *testing.T) {
1320 e := newEnv(t, "")
1321 var err error
1322 if e.cfg.GuestForge, err = sandbox.ParseGuestForge("https://ermite.example.org:8443", false); err != nil {
1323 t.Fatal(err)
1324 }
1325 var buf bytes.Buffer
1326 ctx := zerowrap.WithCtx(context.Background(), zerowrap.New(zerowrap.Config{Level: "debug", Format: "json", Output: &buf}))
1327 d, err := New(ctx, e.cfg, Options{Run: e.run, Net: e.plumb, BootTimeout: 5 * time.Second})
1328 if err != nil {
1329 t.Fatal(err)
1330 }
1331 defer func() { _ = d.Close() }()
1332 msg := "guest forge access: sandboxes may reach https://ermite.example.org:8443 on the host"
1333 if n := strings.Count(buf.String(), msg); n != 1 {
1334 t.Fatalf("one warning at driver start, got %d", n)
1335 }
1336 for i := range 2 {
1337 sb, _ := d.New()
1338 conn, err := sb.Start(ctx, sandbox.JobSpec{JobID: fmt.Sprint("job", i)})
1339 if err != nil {
1340 t.Fatal(err)
1341 }
1342 if len(e.plumb.Attached) != 1 {
1343 t.Fatalf("%d attached", len(e.plumb.Attached))
1344 }
1345 for _, g := range e.plumb.Attached {
1346 if g.ForgePort != 8443 {
1347 t.Errorf("guest attached with ForgePort %d", g.ForgePort)
1348 }
1349 }
1350 runStub(t, conn)
1351 _ = sb.Stop(ctx)
1352 }
1353 if n := strings.Count(buf.String(), msg); n != 3 {
1354 t.Fatalf("the warning must be logged at every job start (1 + 2), got %d", n)
1355 }
1356}
1357
1358// The host's external tools ride on a fourth, read-only drive, keyed by their
1359// content: a new Node.js builds a new disk and leaves the exec disk alone.
1360func TestExternalsDiskIsAttachedReadOnlyAndKeyedByContent(t *testing.T) {
1361 e := newEnv(t, "")
1362 src := &sandboxtest.Externals{}
1363 n24 := sandboxtest.MakeExternal(t, "node24", "node24-24.21.0-x64-aaaa", "v24.21.0")
1364 n20 := sandboxtest.MakeExternal(t, "node20", "node20-20.20.2-x64-bbbb", "v20.20.2")
1365 src.Set(n24, n20)
1366 e.cfg.Externals = src
1367 d := e.driver(t)
1368 execDisk := d.execDisk
1369
1370 boot := func() (drives []string, log string) {
1371 t.Helper()
1372 _ = os.Remove(e.log)
1373 sb, _ := d.New()
1374 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
1375 if err != nil {
1376 t.Fatal(err)
1377 }
1378 runStub(t, conn)
1379 defer func() { _ = sb.Stop(context.Background()) }()
1380 b, _ := os.ReadFile(e.log)
1381 log = string(b)
1382 for _, l := range strings.Split(log, "\n") {
1383 if f := strings.Fields(l); len(f) >= 2 && f[0] == "PUT" && strings.HasPrefix(f[1], "/drives/") {
1384 drives = append(drives, f[1])
1385 }
1386 }
1387 return drives, log
1388 }
1389 drives, log := boot()
1390 if strings.Join(drives, ",") != "/drives/rootfs,/drives/base,/drives/scratch,/drives/externals" {
1391 t.Fatalf("drives: %v", drives)
1392 }
1393 if !strings.Contains(log, `"drive_id":"externals"`) || !strings.Contains(log, `"is_root_device":false,"is_read_only":true`) || !strings.Contains(log, "ermite.externals=1") {
1394 t.Fatalf("API log:\n%s", log)
1395 }
1396 // The scratch drive is still the third (vdc), and the only writable one.
1397 if strings.Count(log, `"is_read_only":false`) != 1 {
1398 t.Errorf("only the scratch drive is writable:\n%s", log)
1399 }
1400 first, _ := filepath.Glob(filepath.Join(e.cfg.StateDir, "cache", "extdisk-*"))
1401 if len(first) != 1 {
1402 t.Fatalf("externals disks: %v", first)
1403 }
1404 mkfs := func() (n int) {
1405 for _, c := range e.run.Calls {
1406 if c.Name == "mkfs.ext4" && len(c.Args) > 1 {
1407 for _, a := range c.Args {
1408 if a == "ermite-ext" {
1409 n++
1410 }
1411 }
1412 }
1413 }
1414 return
1415 }
1416 if mkfs() != 1 {
1417 t.Fatalf("externals disk built %d times", mkfs())
1418 }
1419
1420 // A second job with the same tools reuses the disk.
1421 boot()
1422 if mkfs() != 1 {
1423 t.Fatal("externals disk rebuilt for the same content")
1424 }
1425
1426 // A new Node.js: another disk, and the exec disk is untouched.
1427 src.Set(sandboxtest.MakeExternal(t, "node24", "node24-24.22.0-x64-cccc", "v24.22.0"), n20)
1428 boot()
1429 if mkfs() != 2 {
1430 t.Fatal("externals disk not rebuilt for a new version")
1431 }
1432 if d.execDisk != execDisk {
1433 t.Fatal("exec disk changed")
1434 }
1435 if _, err := os.Stat(execDisk); err != nil {
1436 t.Fatal("exec disk removed")
1437 }
1438 if now, _ := filepath.Glob(filepath.Join(e.cfg.StateDir, "cache", "extdisk-*")); len(now) != 2 || (now[0] != first[0] && now[1] != first[0]) {
1439 // The previous disk may still be linked by a starting VM: kept for an hour.
1440 t.Fatalf("externals disks: %v", now)
1441 }
1442}
1443
1444func TestNoExternalsMeansThreeDrives(t *testing.T) {
1445 for _, src := range []sandbox.ExternalsSource{nil, &sandboxtest.Externals{}} {
1446 e := newEnv(t, "")
1447 e.cfg.Externals = src
1448 d := e.driver(t)
1449 sb, _ := d.New()
1450 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
1451 if err != nil {
1452 t.Fatal(err)
1453 }
1454 runStub(t, conn)
1455 b, _ := os.ReadFile(e.log)
1456 if strings.Contains(string(b), "externals") {
1457 t.Errorf("externals attached without any:\n%s", b)
1458 }
1459 _ = sb.Stop(context.Background())
1460 if left, _ := filepath.Glob(filepath.Join(e.cfg.StateDir, "cache", "extdisk-*")); len(left) != 0 {
1461 t.Errorf("disk built for nothing: %v", left)
1462 }
1463 }
1464}
1465
1466func TestExternalsDiskFailureDoesNotStopTheJob(t *testing.T) {
1467 e := newEnv(t, "")
1468 src := &sandboxtest.Externals{}
1469 src.Set(sandbox.External{Name: "node24", ID: "x", Dir: filepath.Join(t.TempDir(), "missing")})
1470 e.cfg.Externals = src
1471 var buf bytes.Buffer
1472 ctx := zerowrap.WithCtx(context.Background(), zerowrap.New(zerowrap.Config{Level: "debug", Format: "json", Output: &buf}))
1473 d := e.driver(t)
1474 sb, _ := d.New()
1475 conn, err := sb.Start(ctx, sandbox.JobSpec{JobID: "j"})
1476 if err != nil {
1477 t.Fatalf("a broken externals set must not stop the job: %v", err)
1478 }
1479 runStub(t, conn)
1480 _ = sb.Stop(ctx)
1481 if !strings.Contains(buf.String(), "external tools (Node.js) not attached") {
1482 t.Fatalf("failure not logged: %s", buf.String())
1483 }
1484 b, _ := os.ReadFile(e.log)
1485 if strings.Contains(string(b), `"externals"`) {
1486 t.Fatal("drive attached after a failure")
1487 }
1488}
1489
1490// With the real mkfs.ext4 the disk holds <name>/bin/node at its root, which the
1491// guest mounts at /home/runner/externals.
1492func TestExternalsDiskContent(t *testing.T) {
1493 if _, err := exec.LookPath("mkfs.ext4"); err != nil {
1494 t.Skip("mkfs.ext4 needed")
1495 }
1496 dbg, err := exec.LookPath("debugfs")
1497 if err != nil {
1498 t.Skip("debugfs needed")
1499 }
1500 set := []sandbox.External{
1501 sandboxtest.MakeExternal(t, "node24", "a", "v24"),
1502 sandboxtest.MakeExternal(t, "node20", "b", "v20"),
1503 }
1504 path, err := ensureExternalsDisk(context.Background(), cmdrun.Exec{}, set, t.TempDir())
1505 if err != nil {
1506 t.Fatal(err)
1507 }
1508 out := func(cmd string) string {
1509 b, err := exec.Command(dbg, "-R", cmd, path).CombinedOutput()
1510 if err != nil {
1511 t.Fatalf("debugfs %s: %v\n%s", cmd, err, b)
1512 }
1513 _, body, _ := strings.Cut(string(b), "\n") // the banner line
1514 return body
1515 }
1516 if ls := out("ls -l /"); !strings.Contains(ls, "node24") || !strings.Contains(ls, "node20") {
1517 t.Fatalf("root:\n%s", ls)
1518 }
1519 if got := out("cat /node24/bin/node"); got != "v24" {
1520 t.Fatalf("node24/bin/node: %q", got)
1521 }
1522 if ls := out("stat /node20/bin/node"); !strings.Contains(ls, "Mode: 0755") {
1523 t.Fatalf("mode:\n%s", ls)
1524 }
1525 if ls := out("stat /node20/nodejs"); !strings.Contains(ls, "Type: symlink") || !strings.Contains(ls, `"bin/node"`) {
1526 t.Fatalf("symlink:\n%s", ls)
1527 }
1528 if st, _ := os.Stat(path); st.Size() > 64<<20 {
1529 t.Fatalf("a tiny tree gave a %d byte disk", st.Size())
1530 }
1531}