1// SPDX-License-Identifier: MIT
2
3package vm
4
5import (
6 "bytes"
7 "context"
8 "encoding/json"
9 "errors"
10 "fmt"
11 "io"
12 "io/fs"
13 "net"
14 "net/http"
15 "net/netip"
16 "os"
17 "os/exec"
18 "path/filepath"
19 "strings"
20 "sync"
21 "syscall"
22 "testing"
23 "time"
24
25 "github.com/bnema/zerowrap"
26 "golang.org/x/sys/unix"
27
28 "bnema.dev/ermite-runner/internal/cmdrun"
29 "bnema.dev/ermite-runner/internal/execstream"
30 "bnema.dev/ermite-runner/internal/execstream/execpb"
31 "bnema.dev/ermite-runner/internal/sandbox"
32 "bnema.dev/ermite-runner/internal/sandbox/disk"
33 "bnema.dev/ermite-runner/internal/sandbox/sandboxtest"
34)
35
36// The tests run a fake Firecracker: this test binary, re-executed through a
37// shell script named "firecracker" with FC_HELPER set. It serves the HTTP API on
38// the unix socket, records every request, and on InstanceStart plays the guest:
39// it connects to the vsock socket and speaks the executor protocol.
40func TestMain(m *testing.M) {
41 if os.Getenv("FC_HELPER") != "" {
42 fakeFirecracker()
43 return
44 }
45 os.Exit(m.Run())
46}
47
48func fakeFirecracker() {
49 var sock, id string
50 args := os.Args[1:]
51 for i := 0; i+1 < len(args); i++ {
52 switch args[i] {
53 case "--api-sock":
54 sock = args[i+1]
55 case "--id":
56 id = args[i+1]
57 }
58 }
59 _ = id
60 mode := os.Getenv("FC_MODE")
61 if mode == "die" {
62 fmt.Fprintln(os.Stderr, "fake firecracker: refusing to start")
63 os.Exit(3)
64 }
65 logf, _ := os.OpenFile(os.Getenv("FC_LOG"), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
66 var vsock string
67 mux := http.NewServeMux()
68 mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
69 body, _ := io.ReadAll(r.Body)
70 if r.Method == http.MethodPut {
71 _, _ = fmt.Fprintf(logf, "%s %s %s\n", r.Method, r.URL.Path, body)
72 }
73 if r.URL.Path == "/vsock" {
74 var v vsockDevice
75 _ = json.Unmarshal(body, &v)
76 vsock = v.UDSPath
77 }
78 w.WriteHeader(http.StatusNoContent)
79 if r.URL.Path == "/actions" && mode != "hang" {
80 go func() {
81 c, err := net.Dial("unix", fmt.Sprintf("%s_%d", vsock, sandbox.VsockPort))
82 if err != nil {
83 _, _ = fmt.Fprintln(logf, "vsock dial:", err)
84 return
85 }
86 defer func() { _ = c.Close() }()
87 g := execstream.NewConn(c)
88 _ = g.Send(&execpb.Frame{Body: &execpb.Frame_Hello{Hello: &execpb.Hello{Sandbox: "vm"}}})
89 if _, err := g.Recv(); err != nil {
90 return
91 }
92 _ = g.Send(&execpb.Frame{Body: &execpb.Frame_Output{Output: &execpb.Output{Data: []byte("hello from fake guest\n")}}})
93 _ = g.Send(&execpb.Frame{Body: &execpb.Frame_Exit{Exit: &execpb.Exit{Code: 0}}})
94 // Like a real VM, stay alive until killed.
95 select {}
96 }()
97 }
98 })
99 ln, err := net.Listen("unix", sock)
100 if err != nil {
101 fmt.Fprintln(os.Stderr, err)
102 os.Exit(2)
103 }
104 _ = http.Serve(ln, mux)
105}
106
107type env struct {
108 cfg sandbox.Config
109 run *cmdrun.Fake
110 plumb *sandboxtest.Plumbing
111 log string
112 taps map[string]bool
113 mu sync.Mutex
114}
115
116// newEnv prepares a state dir, fake binaries, kernel and images.
117func newEnv(t *testing.T, mode string) *env { return newEnvNamed(t, mode, "firecracker") }
118
119// newEnvNamed runs the fake VMM under another program name (argv[0]).
120func newEnvNamed(t *testing.T, mode, argv0 string) *env {
121 t.Helper()
122 bash, err := exec.LookPath("bash")
123 if err != nil {
124 t.Skip("bash needed for the fake firecracker")
125 }
126 dir := t.TempDir()
127 e := &env{taps: map[string]bool{}, plumb: sandboxtest.NewPlumbing(), log: filepath.Join(dir, "fc.log")}
128 self, _ := os.Executable()
129 fc := filepath.Join(dir, "firecracker")
130 script := fmt.Sprintf("#!%s\nexport FC_HELPER=1 FC_MODE=%q FC_LOG=%q\nexec -a %q %q \"$@\"\n", bash, mode, e.log, argv0, self)
131 if err := os.WriteFile(fc, []byte(script), 0o755); err != nil {
132 t.Fatal(err)
133 }
134 img := filepath.Join(dir, "image")
135 _ = os.MkdirAll(img, 0o755)
136 for _, f := range []string{"vmlinux", "base.ext4"} {
137 _ = os.WriteFile(filepath.Join(img, f), []byte("x"), 0o644)
138 }
139 exe := filepath.Join(dir, "runner")
140 sandboxtest.WriteStaticELF(t, exe)
141 e.cfg = sandbox.Config{
142 Isolation: sandbox.VM, StateDir: filepath.Join(dir, "state"), Capacity: 2, Image: img,
143 JobCPUs: 1, JobMemoryMiB: 512, JobDiskMiB: 1024, UIDBase: 900000, KVMPath: "/dev/null",
144 Network: mustPrefix("10.200.0.0/16"), Firecracker: fc, JailerOff: true, Executable: exe, DNSPort: 0,
145 LockFile: filepath.Join(dir, "host.lock"), JobPids: 1000, DiskBandwidthMiB: 400, DiskIOPS: 20000, NetMbit: 1000,
146 }
147 e.run = &cmdrun.Fake{Handle: func(c cmdrun.Call) ([]byte, error) {
148 e.mu.Lock()
149 defer e.mu.Unlock()
150 switch {
151 case c.Name == "mkfs.ext4" && len(c.Args) == 1 && c.Args[0] == "-V":
152 return []byte("mke2fs 1.47.4 (6-Mar-2025)\n"), nil
153 case c.Name == "ip" && len(c.Args) > 1 && c.Args[0] == "tuntap":
154 e.taps[c.Args[3]] = true
155 case c.Name == "ip" && len(c.Args) > 2 && c.Args[0] == "link" && c.Args[1] == "del":
156 if !e.taps[c.Args[3]] {
157 return []byte("Cannot find device"), errors.New("exit status 1")
158 }
159 delete(e.taps, c.Args[3])
160 case c.Name == "ip" && len(c.Args) > 1 && c.Args[0] == "-o":
161 var b strings.Builder
162 i := 1
163 for t := range e.taps {
164 fmt.Fprintf(&b, "%d: %s: <BROADCAST> mtu 1500\n", i, t)
165 i++
166 }
167 return []byte(b.String()), nil
168 }
169 return nil, nil
170 }}
171 return e
172}
173
174func (e *env) driverScope() scope { return newScope(e.cfg.StateDir) }
175
176func (e *env) tapCount() int { e.mu.Lock(); defer e.mu.Unlock(); return len(e.taps) }
177
178func (e *env) driver(t *testing.T) *Driver {
179 t.Helper()
180 d, err := New(context.Background(), e.cfg, Options{Run: e.run, Net: e.plumb, BootTimeout: 5 * time.Second})
181 if err != nil {
182 t.Fatal(err)
183 }
184 t.Cleanup(func() { _ = d.Close() })
185 return d
186}
187
188// leftovers lists everything that must not survive a job.
189func (e *env) leftovers(t *testing.T) []string {
190 t.Helper()
191 var out []string
192 _ = filepath.WalkDir(e.cfg.StateDir, func(p string, d os.DirEntry, err error) error {
193 rel, _ := filepath.Rel(e.cfg.StateDir, p)
194 switch {
195 case err != nil || rel == ".":
196 case strings.HasPrefix(rel, "vm/") || strings.HasPrefix(rel, "vms/") || strings.HasPrefix(rel, "jail/"):
197 out = append(out, rel)
198 }
199 return nil
200 })
201 if n := e.tapCount(); n != 0 {
202 out = append(out, fmt.Sprintf("%d TAP devices", n))
203 }
204 if n := e.plumb.Live(); n != 0 {
205 out = append(out, fmt.Sprintf("%d attached guests", n))
206 }
207 return out
208}
209
210func runStub(t *testing.T, conn sandbox.Conn) (string, int) {
211 t.Helper()
212 var out strings.Builder
213 _, exit, err := execstream.Drive(conn, &execpb.Run{}, nil, func(o *execpb.Output) { out.Write(o.Data) })
214 if err != nil {
215 t.Fatal(err)
216 }
217 return out.String(), int(exit.Code)
218}
219
220func TestStartRunStop(t *testing.T) {
221 e := newEnv(t, "")
222 d := e.driver(t)
223 sb, err := d.New()
224 if err != nil {
225 t.Fatal(err)
226 }
227 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "job-1"})
228 if err != nil {
229 t.Fatal(err)
230 }
231 out, code := runStub(t, conn)
232 if out != "hello from fake guest\n" || code != 0 {
233 t.Fatalf("out=%q code=%d", out, code)
234 }
235 // While running: record, TAP, attachment, process.
236 if e.tapCount() != 1 || e.plumb.Live() != 1 {
237 t.Fatalf("tap=%d attached=%d", e.tapCount(), e.plumb.Live())
238 }
239 if recs, _ := d.store.list(); len(recs) != 1 || recs[0].Job != "job-1" || recs[0].PID == 0 || recs[0].Tap != "ertap0" {
240 t.Fatalf("records: %+v", recs)
241 }
242 pid := sb.(*vmSandbox).rec.PID
243
244 if err := sb.Stop(context.Background()); err != nil {
245 t.Fatal(err)
246 }
247 if left := e.leftovers(t); len(left) != 0 {
248 t.Fatalf("leftovers after Stop: %v", left)
249 }
250 if err := syscall.Kill(pid, 0); err == nil {
251 t.Fatalf("firecracker pid %d still alive", pid)
252 }
253 if err := sb.Stop(context.Background()); err != nil {
254 t.Fatalf("Stop must be idempotent: %v", err)
255 }
256 if d.slots.InUse() != 0 {
257 t.Fatal("slot not released")
258 }
259}
260
261// The API calls must follow the order Firecracker needs and carry the
262// plan's disk layout: exec disk (boot root, RO), base (RO), scratch (RW).
263func TestAPICallsAndDiskLayout(t *testing.T) {
264 e := newEnv(t, "")
265 d := e.driver(t)
266 sb, _ := d.New()
267 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j", CPUs: 3, MemoryMiB: 768, DiskMiB: 2048})
268 if err != nil {
269 t.Fatal(err)
270 }
271 runStub(t, conn)
272 defer func() { _ = sb.Stop(context.Background()) }()
273
274 b, _ := os.ReadFile(e.log)
275 lines := strings.Split(strings.TrimSpace(string(b)), "\n")
276 var paths []string
277 for _, l := range lines {
278 f := strings.Fields(l)
279 if len(f) >= 2 && f[0] == "PUT" {
280 paths = append(paths, f[1])
281 }
282 }
283 want := []string{"/machine-config", "/boot-source", "/drives/rootfs", "/drives/base", "/drives/scratch", "/network-interfaces/eth0", "/vsock", "/entropy", "/actions"}
284 if strings.Join(paths, ",") != strings.Join(want, ",") {
285 t.Fatalf("call order:\n got %v\nwant %v", paths, want)
286 }
287 log := string(b)
288 for _, frag := range []string{
289 `"vcpu_count":3`, `"mem_size_mib":768`, `"smt":false`,
290 `"drive_id":"rootfs"`, `"is_root_device":true,"is_read_only":true`,
291 `"drive_id":"base"`, `"is_root_device":false,"is_read_only":true`,
292 `"drive_id":"scratch"`, `"is_root_device":false,"is_read_only":false`,
293 `"host_dev_name":"ertap0"`, `"guest_cid":3`, `"action_type":"InstanceStart"`,
294 "ipv6.disable=1", "ermite.connect=vsock:1024", "ermite.gw=10.200.0.1", "ip=10.200.0.2::10.200.0.1:255.255.255.252",
295 } {
296 if !strings.Contains(log, frag) {
297 t.Errorf("API log lacks %q", frag)
298 }
299 }
300 // The scratch disk was created sparse at the requested size, and formatted.
301 var mkfs []string
302 for _, l := range e.run.Lines() {
303 if strings.HasPrefix(l, "mkfs.ext4") {
304 mkfs = append(mkfs, l)
305 }
306 }
307 if len(mkfs) == 0 {
308 t.Fatalf("no mkfs call: %v", e.run.Lines())
309 }
310}
311
312func TestJobsDoNotShareSlotsOrTaps(t *testing.T) {
313 e := newEnv(t, "")
314 d := e.driver(t)
315 var sbs []sandbox.Sandbox
316 for i := range 2 {
317 sb, err := d.New()
318 if err != nil {
319 t.Fatal(err)
320 }
321 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: fmt.Sprint(i)})
322 if err != nil {
323 t.Fatal(err)
324 }
325 runStub(t, conn)
326 sbs = append(sbs, sb)
327 }
328 if _, err := d.New(); !errors.Is(err, sandbox.ErrCapacity) {
329 t.Fatalf("capacity not enforced: %v", err)
330 }
331 if e.tapCount() != 2 {
332 t.Fatalf("taps: %v", e.taps)
333 }
334 for _, sb := range sbs {
335 _ = sb.Stop(context.Background())
336 }
337 if left := e.leftovers(t); len(left) != 0 {
338 t.Fatalf("leftovers: %v", left)
339 }
340}
341
342func TestStartFailureCleansEverything(t *testing.T) {
343 e := newEnv(t, "die")
344 d := e.driver(t)
345 sb, _ := d.New()
346 _, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
347 if err == nil || !strings.Contains(err.Error(), "refusing to start") {
348 t.Fatalf("want the VMM console in the error, got %v", err)
349 }
350 if left := e.leftovers(t); len(left) != 0 {
351 t.Fatalf("leftovers after failed Start: %v", left)
352 }
353 if d.slots.InUse() != 0 {
354 t.Fatal("slot leaked by a failed Start")
355 }
356}
357
358func TestBootTimeoutCleansEverything(t *testing.T) {
359 e := newEnv(t, "hang")
360 d, err := New(context.Background(), e.cfg, Options{Run: e.run, Net: e.plumb, BootTimeout: 300 * time.Millisecond})
361 if err != nil {
362 t.Fatal(err)
363 }
364 defer func() { _ = d.Close() }()
365 sb, _ := d.New()
366 _, err = sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
367 if err == nil || !strings.Contains(err.Error(), "did not connect") {
368 t.Fatalf("got %v", err)
369 }
370 if left := e.leftovers(t); len(left) != 0 {
371 t.Fatalf("leftovers: %v", left)
372 }
373}
374
375func TestStartCancelledByContext(t *testing.T) {
376 e := newEnv(t, "hang")
377 d := e.driver(t)
378 sb, _ := d.New()
379 ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
380 defer cancel()
381 if _, err := sb.Start(ctx, sandbox.JobSpec{JobID: "j"}); !errors.Is(err, context.DeadlineExceeded) {
382 t.Fatalf("got %v", err)
383 }
384 if left := e.leftovers(t); len(left) != 0 {
385 t.Fatalf("leftovers: %v", left)
386 }
387}
388
389func TestNetworkAttachFailureCleans(t *testing.T) {
390 e := newEnv(t, "")
391 d := e.driver(t)
392 e.plumb.FailNext = errors.New("nft broke")
393 sb, _ := d.New()
394 if _, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"}); err == nil || !strings.Contains(err.Error(), "nft broke") {
395 t.Fatalf("got %v", err)
396 }
397 if left := e.leftovers(t); len(left) != 0 {
398 t.Fatalf("leftovers: %v", left)
399 }
400}
401
402func TestStopWithoutStartReleasesSlot(t *testing.T) {
403 e := newEnv(t, "")
404 d := e.driver(t)
405 sb, _ := d.New()
406 if err := sb.Stop(context.Background()); err != nil {
407 t.Fatal(err)
408 }
409 if d.slots.InUse() != 0 {
410 t.Fatal("slot leaked")
411 }
412 if _, err := sb.Start(context.Background(), sandbox.JobSpec{}); err == nil {
413 t.Fatal("Start after Stop accepted")
414 }
415}
416
417// Stop must finish the cleanup even when the caller's context is already done
418// (a cancelled job).
419func TestStopRunsWithCancelledContext(t *testing.T) {
420 e := newEnv(t, "")
421 d := e.driver(t)
422 sb, _ := d.New()
423 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
424 if err != nil {
425 t.Fatal(err)
426 }
427 runStub(t, conn)
428 ctx, cancel := context.WithCancel(context.Background())
429 cancel()
430 if err := sb.Stop(ctx); err != nil {
431 t.Fatal(err)
432 }
433 if left := e.leftovers(t); len(left) != 0 {
434 t.Fatalf("leftovers: %v", left)
435 }
436}
437
438// A killed runner leaves a running Firecracker, a TAP, nft rules and files. The
439// next start must reap all of it (AC4: nothing survives, incl. killed jobs).
440func TestReapAfterRunnerCrash(t *testing.T) {
441 e := newEnv(t, "")
442 d1 := e.driver(t)
443 sb, _ := d1.New()
444 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
445 if err != nil {
446 t.Fatal(err)
447 }
448 runStub(t, conn)
449 pid := sb.(*vmSandbox).rec.PID
450 if err := syscall.Kill(pid, 0); err != nil {
451 t.Fatal("fake VM not running")
452 }
453 // The runner dies: its lock goes away, nothing else is cleaned.
454 _ = d1.lock.Release()
455 _ = conn.Close()
456 if e.tapCount() != 1 {
457 t.Fatal("setup: no TAP")
458 }
459
460 d2 := e.driver(t)
461 if err := d2.Reap(context.Background()); err != nil {
462 t.Fatal(err)
463 }
464 if left := e.leftovers(t); len(left) != 0 {
465 t.Fatalf("leftovers after Reap: %v", left)
466 }
467 deadline := time.Now().Add(3 * time.Second)
468 for syscall.Kill(pid, 0) == nil && time.Now().Before(deadline) {
469 // The fake is a child of this test process: it is a zombie until the
470 // original driver's Wait goroutine collects it.
471 if b, err := os.ReadFile(fmt.Sprintf("/proc/%d/stat", pid)); err == nil && strings.Contains(string(b), ") Z") {
472 break
473 }
474 time.Sleep(20 * time.Millisecond)
475 }
476 if b, _ := os.ReadFile(fmt.Sprintf("/proc/%d/stat", pid)); len(b) > 0 && !strings.Contains(string(b), ") Z") {
477 t.Fatalf("firecracker %d survived Reap: %s", pid, b)
478 }
479 if e.plumb.Resets == 0 {
480 t.Fatal("the nftables table was not reset")
481 }
482}
483
484func TestReapRemovesCorruptRecordAndTempFiles(t *testing.T) {
485 e := newEnv(t, "")
486 d := e.driver(t)
487 _ = os.MkdirAll(d.store.dir, 0o700)
488 _ = os.WriteFile(filepath.Join(d.store.dir, "er00112233.json"), []byte("{garbage"), 0o600)
489 _ = os.WriteFile(filepath.Join(d.store.dir, "er44556677.json.tmp"), []byte("{"), 0o600)
490 _ = os.MkdirAll(filepath.Join(e.cfg.StateDir, "vm", "er00112233"), 0o700)
491 _ = os.MkdirAll(filepath.Join(e.cfg.StateDir, "vm", "not-ours"), 0o700) // must be left alone
492 if err := d.Reap(context.Background()); err != nil {
493 t.Fatal(err)
494 }
495 if _, err := os.Stat(filepath.Join(e.cfg.StateDir, "vm", "not-ours")); err != nil {
496 t.Fatal("Reap removed a directory that is not ours")
497 }
498 _ = os.Remove(filepath.Join(e.cfg.StateDir, "vm", "not-ours"))
499 if left := e.leftovers(t); len(left) != 0 {
500 t.Fatalf("leftovers: %v", left)
501 }
502}
503
504func TestReapLeavesForeignTapsAlone(t *testing.T) {
505 e := newEnv(t, "")
506 e.taps["tap0"] = true
507 e.taps["ertapx"] = true // not ertap<digits>
508 e.taps["ertap7"] = true
509 d := e.driver(t)
510 if err := d.Reap(context.Background()); err != nil {
511 t.Fatal(err)
512 }
513 if !e.taps["tap0"] || !e.taps["ertapx"] || e.taps["ertap7"] {
514 t.Fatalf("taps after reap: %v", e.taps)
515 }
516}
517
518func TestSecondRunnerOnSameStateDirRefused(t *testing.T) {
519 e := newEnv(t, "")
520 _ = e.driver(t)
521 _, err := New(context.Background(), e.cfg, Options{Run: e.run, Net: e.plumb})
522 if err == nil || !strings.Contains(err.Error(), "another ermite-runner is running on this host") {
523 t.Fatalf("got %v", err)
524 }
525}
526
527func TestNewRejectsDynamicExecutable(t *testing.T) {
528 e := newEnv(t, "")
529 e.cfg.Executable = "/bin/sh"
530 if _, err := New(context.Background(), e.cfg, Options{Run: e.run, Net: e.plumb}); err == nil {
531 t.Skip("/bin/sh is static here")
532 }
533}
534
535func TestExecDiskIsKeyedByExecutableHash(t *testing.T) {
536 e := newEnv(t, "")
537 d1 := e.driver(t)
538 first := d1.execDisk
539 _ = d1.Close()
540 // Same executable: reused, no second mkfs of an exec disk.
541 n := len(e.run.Calls)
542 d2 := e.driver(t)
543 if d2.execDisk != first {
544 t.Fatal("exec disk rebuilt for the same executable")
545 }
546 for _, c := range e.run.Calls[n:] {
547 if c.Name == "mkfs.ext4" && (len(c.Args) != 1 || c.Args[0] != "-V") {
548 t.Fatal("mkfs run again for an unchanged executable")
549 }
550 }
551 _ = d2.Close()
552 // A runner upgrade: new disk, old one removed.
553 if err := os.WriteFile(e.cfg.Executable, append(readFile(t, e.cfg.Executable), 0), 0o755); err != nil {
554 t.Fatal(err)
555 }
556 d3 := e.driver(t)
557 if d3.execDisk == first {
558 t.Fatal("exec disk not rebuilt after the executable changed")
559 }
560 if _, err := os.Stat(first); err == nil {
561 t.Fatal("stale exec disk kept")
562 }
563}
564
565func readFile(t *testing.T, p string) []byte {
566 t.Helper()
567 b, err := os.ReadFile(p)
568 if err != nil {
569 t.Fatal(err)
570 }
571 return b
572}
573
574func TestJailedLayout(t *testing.T) {
575 l := newLayout("/var/lib/er", "firecracker", "er00112233", true, 900003)
576 if l.root != "/var/lib/er/jail/firecracker/er00112233/root" || l.dir != "/var/lib/er/jail/firecracker/er00112233" {
577 t.Fatalf("%+v", l)
578 }
579 if l.fc("scratch.ext4") != "/scratch.ext4" || l.host("scratch.ext4") != l.root+"/scratch.ext4" {
580 t.Fatal("jailed paths")
581 }
582 if jailBase("/var/lib/er") != "/var/lib/er/jail" {
583 t.Fatal("jail base")
584 }
585 if l2 := newLayout("/var/lib/er", "firecracker-v1.17", "er00112233", true, 1); l2.dir != "/var/lib/er/jail/firecracker-v1.17/er00112233" {
586 t.Fatalf("the jailer names its chroot after the exec file: %s", l2.dir)
587 }
588 u := newLayout("/var/lib/er", "firecracker", "er00112233", false, 1000)
589 if u.fc("api.sock") != "/var/lib/er/vm/er00112233/api.sock" {
590 t.Fatalf("unjailed path %s", u.fc("api.sock"))
591 }
592 if p, _ := u.share("/img/base.ext4", "base.ext4"); p != "/img/base.ext4" {
593 t.Fatal("unjailed share must use the file in place")
594 }
595}
596
597func TestJailedShareHardLinks(t *testing.T) {
598 dir := t.TempDir()
599 src := filepath.Join(dir, "base.ext4")
600 _ = os.WriteFile(src, []byte("data"), 0o644)
601 l := newLayout(dir, "firecracker", "er00112233", true, os.Getuid())
602 if err := os.MkdirAll(l.root, 0o700); err != nil {
603 t.Fatal(err)
604 }
605 p, err := l.share(src, "base.ext4")
606 if err != nil || p != "/base.ext4" {
607 t.Fatalf("%q %v", p, err)
608 }
609 a, _ := os.Stat(src)
610 b, _ := os.Stat(l.host("base.ext4"))
611 if !os.SameFile(a, b) {
612 t.Fatal("shared file must be a hard link, never a copy")
613 }
614 if err := checkWorldReadable(src); err != nil {
615 t.Fatal(err)
616 }
617 _ = os.Chmod(src, 0o600)
618 if err := checkWorldReadable(src); err == nil {
619 t.Fatal("unreadable-by-others file accepted for the jail")
620 }
621}
622
623func TestBootArgs(t *testing.T) {
624 a := bootArgs(mustAddr("10.200.0.6"), mustAddr("10.200.0.5"), 1791818522)
625 for _, want := range []string{
626 "ipv6.disable=1", "root=/dev/vda ro", "init=/ermite-runner", "reboot=k", "panic=1",
627 "ip=10.200.0.6::10.200.0.5:255.255.255.252:ermite-job:eth0:off:10.200.0.5",
628 "ermite.sandbox=vm", "ermite.connect=vsock:1024", "ermite.gw=10.200.0.5", "ermite.time=1791818522",
629 } {
630 if !strings.Contains(a, want) {
631 t.Errorf("boot args lack %q: %s", want, a)
632 }
633 }
634 if !strings.HasSuffix(a, "-- init") {
635 t.Errorf("init subcommand missing: %s", a)
636 }
637 if macFor(0) != "06:00:ac:10:00:00" || macFor(257) != "06:00:ac:10:01:01" {
638 t.Fatal(macFor(257))
639 }
640}
641
642func TestParseTapNames(t *testing.T) {
643 in := "1: lo: <LOOPBACK> mtu 65536\n2: eth0: <UP> mtu 1500\n5: ertap0: <BROADCAST> mtu 1500\n9: ertap12@NONE: <X>\n10: ertapx: <X>\n11: tap0: <X>\n12: ertap: <X>\n"
644 got := strings.Join(parseTapNames(in), ",")
645 if got != "ertap0,ertap12" {
646 t.Fatalf("got %q", got)
647 }
648}
649
650func TestFirecrackerProcsScan(t *testing.T) {
651 proc := t.TempDir()
652 mk := func(pid, cmdline string) {
653 _ = os.MkdirAll(filepath.Join(proc, pid), 0o755)
654 _ = os.WriteFile(filepath.Join(proc, pid, "cmdline"), []byte(cmdline), 0o644)
655 }
656 mk("100", "firecracker\x00--id\x00er00112233\x00--api-sock\x00/var/lib/er/vm/er00112233/api.sock\x00")
657 mk("101", "/usr/bin/firecracker\x00--api-sock\x00/var/lib/er/vm/er44556677/api.sock\x00--id\x00er44556677\x00")
658 mk("102", "firecracker\x00--id\x00other-tool-vm\x00") // someone else's VM
659 mk("103", "bash\x00--id\x00er00112233\x00") // not firecracker
660 mk("1", "firecracker\x00--id\x00er99999999\x00") // pid 1 is never touched
661 mk("abc", "")
662 got := vmmProcs(proc, newScope("/var/lib/er"))
663 if len(got) != 2 || got[100] != "er00112233" || got[101] != "er44556677" {
664 t.Fatalf("%v", got)
665 }
666}
667
668func TestRingKeepsTail(t *testing.T) {
669 r := newRing(8)
670 _, _ = r.Write([]byte("0123456789"))
671 _, _ = r.Write([]byte("ab"))
672 if r.String() != "23456789ab"[2:] {
673 t.Fatalf("%q", r.String())
674 }
675}
676
677func TestAPIErrorsSurfaceFirecrackerFault(t *testing.T) {
678 sock := filepath.Join(t.TempDir(), "s")
679 ln, err := net.Listen("unix", sock)
680 if err != nil {
681 t.Fatal(err)
682 }
683 defer func() { _ = ln.Close() }()
684 go func() {
685 _ = http.Serve(ln, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
686 w.WriteHeader(400)
687 _, _ = w.Write([]byte(`{"fault_message":"Invalid request"}`))
688 }))
689 }()
690 err = newAPIClient(sock).put(context.Background(), "/boot-source", bootSource{})
691 if err == nil || !strings.Contains(err.Error(), "Invalid request") || !strings.Contains(err.Error(), "/boot-source") {
692 t.Fatalf("got %v", err)
693 }
694}
695
696func mustPrefix(s string) netip.Prefix { return netip.MustParsePrefix(s) }
697func mustAddr(s string) netip.Addr { return netip.MustParseAddr(s) }
698
699// C3: processes are matched by "--id <our id>" in their command line, not by the
700// program name: a renamed or versioned Firecracker binary is still found and
701// reaped, and a recycled pid is never killed on the strength of a record alone.
702func TestReapFindsRenamedVMMByID(t *testing.T) {
703 e := newEnvNamed(t, "", "fc-renamed-1.17")
704 d1 := e.driver(t)
705 sb, _ := d1.New()
706 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
707 if err != nil {
708 t.Fatal(err)
709 }
710 runStub(t, conn)
711 pid := sb.(*vmSandbox).rec.PID
712 if got := cmdlineID("/proc", pid, e.driverScope()); got != sb.ID() {
713 t.Fatalf("cmdlineID=%q want %q (argv[0] is %q)", got, sb.ID(), "fc-renamed-1.17")
714 }
715 _ = d1.lock.Release()
716 _ = conn.Close()
717 d2 := e.driver(t)
718 if err := d2.Reap(context.Background()); err != nil {
719 t.Fatal(err)
720 }
721 if pidAlive(pid) {
722 t.Fatalf("a VMM with a renamed binary survived Reap (pid %d)", pid)
723 }
724}
725
726func TestIDFromCmdline(t *testing.T) {
727 for name, tc := range map[string]struct {
728 args []string
729 want string
730 }{
731 "firecracker": {[]string{"firecracker", "--id", "er00112233", "--api-sock", "/x"}, "er00112233"},
732 "jailer": {[]string{"jailer", "--id", "er00112233", "--exec-file", "/fc", "--uid", "1"}, "er00112233"},
733 "renamed": {[]string{"anything", "--api-sock", "/x", "--id", "er44556677"}, "er44556677"},
734 // What a real jailer 1.17 leaves: /firecracker --id <id> --start-time-us ...
735 "jailed firecracker": {[]string{"/firecracker", "--id", "er00112233", "--start-time-us", "150801025577", "--start-time-cpu-us", "60", "--parent-cpu-time-us", "1332"}, "er00112233"},
736 "foreign id": {[]string{"firecracker", "--id", "my-vm", "--api-sock", "/x"}, ""},
737 "no marker": {[]string{"python", "--id", "er00112233"}, ""},
738 "id without value": {[]string{"firecracker", "--api-sock", "/x", "--id"}, ""},
739 "path-like id": {[]string{"firecracker", "--id", "er../../../x", "--api-sock", "/x"}, ""},
740 } {
741 if got := IDFromCmdline(tc.args); got != tc.want {
742 t.Errorf("%s: got %q want %q", name, got, tc.want)
743 }
744 }
745}
746
747func TestReapDoesNotTrustARecycledPid(t *testing.T) {
748 e := newEnv(t, "")
749 d := e.driver(t)
750 // A record pointing at this very test process: its command line carries no
751 // VM id, so it must not be killed.
752 _ = d.store.save(record{ID: "er00112233", Slot: 0, PID: os.Getpid()})
753 if err := d.Reap(context.Background()); err != nil {
754 t.Fatal(err)
755 }
756 if !pidAlive(os.Getpid()) {
757 t.Fatal("Reap killed a process on the strength of a stale record")
758 }
759}
760
761// C5 + C8: when the VMM cannot be killed, the TAP, the firewall policy and the
762// files are left alone (the VM may still be running), the error wraps ErrLeaked,
763// the slot stays taken, and Reap (with a working kill) finishes the cleanup and
764// frees the slot.
765func TestStopKeepsNetworkWhenTheVMMSurvives(t *testing.T) {
766 e := newEnv(t, "")
767 stubborn := true
768 d, err := New(context.Background(), e.cfg, Options{Run: e.run, Net: e.plumb, BootTimeout: 5 * time.Second,
769 kill: func(pid int, still func() bool, to time.Duration) bool {
770 if stubborn {
771 return false
772 }
773 return killAndWait(pid, still, to)
774 }})
775 if err != nil {
776 t.Fatal(err)
777 }
778 defer func() { _ = d.Close() }()
779 sb, _ := d.New()
780 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
781 if err != nil {
782 t.Fatal(err)
783 }
784 runStub(t, conn)
785
786 err = sb.Stop(context.Background())
787 if !errors.Is(err, sandbox.ErrLeaked) {
788 t.Fatalf("Stop must report ErrLeaked, got %v", err)
789 }
790 if e.tapCount() != 1 || e.plumb.Live() != 1 {
791 t.Fatalf("the TAP and the policy must stay while the VMM may run: taps=%d attached=%d", e.tapCount(), e.plumb.Live())
792 }
793 if recs, _ := d.store.list(); len(recs) != 1 {
794 t.Fatalf("the state record must stay for Reap: %v", recs)
795 }
796 if err := sb.Stop(context.Background()); !errors.Is(err, sandbox.ErrLeaked) {
797 t.Fatalf("a second Stop must keep reporting the leak, got %v", err)
798 }
799 if d.slots.InUse() != 1 {
800 t.Fatal("the slot of a leaked sandbox stays taken")
801 }
802 // Other jobs keep running on the free slot; the leaked one is never reused.
803 other, err := d.New()
804 if err != nil {
805 t.Fatal(err)
806 }
807 if other.(*vmSandbox).slot == sb.(*vmSandbox).slot {
808 t.Fatal("a leaked slot was handed out again")
809 }
810 // Reap refuses while a sandbox is running...
811 if err := d.Reap(context.Background()); !errors.Is(err, sandbox.ErrBusy) {
812 t.Fatalf("Reap with a handed-out sandbox: %v", err)
813 }
814 _ = other.Stop(context.Background())
815 // ...and finishes the job once idle.
816 stubborn = false
817 if err := d.Reap(context.Background()); err != nil {
818 t.Fatal(err)
819 }
820 if left := e.leftovers(t); len(left) != 0 {
821 t.Fatalf("leftovers after Reap: %v", left)
822 }
823 if d.slots.InUse() != 0 {
824 t.Fatal("Reap must free the slot of the leaked sandbox")
825 }
826}
827
828// C7: Stop from another goroutine aborts a boot in progress at once, instead of
829// waiting for the boot timeout, and leaves nothing behind.
830func TestStopAbortsABootInProgress(t *testing.T) {
831 e := newEnv(t, "hang")
832 d, err := New(context.Background(), e.cfg, Options{Run: e.run, Net: e.plumb, BootTimeout: time.Minute})
833 if err != nil {
834 t.Fatal(err)
835 }
836 defer func() { _ = d.Close() }()
837 sb, _ := d.New()
838 started := make(chan error, 1)
839 go func() {
840 _, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
841 started <- err
842 }()
843 time.Sleep(700 * time.Millisecond) // the fake is now waiting for the executor
844 t0 := time.Now()
845 if err := sb.Stop(context.Background()); err != nil {
846 t.Fatal(err)
847 }
848 select {
849 case err := <-started:
850 if !errors.Is(err, sandbox.ErrAborted) {
851 t.Fatalf("Start must report that Stop aborted it, got %v", err)
852 }
853 case <-time.After(10 * time.Second):
854 t.Fatal("Start was not aborted by Stop")
855 }
856 if time.Since(t0) > 8*time.Second {
857 t.Fatalf("Stop took %v", time.Since(t0))
858 }
859 if left := e.leftovers(t); len(left) != 0 {
860 t.Fatalf("leftovers: %v", left)
861 }
862 // Stop before Start: Start must refuse.
863 sb2, _ := d.New()
864 _ = sb2.Stop(context.Background())
865 if _, err := sb2.Start(context.Background(), sandbox.JobSpec{}); err == nil {
866 t.Fatal("Start after Stop accepted")
867 }
868}
869
870// C16: the sandbox exposes its id, which names its files.
871func TestSandboxIDNamesItsFiles(t *testing.T) {
872 e := newEnv(t, "")
873 d := e.driver(t)
874 sb, _ := d.New()
875 if !sandbox.ValidID(sb.ID()) {
876 t.Fatalf("id %q", sb.ID())
877 }
878 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
879 if err != nil {
880 t.Fatal(err)
881 }
882 runStub(t, conn)
883 defer func() { _ = sb.Stop(context.Background()) }()
884 if _, err := os.Stat(filepath.Join(e.cfg.StateDir, "vm", sb.ID())); err != nil {
885 t.Fatalf("the VM directory must be named after the sandbox id: %v", err)
886 }
887 if d.Isolation() != sandbox.IsolationVM || d.Mode() != sandbox.VM {
888 t.Fatal("isolation")
889 }
890}
891
892// S7: cgroup v2 limits, a new PID namespace, and the jailer's own options.
893func TestJailerArgs(t *testing.T) {
894 args := jailerArgs("/usr/bin/jailer", "/usr/bin/firecracker", "er00112233", 900003, "/var/lib/er/jail",
895 sandbox.JobSpec{CPUs: 4, MemoryMiB: 2048}, nil)
896 line := " " + strings.Join(args, " ") + " "
897 for _, want := range []string{
898 " --id er00112233 ", " --exec-file /usr/bin/firecracker ", " --uid 900003 ", " --gid 900003 ",
899 " --chroot-base-dir /var/lib/er/jail ", " --cgroup-version 2 ", " --new-pid-ns ",
900 " --cgroup memory.max=" + fmt.Sprint((2048+vmmOverheadMiB)<<20) + " ",
901 " --cgroup pids.max=64 ", " --resource-limit no-file=4096 ", " -- ",
902 } {
903 if !strings.Contains(line, want) {
904 t.Errorf("jailer args lack %q: %s", want, line)
905 }
906 }
907 if args[len(args)-1] != "--" {
908 t.Errorf("the jailer separator must come last when there are no extra args: %v", args)
909 }
910 more := jailerArgs("j", "f", "er00112233", 1, "/b", sandbox.JobSpec{CPUs: 1, MemoryMiB: 512}, []string{"--x"})
911 if more[len(more)-2] != "--" || more[len(more)-1] != "--x" {
912 t.Errorf("extra args follow the separator: %v", more)
913 }
914}
915
916// S7: sane rate limiters on the scratch disk and both directions of eth0, off
917// when configured to zero, and only on the writable disk.
918func TestRateLimitersInTheVMConfig(t *testing.T) {
919 e := newEnv(t, "")
920 d := e.driver(t)
921 sb, _ := d.New()
922 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
923 if err != nil {
924 t.Fatal(err)
925 }
926 runStub(t, conn)
927 defer func() { _ = sb.Stop(context.Background()) }()
928 b, _ := os.ReadFile(e.log)
929 log := string(b)
930 for _, want := range []string{
931 `"rate_limiter":{"bandwidth":{"size":419430400,"one_time_burst":419430400,"refill_time":1000},"ops":{"size":20000,"one_time_burst":20000,"refill_time":1000}}`,
932 `"rx_rate_limiter":{"bandwidth":{"size":125000000,"one_time_burst":125000000,"refill_time":1000}}`,
933 `"tx_rate_limiter":{"bandwidth":{"size":125000000,"one_time_burst":125000000,"refill_time":1000}}`,
934 } {
935 if !strings.Contains(log, want) {
936 t.Errorf("API log lacks %s\n%s", want, log)
937 }
938 }
939 if n := strings.Count(log, `"rate_limiter"`); n != 1 {
940 t.Errorf("only the scratch drive is rate limited, got %d rate_limiter fields", n)
941 }
942 if diskLimiter(0, 0) != nil || netLimiter(0) != nil {
943 t.Error("zero must disable the limiters")
944 }
945 if l := diskLimiter(0, 100); l.Bandwidth != nil || l.Ops == nil {
946 t.Errorf("%+v", l)
947 }
948}
949
950// S8: the free-space check warns, and only warns.
951func TestNewWarnsOnLowFreeSpace(t *testing.T) {
952 e := newEnv(t, "")
953 var buf bytes.Buffer
954 ctx := zerowrap.WithCtx(context.Background(), zerowrap.New(zerowrap.Config{Level: "debug", Format: "json", Output: &buf}))
955 d, err := New(ctx, e.cfg, Options{Run: e.run, Net: e.plumb, FreeSpace: func(string) (int64, error) { return 1, nil }})
956 if err != nil {
957 t.Fatalf("low space must not fail New: %v", err)
958 }
959 defer func() { _ = d.Close() }()
960 if !strings.Contains(buf.String(), "less free space") {
961 t.Fatalf("no warning: %s", buf.String())
962 }
963}
964
965// S11: the scratch disk is created without following symlinks and chowned
966// through its descriptor.
967func TestMakeScratchRefusesSymlinksAndExistingFiles(t *testing.T) {
968 dir := t.TempDir()
969 target := filepath.Join(dir, "victim")
970 _ = os.WriteFile(target, []byte("precious"), 0o600)
971 link := filepath.Join(dir, "scratch.ext4")
972 if err := os.Symlink(target, link); err != nil {
973 t.Fatal(err)
974 }
975 run := &cmdrun.Fake{}
976 if err := disk.CreateScratch(context.Background(), run, link, 8, -1); err == nil {
977 t.Fatal("a planted symlink must be refused")
978 }
979 if b, _ := os.ReadFile(target); string(b) != "precious" {
980 t.Fatalf("the symlink target was modified: %q", b)
981 }
982 if len(run.Calls) != 0 {
983 t.Fatalf("mkfs must not run: %v", run.Lines())
984 }
985 ok := filepath.Join(dir, "fresh.ext4")
986 if err := disk.CreateScratch(context.Background(), run, ok, 8, os.Getuid()); err != nil {
987 t.Fatal(err)
988 }
989 if st, err := os.Stat(ok); err != nil || st.Size() != 8<<20 || st.Mode().Perm() != 0o600 {
990 t.Fatalf("%v %v", st, err)
991 }
992 if err := disk.CreateScratch(context.Background(), run, ok, 8, -1); err == nil {
993 t.Fatal("an existing file must be refused (O_EXCL)")
994 }
995}
996
997func TestOwnUsesLchown(t *testing.T) {
998 dir := t.TempDir()
999 l := newLayout(dir, "firecracker", "er00112233", true, os.Getuid())
1000 _ = os.MkdirAll(l.root, 0o700)
1001 target := filepath.Join(dir, "victim")
1002 _ = os.WriteFile(target, nil, 0o600)
1003 if err := os.Symlink(target, l.host("link")); err != nil {
1004 t.Fatal(err)
1005 }
1006 if err := l.own("link"); err != nil {
1007 t.Fatalf("Lchown on a link must act on the link itself: %v", err)
1008 }
1009 if err := l.own("missing"); err == nil {
1010 t.Fatal("a missing file is an error")
1011 }
1012 if u := newLayout(dir, "f", "er00112233", false, 1); u.own("anything") != nil {
1013 t.Fatal("no-op without jailer")
1014 }
1015}
1016
1017func TestRemoveCgroup(t *testing.T) {
1018 root := t.TempDir()
1019 _ = os.MkdirAll(filepath.Join(root, "firecracker", "er00112233"), 0o755)
1020 if err := removeCgroup(root, "firecracker", "er00112233"); err != nil {
1021 t.Fatal(err)
1022 }
1023 if err := removeCgroup(root, "firecracker", "er00112233"); err != nil {
1024 t.Fatalf("a missing cgroup is fine: %v", err)
1025 }
1026 _ = os.MkdirAll(filepath.Join(root, "firecracker", "er44556677", "child"), 0o755)
1027 if err := removeCgroup(root, "firecracker", "er44556677"); err == nil {
1028 t.Fatal("a non-empty cgroup cannot be removed and must be reported")
1029 }
1030}
1031
1032// Jailer 1.17 canonicalizes --chroot-base-dir and exits when it does not exist:
1033// the driver must create it (seen on a real host, invisible with the jailer off).
1034func TestNewCreatesTheJailerChrootBase(t *testing.T) {
1035 e := newEnv(t, "ok")
1036 e.cfg.JailerOff = false
1037 e.cfg.Jailer = e.cfg.Firecracker // any executable: it is only resolved here
1038 short, err := os.MkdirTemp("", "er")
1039 if err != nil {
1040 t.Fatal(err)
1041 }
1042 defer func() { _ = os.RemoveAll(short) }()
1043 e.cfg.StateDir = filepath.Join(short, "s") // the jailed API socket path must fit in sun_path
1044 d, err := New(context.Background(), e.cfg, Options{Run: e.run, Net: e.plumb, BootTimeout: 5 * time.Second})
1045 if err != nil {
1046 t.Fatal(err)
1047 }
1048 defer func() { _ = d.Close() }()
1049 st, err := os.Stat(jailBase(e.cfg.StateDir))
1050 if err != nil || !st.IsDir() {
1051 t.Fatalf("jail base not created: %v", err)
1052 }
1053}
1054
1055// GO-004: the longest socket path of a VM, in both jailer modes, 107 bytes ok /
1056// 108 rejected.
1057func TestSocketPathBoundaryInBothModes(t *testing.T) {
1058 for _, jailed := range []bool{true, false} {
1059 name := map[bool]string{true: "jailed", false: "unjailed"}[jailed]
1060 // Find the longest socket path with an empty-ish state dir, then pad the
1061 // state dir so that it is exactly maxSockPath, then one more.
1062 longest := func(state string) int {
1063 n := 0
1064 for _, p := range socketPaths(state, "firecracker", jailed) {
1065 n = max(n, len(p))
1066 }
1067 return n
1068 }
1069 base := "/s"
1070 pad := maxSockPath - longest(base)
1071 if pad < 0 {
1072 t.Fatalf("%s: base already too long", name)
1073 }
1074 ok := base + strings.Repeat("d", pad)
1075 if longest(ok) != maxSockPath {
1076 t.Fatalf("%s: longest=%d want %d", name, longest(ok), maxSockPath)
1077 }
1078 if err := checkSocketPaths(ok, "firecracker", jailed); err != nil {
1079 t.Errorf("%s: %d bytes rejected: %v", name, maxSockPath, err)
1080 }
1081 err := checkSocketPaths(ok+"d", "firecracker", jailed)
1082 if err == nil || !strings.Contains(err.Error(), "limit of a Unix socket") {
1083 t.Errorf("%s: %d bytes accepted: %v", name, maxSockPath+1, err)
1084 }
1085 }
1086}
1087
1088func TestNewRejectsAStateDirThatMakesASocketPathTooLong(t *testing.T) {
1089 for _, jailerOff := range []bool{false, true} {
1090 e := newEnv(t, "ok")
1091 e.cfg.JailerOff = jailerOff
1092 e.cfg.Jailer = e.cfg.Firecracker
1093 e.cfg.StateDir = filepath.Join(e.cfg.StateDir, strings.Repeat("d", 60))
1094 _, err := New(context.Background(), e.cfg, Options{Run: e.run, Net: e.plumb, BootTimeout: 5 * time.Second})
1095 if err == nil || !strings.Contains(err.Error(), "limit of a Unix socket") {
1096 t.Fatalf("jailerOff=%v: want a clear socket path error, got %v", jailerOff, err)
1097 }
1098 }
1099}
1100
1101// GO-003: the chroot base must be a real directory, owned by root (or the
1102// runner's own user), not writable by group or others; an existing one is never
1103// chmod-ed into compliance.
1104func TestEnsureJailBase(t *testing.T) {
1105 root := t.TempDir()
1106 good := filepath.Join(root, "good")
1107 if err := ensureJailBase(good); err != nil {
1108 t.Fatal(err)
1109 }
1110 if fi, _ := os.Stat(good); fi.Mode().Perm() != 0o700 {
1111 t.Fatalf("created with %v", fi.Mode().Perm())
1112 }
1113 if err := ensureJailBase(good); err != nil { // idempotent
1114 t.Fatal(err)
1115 }
1116 for name, mode := range map[string]os.FileMode{"group-writable": 0o770, "other-writable": 0o707, "sticky-world": 0o777} {
1117 d := filepath.Join(root, name)
1118 _ = os.Mkdir(d, 0o700)
1119 _ = os.Chmod(d, mode)
1120 if err := ensureJailBase(d); err == nil || !strings.Contains(err.Error(), "writable by group or others") {
1121 t.Errorf("%s: %v", name, err)
1122 }
1123 if fi, _ := os.Stat(d); fi.Mode().Perm() != mode {
1124 t.Errorf("%s: mode was changed to %v", name, fi.Mode().Perm())
1125 }
1126 }
1127 rx := filepath.Join(root, "rx")
1128 _ = os.Mkdir(rx, 0o755) // readable by others is fine
1129 _ = os.Chmod(rx, 0o755)
1130 if err := ensureJailBase(rx); err != nil {
1131 t.Errorf("0755 refused: %v", err)
1132 }
1133 file := filepath.Join(root, "file")
1134 _ = os.WriteFile(file, nil, 0o600)
1135 if err := ensureJailBase(file); err == nil {
1136 t.Error("a file accepted")
1137 }
1138 link := filepath.Join(root, "link")
1139 _ = os.Symlink(good, link)
1140 if err := ensureJailBase(link); err == nil || !strings.Contains(err.Error(), "not a directory") {
1141 t.Errorf("a symlink accepted: %v", err)
1142 }
1143}
1144
1145type fakeInfo struct {
1146 fs.FileInfo
1147 uid uint32
1148}
1149
1150func (f fakeInfo) Sys() any { return &syscall.Stat_t{Uid: f.uid} }
1151
1152func TestJailBaseOwner(t *testing.T) {
1153 fi, err := os.Stat(t.TempDir())
1154 if err != nil {
1155 t.Fatal(err)
1156 }
1157 d := "/x/jail"
1158 mine := os.Geteuid()
1159 if err := checkJailBaseInfo(d, fakeInfo{fi, 0}, mine); err != nil {
1160 t.Errorf("root-owned refused: %v", err)
1161 }
1162 if err := checkJailBaseInfo(d, fakeInfo{fi, uint32(mine)}, mine); err != nil {
1163 t.Errorf("own user refused: %v", err)
1164 }
1165 if err := checkJailBaseInfo(d, fakeInfo{fi, uint32(mine) + 4242}, mine); err == nil || !strings.Contains(err.Error(), "owned by uid") {
1166 t.Errorf("foreign owner accepted: %v", err)
1167 }
1168}
1169
1170// GO-001: a Firecracker with only the jailer's --start-time-us is ours only when
1171// it lives in our jail base; a jailer only when it was given our chroot base.
1172func TestScanIsScopedToOurJailBase(t *testing.T) {
1173 state := t.TempDir()
1174 sc := newScope(state)
1175 foreign := t.TempDir()
1176 proc := t.TempDir()
1177 mk := func(pid, root string, args ...string) {
1178 dir := filepath.Join(proc, pid)
1179 _ = os.MkdirAll(dir, 0o755)
1180 _ = os.WriteFile(filepath.Join(dir, "cmdline"), []byte(strings.Join(args, "\x00")+"\x00"), 0o644)
1181 if root != "" {
1182 if err := os.Symlink(root, filepath.Join(dir, "root")); err != nil {
1183 t.Fatal(err)
1184 }
1185 }
1186 }
1187 ours := filepath.Join(jailBase(state), "firecracker", "er00112233", "root")
1188 if err := os.MkdirAll(ours, 0o755); err != nil {
1189 t.Fatal(err)
1190 }
1191 // What a pivot_root'ed jailed VMM looks like: its root is our chroot directory
1192 // but readlink does not say so (it says "/"); a symlinked alias reproduces
1193 // that: another path, same inode.
1194 alias := filepath.Join(t.TempDir(), "alias")
1195 if err := os.Symlink(ours, alias); err != nil {
1196 t.Fatal(err)
1197 }
1198 // Our jail base has a chroot for another id: a foreign VMM must not match it.
1199 if err := os.MkdirAll(filepath.Join(jailBase(state), "firecracker", "er44556677", "root"), 0o755); err != nil {
1200 t.Fatal(err)
1201 }
1202 foreignRoot := filepath.Join(foreign, "jail", "firecracker", "er44556677", "root")
1203 _ = os.MkdirAll(foreignRoot, 0o755)
1204 mk("100", ours, "/firecracker", "--id", "er00112233", "--start-time-us", "1", "--start-time-cpu-us", "2")
1205 mk("101", foreignRoot, "/firecracker", "--id", "er44556677", "--start-time-us", "1")
1206 mk("109", alias, "/firecracker", "--id", "er00112233", "--start-time-us", "1") // same inode, other path
1207 mk("110", alias, "/firecracker", "--id", "er77777777", "--start-time-us", "1") // our dir, but not that id's chroot
1208 mk("102", "/", "/firecracker", "--id", "er8899aabb", "--start-time-us", "1") // not chrooted at all
1209 mk("103", "", "/firecracker", "--id", "er0000cccc", "--start-time-us", "1") // root unreadable
1210 mk("104", "/", "jailer", "--id", "er11111111", "--exec-file", "/fc", "--chroot-base-dir", jailBase(state))
1211 mk("105", "/", "jailer", "--id", "er22222222", "--exec-file", "/fc", "--chroot-base-dir", filepath.Join(foreign, "jail"))
1212 mk("106", "/", "jailer", "--id", "er33333333", "--exec-file", "/fc")
1213 mk("107", ours+"/../../../../x", "/firecracker", "--id", "er55555555", "--start-time-us", "1") // lexical escape
1214 vmDir := filepath.Join(state, "vm", "er66666666")
1215 mk("108", "/", "firecracker", "--id", "er66666666", "--api-sock", filepath.Join(vmDir, "api.sock")) // unjailed, in our dir
1216 mk("111", "/", "firecracker", "--id", "er99999999", "--api-sock", "/x") // socket elsewhere
1217 mk("112", "/", "firecracker", "--id", "erabababab", "--api-sock", filepath.Join(vmDir, "api.sock")) // our dir, another id
1218 mk("113", "/", "firecracker", "--id", "er66666666", "--api-sock", vmDir+"/../../x/api.sock") // escapes by ..
1219 mk("114", "/", "firecracker", "--id", "er66666666", "--api-sock", "vm/er66666666/api.sock") // relative
1220 mk("115", "/", "firecracker", "--id", "er66666666", "--api-sock") // no value
1221 mk("116", "/", "firecracker", "--id", "er66666666", "--api-sock", vmDir+"x/api.sock") // prefix sibling
1222 got := vmmProcs(proc, sc)
1223 want := map[int]string{100: "er00112233", 104: "er11111111", 108: "er66666666", 109: "er00112233"}
1224 if len(got) != len(want) {
1225 t.Fatalf("got %v want %v", got, want)
1226 }
1227 for pid, id := range want {
1228 if got[pid] != id {
1229 t.Fatalf("got %v want %v", got, want)
1230 }
1231 }
1232 if cmdlineID(proc, 101, sc) != "" || cmdlineID(proc, 100, sc) != "er00112233" {
1233 t.Fatal("cmdlineID must apply the same scope")
1234 }
1235}
1236
1237// GO-001 (pid reuse): the kill goes through a pidfd, after the identity check.
1238func startSleeper(t *testing.T) *exec.Cmd {
1239 t.Helper()
1240 c := exec.Command("sleep", "60")
1241 if err := c.Start(); err != nil {
1242 t.Skipf("cannot start sleep: %v", err)
1243 }
1244 t.Cleanup(func() { _ = c.Process.Kill(); _ = c.Wait() })
1245 return c
1246}
1247
1248func needPidfd(t *testing.T) {
1249 t.Helper()
1250 fd, err := unix.PidfdOpen(os.Getpid(), 0)
1251 if err != nil {
1252 t.Skipf("no pidfd on this kernel: %v", err)
1253 }
1254 _ = unix.Close(fd)
1255}
1256
1257func TestKillAndWaitKillsThroughThePidfd(t *testing.T) {
1258 needPidfd(t)
1259 c := startSleeper(t)
1260 asked := false
1261 if !killAndWait(c.Process.Pid, func() bool { asked = true; return true }, 5*time.Second) {
1262 t.Fatal("the process is still there")
1263 }
1264 if !asked {
1265 t.Fatal("the identity check did not run")
1266 }
1267 // The child is a zombie now (we have not waited for it): it must count as gone.
1268 if _, err := c.Process.Wait(); err != nil {
1269 t.Fatal(err)
1270 }
1271}
1272
1273func TestKillAndWaitDoesNotKillAMismatchAfterTheOpen(t *testing.T) {
1274 needPidfd(t)
1275 c := startSleeper(t)
1276 // The pid now belongs to something that is not the VMM we meant (what a
1277 // recycled pid looks like): no signal, and "gone" (the VMM we meant is).
1278 if !killAndWait(c.Process.Pid, func() bool { return false }, 2*time.Second) {
1279 t.Fatal("a mismatch means the intended process is gone: true expected")
1280 }
1281 if !pidAlive(c.Process.Pid) {
1282 t.Fatal("a process that failed the identity check was killed")
1283 }
1284}
1285
1286func TestKillAndWaitOnAProcessThatIsAlreadyGone(t *testing.T) {
1287 needPidfd(t)
1288 c := exec.Command("true")
1289 if err := c.Run(); err != nil {
1290 t.Skip(err)
1291 }
1292 called := false
1293 if !killAndWait(c.Process.Pid, func() bool { called = true; return true }, time.Second) {
1294 t.Fatal("a reaped process must count as gone")
1295 }
1296 _ = called
1297}
1298
1299func TestKillAndWaitReportsAProcessThatStays(t *testing.T) {
1300 needPidfd(t)
1301 // pid 1 cannot be killed by us in a test (and must never be: the identity
1302 // check says it is ours here only to show the failure path would not hang).
1303 if os.Geteuid() == 0 {
1304 t.Skip("root could signal pid 1")
1305 }
1306 if killAndWait(1, func() bool { return true }, 200*time.Millisecond) {
1307 t.Fatal("an unkillable process was reported gone")
1308 }
1309}