1// SPDX-License-Identifier: MIT
2
3// Package vm is the micro-VM sandbox: one Firecracker VM per job.
4//
5// Per VM, on the host: a jailer chroot (or, without jailer, a plain directory)
6// holding the per-job sparse scratch disk and the vsock socket; a TAP device
7// with a /30; rules in the dedicated nftables table; the DNS forwarder bound on
8// the TAP's host address. In the guest: three virtio disks (read-only runner
9// executable as the boot root, read-only shared base image, per-job scratch)
10// and a vsock device. Everything is traced in a state-directory record before it
11// is created, so that Reap can remove it after a crash.
12package vm
13
14import (
15 "context"
16 "errors"
17 "fmt"
18 "io/fs"
19 "net"
20 "net/netip"
21 "os"
22 "os/exec"
23 "path/filepath"
24 "strconv"
25 "sync"
26 "syscall"
27 "time"
28
29 "github.com/bnema/zerowrap"
30
31 "bnema.dev/ermite-runner/internal/cmdrun"
32 "bnema.dev/ermite-runner/internal/netpolicy"
33 "bnema.dev/ermite-runner/internal/sandbox"
34 "bnema.dev/ermite-runner/internal/sandbox/disk"
35 "bnema.dev/ermite-runner/internal/sandbox/netsetup"
36)
37
38// Options are the testable knobs of the driver.
39type Options struct {
40 Run cmdrun.Runner
41 // ProcDir is the /proc used to find orphan VMM processes.
42 ProcDir string
43 // CgroupRoot is the cgroup v2 mount, where the jailer creates one cgroup per
44 // VM (default /sys/fs/cgroup).
45 CgroupRoot string
46 // BootTimeout bounds boot + init + executor connection (default 90 s).
47 BootTimeout time.Duration
48 // Net overrides the network plumbing (tests).
49 Net netsetup.Plumbing
50 // FirecrackerArgs are extra arguments for Firecracker (tests use them to
51 // select the fake).
52 FirecrackerArgs []string
53 // FreeSpace overrides statfs for the free-space warning (tests).
54 FreeSpace func(string) (int64, error)
55 // kill overrides killAndWait (tests simulate a VMM that cannot be killed).
56 kill func(pid int, still func() bool, timeout time.Duration) bool
57}
58
59// vmmOverheadMiB is what the Firecracker process itself may use on top of the
60// guest memory (device emulation, the API server, the seccomp filter, stacks):
61// it is added to memory.max so that the guest, not the VMM, is what gets limited.
62const vmmOverheadMiB = 256
63
64// Driver implements sandbox.Driver for micro-VMs.
65type Driver struct {
66 cfg sandbox.Config
67 opt Options
68 fcPath string
69 jailer string
70 execDisk string
71 store store
72 net netsetup.Plumbing
73 lock *sandbox.HostLock
74
75 mu sync.Mutex
76 slots *sandbox.Slots
77}
78
79// New builds the driver: takes the host-wide lock, resolves binaries, builds the
80// exec disk and enables IPv4 forwarding (a host-wide sysctl that stays on: see
81// the README). Run Reap before New sandboxes.
82func New(ctx context.Context, cfg sandbox.Config, opt Options) (*Driver, error) {
83 if opt.Run == nil {
84 opt.Run = cmdrun.Exec{}
85 }
86 if opt.ProcDir == "" {
87 opt.ProcDir = "/proc"
88 }
89 if opt.CgroupRoot == "" {
90 opt.CgroupRoot = "/sys/fs/cgroup"
91 }
92 if opt.BootTimeout == 0 {
93 opt.BootTimeout = 90 * time.Second
94 }
95 if opt.kill == nil {
96 opt.kill = killAndWait
97 }
98 d := &Driver{cfg: cfg, opt: opt, slots: sandbox.NewSlots(cfg.Capacity), store: store{dir: filepath.Join(cfg.StateDir, "vms")}}
99
100 var err error
101 if d.fcPath, err = resolveBin(cfg.Firecracker, "firecracker"); err != nil {
102 return nil, err
103 }
104 if !cfg.JailerOff {
105 if d.jailer, err = resolveBin(cfg.Jailer, "jailer"); err != nil {
106 return nil, err
107 }
108 }
109 exe, err := cfg.ExecutablePath()
110 if err != nil {
111 return nil, err
112 }
113 if err := sandbox.CheckStatic(exe); err != nil {
114 return nil, err
115 }
116 if err := os.MkdirAll(cfg.StateDir, 0o755); err != nil {
117 return nil, fmt.Errorf("state directory: %w", err)
118 }
119 if d.lock, err = sandbox.AcquireHostLock(cfg.LockFile); err != nil {
120 return nil, err
121 }
122 fail := func(err error) (*Driver, error) { _ = d.Close(); return nil, err }
123
124 // A path over sun_path fails later as a 10 s "API not ready" (or a vsock
125 // that never connects): refuse at start, in both modes.
126 if err := checkSocketPaths(cfg.StateDir, filepath.Base(d.fcPath), !cfg.JailerOff); err != nil {
127 return fail(err)
128 }
129 if !cfg.JailerOff {
130 // Jailer 1.17 canonicalizes --chroot-base-dir and fails when it does not
131 // exist ("Failed to canonicalize path"); older releases created it.
132 if err := ensureJailBase(jailBase(cfg.StateDir)); err != nil {
133 return fail(err)
134 }
135 for _, p := range []string{cfg.KernelPath(), cfg.BaseImagePath()} {
136 if err := checkWorldReadable(p); err != nil {
137 return fail(err)
138 }
139 }
140 }
141 sandbox.WarnIfLowSpace(ctx, cfg.StateDir, cfg.Capacity, cfg.JobDiskMiB, opt.FreeSpace)
142 if err := disk.CheckMkfs(ctx, opt.Run); err != nil {
143 return fail(err)
144 }
145 if d.execDisk, err = ensureExecDisk(ctx, opt.Run, exe, cfg.StateDir); err != nil {
146 return fail(err)
147 }
148 d.net = opt.Net
149 if d.net == nil {
150 d.net = netsetup.New(opt.Run, cfg.DNSPort, "")
151 }
152 if err := d.net.EnableForwarding(); err != nil {
153 return fail(err)
154 }
155 return d, nil
156}
157
158func resolveBin(configured, name string) (string, error) {
159 p := configured
160 if p == "" {
161 var err error
162 if p, err = exec.LookPath(name); err != nil {
163 return "", fmt.Errorf("%s not found in PATH (set ERMITE_RUNNER_%s)", name, map[string]string{"firecracker": "FIRECRACKER", "jailer": "JAILER"}[name])
164 }
165 }
166 abs, err := filepath.Abs(p)
167 if err != nil {
168 return "", err
169 }
170 if _, err := os.Stat(abs); err != nil {
171 return "", fmt.Errorf("%s: %w", name, err)
172 }
173 return abs, nil
174}
175
176// Mode implements sandbox.Driver.
177func (d *Driver) Mode() sandbox.Mode { return sandbox.VM }
178
179// Isolation implements sandbox.Driver.
180func (d *Driver) Isolation() sandbox.Isolation { return sandbox.IsolationVM }
181
182// Close releases the host lock and the DNS listeners.
183func (d *Driver) Close() error {
184 if d.net != nil {
185 d.net.Close()
186 }
187 return d.lock.Release()
188}
189
190// New implements sandbox.Driver.
191func (d *Driver) New() (sandbox.Sandbox, error) {
192 id, err := sandbox.NewID()
193 if err != nil {
194 return nil, err
195 }
196 d.mu.Lock()
197 defer d.mu.Unlock()
198 slot, err := d.slots.Acquire()
199 if err != nil {
200 return nil, err
201 }
202 return &vmSandbox{d: d, slot: slot, id: id}, nil
203}
204
205// scope narrows the /proc scan to this driver's own jail base.
206func (d *Driver) scope() scope { return newScope(d.cfg.StateDir) }
207
208func (d *Driver) release(slot int) {
209 d.mu.Lock()
210 d.slots.Release(slot)
211 d.mu.Unlock()
212}
213
214func (d *Driver) markLeaked(slot int) {
215 d.mu.Lock()
216 d.slots.MarkLeaked(slot)
217 d.mu.Unlock()
218}
219
220// vmSandbox is one VM, single use.
221type vmSandbox struct {
222 d *Driver
223 slot int
224 id string
225 gate sandbox.BootGate
226
227 mu sync.Mutex
228 started bool
229 stopped bool
230 // leakErr is the error of a cleanup that left something behind; later Stop
231 // calls return it again.
232 leakErr error
233
234 rec record
235 lay layout
236 cmd *exec.Cmd
237 exited chan struct{}
238 console *ring
239 listener net.Listener
240 conn net.Conn
241 tap string
242 gateway netip.Addr
243 attached bool
244}
245
246// ID implements sandbox.Sandbox.
247func (s *vmSandbox) ID() string { return s.id }
248
249func (s *vmSandbox) uid() int { return s.d.cfg.UIDBase + s.slot }
250
251// Start implements sandbox.Sandbox.
252func (s *vmSandbox) Start(ctx context.Context, spec sandbox.JobSpec) (sandbox.Conn, error) {
253 // A Stop from another goroutine aborts the boot through the gate.
254 bootCtx, err := s.gate.Begin(ctx)
255 if err != nil {
256 return nil, fmt.Errorf("vm sandbox: %w", err)
257 }
258 defer s.gate.End()
259
260 s.mu.Lock()
261 defer s.mu.Unlock()
262 if s.started || s.stopped {
263 return nil, errors.New("vm sandbox: Start called twice or after Stop")
264 }
265 s.started = true
266 log := zerowrap.FromCtx(ctx)
267 d := s.d
268 spec = d.cfg.Resolve(spec)
269
270 jailed := !d.cfg.JailerOff
271 runUID := s.uid()
272 if !jailed {
273 runUID = os.Getuid()
274 }
275 s.lay = newLayout(d.cfg.StateDir, filepath.Base(d.fcPath), s.id, jailed, runUID)
276 s.rec = record{ID: s.id, Slot: s.slot, Job: spec.JobID, Dir: s.lay.dir}
277 // The record exists before the first resource, so any crash from here on is
278 // reapable.
279 if err := d.store.save(s.rec); err != nil {
280 s.releaseLocked()
281 return nil, fmt.Errorf("write state record: %w", err)
282 }
283
284 conn, err := s.boot(bootCtx, spec)
285 if err != nil {
286 console := ""
287 if s.console != nil {
288 console = s.console.String()
289 }
290 cerr := s.cleanupLocked(context.WithoutCancel(ctx))
291 if console != "" {
292 err = fmt.Errorf("%w\nVMM console tail:\n%s", err, console)
293 }
294 return nil, errors.Join(err, cerr)
295 }
296 log.Info().Str("vm", s.id).Int("slot", s.slot).Str("job", spec.JobID).Msg("micro-VM running")
297 return conn, nil
298}
299
300// boot runs the stages in order; each stage records what it created so that
301// cleanupLocked can undo exactly that.
302func (s *vmSandbox) boot(ctx context.Context, spec sandbox.JobSpec) (sandbox.Conn, error) {
303 netw, err := netpolicy.Subnet(s.d.cfg.Network, s.slot)
304 if err != nil {
305 return nil, err
306 }
307 if err := s.bootNetwork(ctx, netw); err != nil {
308 return nil, err
309 }
310 api, err := s.bootVMM(ctx, spec)
311 if err != nil {
312 return nil, err
313 }
314 files, err := s.bootFiles(ctx, spec)
315 if err != nil {
316 return nil, err
317 }
318 accepted, err := s.bootVsock()
319 if err != nil {
320 return nil, err
321 }
322 if err := api.configure(ctx, s.vmConfig(spec, netw, files)); err != nil {
323 return nil, err
324 }
325 return s.waitExecutor(ctx, accepted)
326}
327
328// bootNetwork creates the TAP (it must exist when Firecracker opens it) and
329// installs the firewall policy and the DNS listener.
330func (s *vmSandbox) bootNetwork(ctx context.Context, netw netpolicy.Net) error {
331 d := s.d
332 s.tap, s.gateway = tapName(s.slot), netw.Gateway
333 s.rec.Tap, s.rec.Gate = s.tap, s.gateway.String()
334 if err := d.store.save(s.rec); err != nil {
335 return err
336 }
337 if err := createTap(ctx, d.opt.Run, s.tap, s.lay.uid, s.gateway); err != nil {
338 return err
339 }
340 s.attached = true // from here Detach is idempotent and always safe
341 return d.net.Attach(ctx, netpolicy.Guest{Iface: s.tap, Gateway: netw.Gateway, Addr: netw.Guest})
342}
343
344// bootVMM starts the jailer (or Firecracker) and waits for its API.
345func (s *vmSandbox) bootVMM(ctx context.Context, spec sandbox.JobSpec) (*apiClient, error) {
346 apiSock := s.lay.host("api.sock")
347 if s.lay.jailed {
348 apiSock = s.lay.host("run/firecracker.socket")
349 } else if err := os.MkdirAll(s.lay.root, 0o700); err != nil {
350 return nil, err
351 }
352 if err := s.startVMM(spec, apiSock); err != nil {
353 return nil, err
354 }
355 api := newAPIClient(apiSock)
356 if err := s.waitAPI(ctx, api, apiSock); err != nil {
357 return nil, err
358 }
359 return api, nil
360}
361
362// vmFiles are the paths Firecracker is given.
363type vmFiles struct{ execDisk, baseDisk, kernel string }
364
365// bootFiles makes the shared files available and creates the scratch disk.
366func (s *vmSandbox) bootFiles(ctx context.Context, spec sandbox.JobSpec) (vmFiles, error) {
367 d := s.d
368 var f vmFiles
369 var err error
370 if f.execDisk, err = s.lay.share(d.execDisk, "exec.ext4"); err != nil {
371 return f, err
372 }
373 if f.baseDisk, err = s.lay.share(d.cfg.BaseImagePath(), "base.ext4"); err != nil {
374 return f, err
375 }
376 if f.kernel, err = s.lay.share(d.cfg.KernelPath(), "kernel"); err != nil {
377 return f, err
378 }
379 owner := -1
380 if s.lay.jailed {
381 owner = s.lay.uid
382 }
383 return f, disk.CreateScratch(ctx, d.opt.Run, s.lay.host("scratch.ext4"), spec.DiskMiB, owner)
384}
385
386// bootVsock listens on the host side of the vsock stream: Firecracker connects
387// to <uds_path>_<port> when the guest connects to the host CID.
388func (s *vmSandbox) bootVsock() (<-chan acceptResult, error) {
389 path := s.lay.host(udsName) + fmt.Sprintf("_%d", sandbox.VsockPort)
390 ln, err := net.Listen("unix", path)
391 if err != nil {
392 return nil, fmt.Errorf("listen vsock socket: %w", err)
393 }
394 s.listener = ln
395 if s.lay.jailed {
396 if err := os.Lchown(path, s.lay.uid, s.lay.uid); err != nil {
397 return nil, err
398 }
399 }
400 accepted := make(chan acceptResult, 1)
401 go func() { c, err := ln.Accept(); accepted <- acceptResult{c, err} }()
402 return accepted, nil
403}
404
405const udsName = "v.sock"
406
407// maxSockPath is the longest Unix socket path Linux accepts (108 with the NUL).
408const maxSockPath = 107
409
410// socketPaths lists the host paths of the unix sockets of a VM: the API socket
411// and the vsock listener (the guest's UDS path plus "_<port>"). With the jailer
412// they are deep in the chroot; without it they sit in <state>/vm/<id>.
413func socketPaths(stateDir, execName string, jailed bool) []string {
414 lay := newLayout(stateDir, execName, "er00000000", jailed, 0)
415 api := lay.host("api.sock")
416 if jailed {
417 api = lay.host("run/firecracker.socket")
418 }
419 return []string{api, lay.host(udsName) + fmt.Sprintf("_%d", sandbox.VsockPort)}
420}
421
422// checkSocketPaths refuses a state directory that makes a VM socket path longer
423// than a unix socket address.
424func checkSocketPaths(stateDir, execName string, jailed bool) error {
425 for _, p := range socketPaths(stateDir, execName, jailed) {
426 if len(p) > maxSockPath {
427 return fmt.Errorf("socket path %q is %d bytes, over the %d-byte limit of a Unix socket: use a shorter %s", p, len(p), maxSockPath, sandbox.EnvStateDir)
428 }
429 }
430 return nil
431}
432
433// ensureJailBase creates the jailer's chroot base and refuses one that others
434// could use to redirect the jailer: anything but a real directory (a symlink is
435// not followed), owned by neither root nor the runner's own user, or writable by
436// group or others. An existing directory is checked, never chmod-ed.
437func ensureJailBase(dir string) error {
438 if err := os.MkdirAll(dir, 0o700); err != nil {
439 return fmt.Errorf("jailer chroot base: %w", err)
440 }
441 fi, err := os.Lstat(dir)
442 if err != nil {
443 return fmt.Errorf("jailer chroot base: %w", err)
444 }
445 return checkJailBaseInfo(dir, fi, os.Geteuid())
446}
447
448func checkJailBaseInfo(dir string, fi fs.FileInfo, euid int) error {
449 if !fi.IsDir() {
450 return fmt.Errorf("jailer chroot base %s is not a directory (%v): remove it", dir, fi.Mode().Type())
451 }
452 if fi.Mode().Perm()&0o022 != 0 {
453 return fmt.Errorf("jailer chroot base %s is writable by group or others (mode %04o): chmod go-w it", dir, fi.Mode().Perm())
454 }
455 st, ok := fi.Sys().(*syscall.Stat_t)
456 if !ok {
457 return fmt.Errorf("jailer chroot base %s: cannot read its owner", dir)
458 }
459 // The jailer needs root, so the runner is root in production; accepting its own
460 // user as well keeps the driver testable without root.
461 if int(st.Uid) != 0 && int(st.Uid) != euid {
462 return fmt.Errorf("jailer chroot base %s is owned by uid %d, not root: chown it", dir, st.Uid)
463 }
464 return nil
465}
466
467func (s *vmSandbox) vmConfig(spec sandbox.JobSpec, netw netpolicy.Net, f vmFiles) vmConfig {
468 cfg := s.d.cfg
469 return vmConfig{
470 Machine: machineConfig{VcpuCount: spec.CPUs, MemSizeMiB: spec.MemoryMiB},
471 Boot: bootSource{KernelImagePath: f.kernel, BootArgs: bootArgs(netw.Guest, netw.Gateway, time.Now().Unix())},
472 Drives: []drive{
473 {DriveID: "rootfs", PathOnHost: f.execDisk, IsRootDevice: true, IsReadOnly: true},
474 {DriveID: "base", PathOnHost: f.baseDisk, IsReadOnly: true},
475 {DriveID: "scratch", PathOnHost: s.lay.fc("scratch.ext4"), RateLimiter: diskLimiter(cfg.DiskBandwidthMiB, cfg.DiskIOPS)},
476 },
477 Net: networkInterface{
478 IfaceID: "eth0", HostDevName: s.tap, GuestMAC: macFor(s.slot),
479 RxRateLimiter: netLimiter(cfg.NetMbit), TxRateLimiter: netLimiter(cfg.NetMbit),
480 },
481 Vsock: vsockDevice{GuestCID: 3, UDSPath: s.lay.fc(udsName)},
482 }
483}
484
485// waitExecutor waits for the guest's executor to connect. Only ctx (the caller's
486// context, or Stop through the boot gate) and the boot timeout end the wait: a
487// boot cannot be aborted any other way.
488func (s *vmSandbox) waitExecutor(ctx context.Context, accepted <-chan acceptResult) (sandbox.Conn, error) {
489 timer := time.NewTimer(s.d.opt.BootTimeout)
490 defer timer.Stop()
491 select {
492 case r := <-accepted:
493 if r.err != nil {
494 return nil, fmt.Errorf("accept executor connection: %w", r.err)
495 }
496 s.conn = r.c
497 return r.c, nil
498 case <-s.exited:
499 return nil, errors.New("firecracker exited before the executor connected")
500 case <-timer.C:
501 return nil, fmt.Errorf("executor did not connect within %s", s.d.opt.BootTimeout)
502 case <-ctx.Done():
503 return nil, context.Cause(ctx)
504 }
505}
506
507type acceptResult struct {
508 c net.Conn
509 err error
510}
511
512// jailerArgs builds the jailer command line (after the binary name).
513//
514// - cgroup v2 with memory.max (guest memory + VMM overhead) and pids.max (a
515// VMM needs a handful of threads per vCPU): a runaway or compromised VMM
516// cannot take the host's memory or fork-bomb it;
517// - a new PID namespace: the VMM cannot see or signal host processes;
518// - no-file: the descriptor budget of the VMM.
519func jailerArgs(jailer, fc, id string, uid int, chrootBase string, spec sandbox.JobSpec, extra []string) []string {
520 memBytes := (int64(spec.MemoryMiB) + vmmOverheadMiB) << 20
521 pids := 32 + 8*spec.CPUs
522 args := []string{
523 "--id", id, "--exec-file", fc,
524 "--uid", strconv.Itoa(uid), "--gid", strconv.Itoa(uid),
525 "--chroot-base-dir", chrootBase,
526 "--cgroup-version", "2",
527 "--cgroup", "memory.max=" + strconv.FormatInt(memBytes, 10),
528 "--cgroup", "pids.max=" + strconv.Itoa(pids),
529 "--new-pid-ns",
530 "--resource-limit", "no-file=4096",
531 "--",
532 }
533 return append(args, extra...)
534}
535
536func (s *vmSandbox) startVMM(spec sandbox.JobSpec, apiSock string) error {
537 d := s.d
538 var cmd *exec.Cmd
539 if s.lay.jailed {
540 args := jailerArgs(d.jailer, d.fcPath, s.id, s.uid(), jailBase(d.cfg.StateDir), spec, d.opt.FirecrackerArgs)
541 cmd = exec.Command(d.jailer, args...)
542 } else {
543 args := append([]string{"--id", s.id, "--api-sock", apiSock}, d.opt.FirecrackerArgs...)
544 cmd = exec.Command(d.fcPath, args...)
545 }
546 s.console = newRing(16 << 10)
547 cmd.Stdout, cmd.Stderr = s.console, s.console
548 cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
549 if err := cmd.Start(); err != nil {
550 return fmt.Errorf("start firecracker: %w", err)
551 }
552 s.cmd = cmd
553 s.exited = make(chan struct{})
554 jailed := s.lay.jailed
555 go func() {
556 _ = cmd.Wait()
557 // With --new-pid-ns the jailer may exit as soon as it has started the
558 // VMM in its namespace: the VMM is then gone only when its own pid is.
559 if jailed {
560 s.awaitVMMGone()
561 }
562 close(s.exited)
563 }()
564 s.rec.PID = cmd.Process.Pid
565 return s.d.store.save(s.rec)
566}
567
568// awaitVMMGone blocks until no process of this VM's id is left.
569func (s *vmSandbox) awaitVMMGone() {
570 for {
571 alive := false
572 for pid, id := range vmmProcs(s.d.opt.ProcDir, s.d.scope()) {
573 if id == s.id && pidAlive(pid) {
574 alive = true
575 }
576 }
577 if !alive {
578 return
579 }
580 time.Sleep(50 * time.Millisecond)
581 }
582}
583
584func (s *vmSandbox) waitAPI(ctx context.Context, api *apiClient, sock string) error {
585 deadline := time.Now().Add(10 * time.Second)
586 for {
587 if err := api.ready(ctx); err == nil {
588 return nil
589 }
590 select {
591 case <-s.exited:
592 return errors.New("firecracker exited before its API was ready")
593 case <-ctx.Done():
594 return context.Cause(ctx)
595 case <-time.After(25 * time.Millisecond):
596 }
597 if time.Now().After(deadline) {
598 return fmt.Errorf("firecracker API socket %s not ready after 10s", sock)
599 }
600 }
601}
602
603// Stop implements sandbox.Sandbox.
604func (s *vmSandbox) Stop(ctx context.Context) error {
605 // Abort a boot in progress first: Start holds s.mu for its whole duration.
606 s.gate.Abort()
607 s.mu.Lock()
608 defer s.mu.Unlock()
609 if s.stopped {
610 return s.leakErr
611 }
612 if !s.started {
613 s.releaseLocked()
614 return nil
615 }
616 return s.cleanupLocked(context.WithoutCancel(ctx))
617}
618
619func (s *vmSandbox) releaseLocked() {
620 s.stopped = true
621 s.d.release(s.slot)
622}
623
624// vmmPIDs returns every host process of this VM: the launched command and
625// whatever the /proc scan finds with this id.
626func (s *vmSandbox) vmmPIDs() []int {
627 set := map[int]bool{}
628 if s.cmd != nil && s.cmd.Process != nil {
629 set[s.cmd.Process.Pid] = true
630 }
631 for pid, id := range vmmProcs(s.d.opt.ProcDir, s.d.scope()) {
632 if id == s.id {
633 set[pid] = true
634 }
635 }
636 var out []int
637 for pid := range set {
638 out = append(out, pid)
639 }
640 return out
641}
642
643// stillMine is the identity check handed to the kill: pid still carries this
644// VM's id and passes the scope (command line and root).
645func (s *vmSandbox) stillMine(pid int) func() bool {
646 return func() bool { return cmdlineID(s.d.opt.ProcDir, pid, s.d.scope()) == s.id }
647}
648
649// cleanupLocked removes everything the VM created, in dependency order. Steps
650// that need the VMM to be dead (network device, firewall rules, files, cgroup)
651// are skipped when it could not be killed: removing the TAP or the policy under
652// a VM that may still run would break the isolation it relies on, so the
653// sandbox is reported as leaked (sandbox.ErrLeaked), its slot stays taken, and
654// Reap finishes the job.
655func (s *vmSandbox) cleanupLocked(ctx context.Context) error {
656 log := zerowrap.FromCtx(ctx)
657 d := s.d
658 var errs []error
659 if s.conn != nil {
660 _ = s.conn.Close()
661 s.conn = nil
662 }
663 if s.listener != nil {
664 _ = s.listener.Close()
665 s.listener = nil
666 }
667 dead := true
668 // No kill(-pgid) of the launched command: once it has been waited for, its
669 // pid (and so its group id) may belong to anyone. Every process of this VM is
670 // found by the scan (id, markers and scope) and killed through a pidfd.
671 for _, pid := range s.vmmPIDs() {
672 if !d.opt.kill(pid, s.stillMine(pid), 5*time.Second) {
673 dead = false
674 errs = append(errs, fmt.Errorf("vmm process %d did not exit", pid))
675 }
676 }
677 if dead && s.exited != nil {
678 select {
679 case <-s.exited:
680 case <-time.After(5 * time.Second):
681 dead = false
682 errs = append(errs, errors.New("vmm exit was not observed"))
683 }
684 }
685 if dead {
686 if s.attached {
687 if err := d.net.Detach(ctx, s.tap, s.gateway); err != nil {
688 errs = append(errs, err)
689 }
690 if err := deleteTap(ctx, d.opt.Run, s.tap); err != nil {
691 errs = append(errs, err)
692 } else {
693 s.attached = false
694 }
695 }
696 if s.lay.dir != "" {
697 if err := removeAll(s.lay.dir); err != nil {
698 errs = append(errs, err)
699 }
700 }
701 if s.lay.jailed {
702 if err := removeCgroup(d.opt.CgroupRoot, filepath.Base(d.fcPath), s.id); err != nil {
703 errs = append(errs, err)
704 }
705 }
706 if len(errs) == 0 {
707 if err := d.store.remove(s.id); err != nil {
708 errs = append(errs, err)
709 }
710 }
711 }
712 if len(errs) == 0 {
713 s.releaseLocked()
714 return nil
715 }
716 s.stopped = true
717 d.markLeaked(s.slot)
718 s.leakErr = fmt.Errorf("%w: vm %s: %w", sandbox.ErrLeaked, s.id, errors.Join(errs...))
719 log.Error().Err(s.leakErr).Str("vm", s.id).Msg("micro-VM cleanup incomplete; call Reap to finish it")
720 return s.leakErr
721}
722
723// removeAll is os.RemoveAll that also reports a directory that is still there.
724func removeAll(dir string) error {
725 if err := os.RemoveAll(dir); err != nil {
726 return err
727 }
728 if _, err := os.Lstat(dir); err == nil {
729 return fmt.Errorf("%s still exists after removal", dir)
730 }
731 return nil
732}
733
734// removeCgroup removes the cgroup the jailer created for a VM
735// (<root>/<parent>/<id>, parent = the exec file name). The jailer never removes
736// it. A missing cgroup is fine; a non-empty one fails (rmdir semantics).
737func removeCgroup(root, parent, id string) error {
738 err := os.Remove(filepath.Join(root, parent, id))
739 if err == nil || errors.Is(err, os.ErrNotExist) {
740 return nil
741 }
742 return fmt.Errorf("remove cgroup %s: %w", filepath.Join(root, parent, id), err)
743}