1// SPDX-License-Identifier: MIT
2
3// Package vm is the micro-VM sandbox: one Firecracker VM per job.
4//
5// Per VM, on the host: a jailer chroot (or, without jailer, a plain directory)
6// holding the per-job sparse scratch disk and the vsock socket; a TAP device
7// with a /30; rules in the dedicated nftables table; the DNS forwarder bound on
8// the TAP's host address. In the guest: three virtio disks (read-only runner
9// executable as the boot root, read-only shared base image, per-job scratch)
10// and a vsock device. Everything is traced in a state-directory record before it
11// is created, so that Reap can remove it after a crash.
12package vm
13
14import (
15 "context"
16 "errors"
17 "fmt"
18 "io/fs"
19 "net"
20 "net/netip"
21 "os"
22 "os/exec"
23 "path/filepath"
24 "strconv"
25 "sync"
26 "syscall"
27 "time"
28
29 "github.com/bnema/zerowrap"
30
31 "bnema.dev/ermite-runner/internal/cmdrun"
32 "bnema.dev/ermite-runner/internal/netpolicy"
33 "bnema.dev/ermite-runner/internal/sandbox"
34 "bnema.dev/ermite-runner/internal/sandbox/disk"
35 "bnema.dev/ermite-runner/internal/sandbox/netsetup"
36)
37
38// Options are the testable knobs of the driver.
39type Options struct {
40 Run cmdrun.Runner
41 // ProcDir is the /proc used to find orphan VMM processes.
42 ProcDir string
43 // CgroupRoot is the cgroup v2 mount, where the jailer creates one cgroup per
44 // VM (default /sys/fs/cgroup).
45 CgroupRoot string
46 // BootTimeout bounds boot + init + executor connection (default 90 s).
47 BootTimeout time.Duration
48 // Net overrides the network plumbing (tests).
49 Net netsetup.Plumbing
50 // FirecrackerArgs are extra arguments for Firecracker (tests use them to
51 // select the fake).
52 FirecrackerArgs []string
53 // FreeSpace overrides statfs for the free-space warning (tests).
54 FreeSpace func(string) (int64, error)
55 // kill overrides killAndWait (tests simulate a VMM that cannot be killed).
56 kill func(pid int, still func() bool, timeout time.Duration) bool
57}
58
59// vmmOverheadMiB is what the Firecracker process itself may use on top of the
60// guest memory (device emulation, the API server, the seccomp filter, stacks):
61// it is added to memory.max so that the guest, not the VMM, is what gets limited.
62const vmmOverheadMiB = 256
63
64// Driver implements sandbox.Driver for micro-VMs.
65type Driver struct {
66 cfg sandbox.Config
67 opt Options
68 fcPath string
69 jailer string
70 execDisk string
71 // extMu serialises the building of the externals disk with the linking of it
72 // into a sandbox (see shareExternals).
73 extMu sync.Mutex
74 store store
75 net netsetup.Plumbing
76 lock *sandbox.HostLock
77
78 mu sync.Mutex
79 slots *sandbox.Slots
80}
81
82// New builds the driver: takes the host-wide lock, resolves binaries, builds the
83// exec disk and enables IPv4 forwarding (a host-wide sysctl that stays on: see
84// the README). Run Reap before New sandboxes.
85func New(ctx context.Context, cfg sandbox.Config, opt Options) (*Driver, error) {
86 if opt.Run == nil {
87 opt.Run = cmdrun.Exec{}
88 }
89 if opt.ProcDir == "" {
90 opt.ProcDir = "/proc"
91 }
92 if opt.CgroupRoot == "" {
93 opt.CgroupRoot = "/sys/fs/cgroup"
94 }
95 if opt.BootTimeout == 0 {
96 opt.BootTimeout = 90 * time.Second
97 }
98 if opt.kill == nil {
99 opt.kill = killAndWait
100 }
101 d := &Driver{cfg: cfg, opt: opt, slots: sandbox.NewSlots(cfg.Capacity), store: store{dir: filepath.Join(cfg.StateDir, "vms")}}
102
103 var err error
104 if d.fcPath, err = resolveBin(cfg.Firecracker, "firecracker"); err != nil {
105 return nil, err
106 }
107 if !cfg.JailerOff {
108 if d.jailer, err = resolveBin(cfg.Jailer, "jailer"); err != nil {
109 return nil, err
110 }
111 }
112 exe, err := cfg.ExecutablePath()
113 if err != nil {
114 return nil, err
115 }
116 if err := sandbox.CheckStatic(exe); err != nil {
117 return nil, err
118 }
119 if err := os.MkdirAll(cfg.StateDir, 0o755); err != nil {
120 return nil, fmt.Errorf("state directory: %w", err)
121 }
122 if d.lock, err = sandbox.AcquireHostLock(cfg.LockFile); err != nil {
123 return nil, err
124 }
125 fail := func(err error) (*Driver, error) { _ = d.Close(); return nil, err }
126
127 // A path over sun_path fails later as a 10 s "API not ready" (or a vsock
128 // that never connects): refuse at start, in both modes.
129 if err := checkSocketPaths(cfg.StateDir, filepath.Base(d.fcPath), !cfg.JailerOff); err != nil {
130 return fail(err)
131 }
132 if !cfg.JailerOff {
133 // Jailer 1.17 canonicalizes --chroot-base-dir and fails when it does not
134 // exist ("Failed to canonicalize path"); older releases created it.
135 if err := ensureJailBase(jailBase(cfg.StateDir)); err != nil {
136 return fail(err)
137 }
138 for _, p := range []string{cfg.KernelPath(), cfg.BaseImagePath()} {
139 if err := checkWorldReadable(p); err != nil {
140 return fail(err)
141 }
142 }
143 }
144 sandbox.WarnIfLowSpace(ctx, cfg.StateDir, cfg.Capacity, cfg.JobDiskMiB, opt.FreeSpace)
145 if err := disk.CheckMkfs(ctx, opt.Run); err != nil {
146 return fail(err)
147 }
148 if d.execDisk, err = ensureExecDisk(ctx, opt.Run, exe, cfg.StateDir); err != nil {
149 return fail(err)
150 }
151 d.net = opt.Net
152 if d.net == nil {
153 n := netsetup.New(opt.Run, cfg.DNSPort, "")
154 n.DNS.Override = cfg.GuestForge.DNSName()
155 d.net = n
156 }
157 if cfg.GuestForge != nil {
158 l := zerowrap.FromCtx(ctx)
159 l.Warn().Msg(cfg.GuestForge.Warning())
160 }
161 if err := d.net.EnableForwarding(); err != nil {
162 return fail(err)
163 }
164 return d, nil
165}
166
167func resolveBin(configured, name string) (string, error) {
168 p := configured
169 if p == "" {
170 var err error
171 if p, err = exec.LookPath(name); err != nil {
172 return "", fmt.Errorf("%s not found in PATH (set ERMITE_RUNNER_%s)", name, map[string]string{"firecracker": "FIRECRACKER", "jailer": "JAILER"}[name])
173 }
174 }
175 abs, err := filepath.Abs(p)
176 if err != nil {
177 return "", err
178 }
179 if _, err := os.Stat(abs); err != nil {
180 return "", fmt.Errorf("%s: %w", name, err)
181 }
182 return abs, nil
183}
184
185// Mode implements sandbox.Driver.
186func (d *Driver) Mode() sandbox.Mode { return sandbox.VM }
187
188// Isolation implements sandbox.Driver.
189func (d *Driver) Isolation() sandbox.Isolation { return sandbox.IsolationVM }
190
191// Close releases the host lock and the DNS listeners.
192func (d *Driver) Close() error {
193 if d.net != nil {
194 d.net.Close()
195 }
196 return d.lock.Release()
197}
198
199// New implements sandbox.Driver.
200func (d *Driver) New() (sandbox.Sandbox, error) {
201 id, err := sandbox.NewID()
202 if err != nil {
203 return nil, err
204 }
205 d.mu.Lock()
206 defer d.mu.Unlock()
207 slot, err := d.slots.Acquire()
208 if err != nil {
209 return nil, err
210 }
211 return &vmSandbox{d: d, slot: slot, id: id}, nil
212}
213
214// scope narrows the /proc scan to this driver's own jail base.
215func (d *Driver) scope() scope { return newScope(d.cfg.StateDir) }
216
217func (d *Driver) release(slot int) {
218 d.mu.Lock()
219 d.slots.Release(slot)
220 d.mu.Unlock()
221}
222
223func (d *Driver) markLeaked(slot int) {
224 d.mu.Lock()
225 d.slots.MarkLeaked(slot)
226 d.mu.Unlock()
227}
228
229// vmSandbox is one VM, single use.
230type vmSandbox struct {
231 d *Driver
232 slot int
233 id string
234 gate sandbox.BootGate
235
236 mu sync.Mutex
237 started bool
238 stopped bool
239 // leakErr is the error of a cleanup that left something behind; later Stop
240 // calls return it again.
241 leakErr error
242
243 rec record
244 lay layout
245 cmd *exec.Cmd
246 exited chan struct{}
247 console *ring
248 listener net.Listener
249 conn net.Conn
250 tap string
251 gateway netip.Addr
252 attached bool
253}
254
255// ID implements sandbox.Sandbox.
256func (s *vmSandbox) ID() string { return s.id }
257
258func (s *vmSandbox) uid() int { return s.d.cfg.UIDBase + s.slot }
259
260// Start implements sandbox.Sandbox.
261func (s *vmSandbox) Start(ctx context.Context, spec sandbox.JobSpec) (sandbox.Conn, error) {
262 // A Stop from another goroutine aborts the boot through the gate.
263 bootCtx, err := s.gate.Begin(ctx)
264 if err != nil {
265 return nil, fmt.Errorf("vm sandbox: %w", err)
266 }
267 defer s.gate.End()
268
269 s.mu.Lock()
270 defer s.mu.Unlock()
271 if s.started || s.stopped {
272 return nil, errors.New("vm sandbox: Start called twice or after Stop")
273 }
274 s.started = true
275 log := zerowrap.FromCtx(ctx)
276 d := s.d
277 spec = d.cfg.Resolve(spec)
278 if d.cfg.GuestForge != nil {
279 // At every job start, as for privileged containers.
280 log.Warn().Str("job", spec.JobID).Str("vm", s.id).Msg(d.cfg.GuestForge.Warning())
281 }
282
283 jailed := !d.cfg.JailerOff
284 runUID := s.uid()
285 if !jailed {
286 runUID = os.Getuid()
287 }
288 s.lay = newLayout(d.cfg.StateDir, filepath.Base(d.fcPath), s.id, jailed, runUID)
289 s.rec = record{ID: s.id, Slot: s.slot, Job: spec.JobID, Dir: s.lay.dir}
290 // The record exists before the first resource, so any crash from here on is
291 // reapable.
292 if err := d.store.save(s.rec); err != nil {
293 s.releaseLocked()
294 return nil, fmt.Errorf("write state record: %w", err)
295 }
296
297 conn, err := s.boot(bootCtx, spec)
298 if err != nil {
299 console := ""
300 if s.console != nil {
301 console = s.console.String()
302 }
303 cerr := s.cleanupLocked(context.WithoutCancel(ctx))
304 if console != "" {
305 err = fmt.Errorf("%w\nVMM console tail:\n%s", err, console)
306 }
307 return nil, errors.Join(err, cerr)
308 }
309 log.Info().Str("vm", s.id).Int("slot", s.slot).Str("job", spec.JobID).Msg("micro-VM running")
310 return conn, nil
311}
312
313// boot runs the stages in order; each stage records what it created so that
314// cleanupLocked can undo exactly that.
315func (s *vmSandbox) boot(ctx context.Context, spec sandbox.JobSpec) (sandbox.Conn, error) {
316 netw, err := netpolicy.Subnet(s.d.cfg.Network, s.slot)
317 if err != nil {
318 return nil, err
319 }
320 if err := s.bootNetwork(ctx, netw); err != nil {
321 return nil, err
322 }
323 api, err := s.bootVMM(ctx, spec)
324 if err != nil {
325 return nil, err
326 }
327 files, err := s.bootFiles(ctx, spec)
328 if err != nil {
329 return nil, err
330 }
331 accepted, err := s.bootVsock()
332 if err != nil {
333 return nil, err
334 }
335 if err := api.configure(ctx, s.vmConfig(spec, netw, files)); err != nil {
336 return nil, err
337 }
338 return s.waitExecutor(ctx, accepted)
339}
340
341// bootNetwork creates the TAP (it must exist when Firecracker opens it) and
342// installs the firewall policy and the DNS listener.
343func (s *vmSandbox) bootNetwork(ctx context.Context, netw netpolicy.Net) error {
344 d := s.d
345 s.tap, s.gateway = tapName(s.slot), netw.Gateway
346 s.rec.Tap, s.rec.Gate = s.tap, s.gateway.String()
347 if err := d.store.save(s.rec); err != nil {
348 return err
349 }
350 if err := createTap(ctx, d.opt.Run, s.tap, s.lay.uid, s.gateway); err != nil {
351 return err
352 }
353 s.attached = true // from here Detach is idempotent and always safe
354 return d.net.Attach(ctx, netpolicy.Guest{Iface: s.tap, Gateway: netw.Gateway, Addr: netw.Guest, ForgePort: d.cfg.GuestForge.TCPPort()})
355}
356
357// bootVMM starts the jailer (or Firecracker) and waits for its API.
358func (s *vmSandbox) bootVMM(ctx context.Context, spec sandbox.JobSpec) (*apiClient, error) {
359 apiSock := s.lay.host("api.sock")
360 if s.lay.jailed {
361 apiSock = s.lay.host("run/firecracker.socket")
362 } else if err := os.MkdirAll(s.lay.root, 0o700); err != nil {
363 return nil, err
364 }
365 if err := s.startVMM(spec, apiSock); err != nil {
366 return nil, err
367 }
368 api := newAPIClient(apiSock)
369 if err := s.waitAPI(ctx, api, apiSock); err != nil {
370 return nil, err
371 }
372 return api, nil
373}
374
375// vmFiles are the paths Firecracker is given.
376type vmFiles struct{ execDisk, baseDisk, kernel, externals string }
377
378// bootFiles makes the shared files available and creates the scratch disk.
379func (s *vmSandbox) bootFiles(ctx context.Context, spec sandbox.JobSpec) (vmFiles, error) {
380 d := s.d
381 var f vmFiles
382 var err error
383 if f.execDisk, err = s.lay.share(d.execDisk, "exec.ext4"); err != nil {
384 return f, err
385 }
386 if f.baseDisk, err = s.lay.share(d.cfg.BaseImagePath(), "base.ext4"); err != nil {
387 return f, err
388 }
389 if f.kernel, err = s.lay.share(d.cfg.KernelPath(), "kernel"); err != nil {
390 return f, err
391 }
392 f.externals = s.shareExternals(ctx)
393 owner := -1
394 if s.lay.jailed {
395 owner = s.lay.uid
396 }
397 return f, disk.CreateScratch(ctx, d.opt.Run, s.lay.host("scratch.ext4"), spec.DiskMiB, owner)
398}
399
400// shareExternals makes the read-only disk of the host's external tools available
401// to Firecracker; "" means the sandbox has none. A disk that cannot be built
402// does not stop the job: it starts without the tools and the failure is logged.
403func (s *vmSandbox) shareExternals(ctx context.Context) string {
404 d := s.d
405 set := d.cfg.ExternalsOf()
406 if len(set) == 0 {
407 return ""
408 }
409 log := zerowrap.FromCtx(ctx)
410 d.extMu.Lock()
411 defer d.extMu.Unlock()
412 path, err := ensureExternalsDisk(ctx, d.opt.Run, set, d.cfg.StateDir)
413 if err == nil && path != "" {
414 path, err = s.lay.share(path, "externals.ext4")
415 }
416 if err != nil {
417 log.Error().Err(err).Str("vm", s.id).Msg("external tools (Node.js) not attached to this job")
418 return ""
419 }
420 return path
421}
422
423// bootVsock listens on the host side of the vsock stream: Firecracker connects
424// to <uds_path>_<port> when the guest connects to the host CID.
425func (s *vmSandbox) bootVsock() (<-chan acceptResult, error) {
426 path := s.lay.host(udsName) + fmt.Sprintf("_%d", sandbox.VsockPort)
427 ln, err := net.Listen("unix", path)
428 if err != nil {
429 return nil, fmt.Errorf("listen vsock socket: %w", err)
430 }
431 s.listener = ln
432 if s.lay.jailed {
433 if err := os.Lchown(path, s.lay.uid, s.lay.uid); err != nil {
434 return nil, err
435 }
436 }
437 accepted := make(chan acceptResult, 1)
438 go func() { c, err := ln.Accept(); accepted <- acceptResult{c, err} }()
439 return accepted, nil
440}
441
442const udsName = "v.sock"
443
444// maxSockPath is the longest Unix socket path Linux accepts (108 with the NUL).
445const maxSockPath = 107
446
447// socketPaths lists the host paths of the unix sockets of a VM: the API socket
448// and the vsock listener (the guest's UDS path plus "_<port>"). With the jailer
449// they are deep in the chroot; without it they sit in <state>/vm/<id>.
450func socketPaths(stateDir, execName string, jailed bool) []string {
451 lay := newLayout(stateDir, execName, "er00000000", jailed, 0)
452 api := lay.host("api.sock")
453 if jailed {
454 api = lay.host("run/firecracker.socket")
455 }
456 return []string{api, lay.host(udsName) + fmt.Sprintf("_%d", sandbox.VsockPort)}
457}
458
459// checkSocketPaths refuses a state directory that makes a VM socket path longer
460// than a unix socket address.
461func checkSocketPaths(stateDir, execName string, jailed bool) error {
462 for _, p := range socketPaths(stateDir, execName, jailed) {
463 if len(p) > maxSockPath {
464 return fmt.Errorf("socket path %q is %d bytes, over the %d-byte limit of a Unix socket: use a shorter %s", p, len(p), maxSockPath, sandbox.EnvStateDir)
465 }
466 }
467 return nil
468}
469
470// ensureJailBase creates the jailer's chroot base and refuses one that others
471// could use to redirect the jailer: anything but a real directory (a symlink is
472// not followed), owned by neither root nor the runner's own user, or writable by
473// group or others. An existing directory is checked, never chmod-ed.
474func ensureJailBase(dir string) error {
475 if err := os.MkdirAll(dir, 0o700); err != nil {
476 return fmt.Errorf("jailer chroot base: %w", err)
477 }
478 fi, err := os.Lstat(dir)
479 if err != nil {
480 return fmt.Errorf("jailer chroot base: %w", err)
481 }
482 return checkJailBaseInfo(dir, fi, os.Geteuid())
483}
484
485func checkJailBaseInfo(dir string, fi fs.FileInfo, euid int) error {
486 if !fi.IsDir() {
487 return fmt.Errorf("jailer chroot base %s is not a directory (%v): remove it", dir, fi.Mode().Type())
488 }
489 if fi.Mode().Perm()&0o022 != 0 {
490 return fmt.Errorf("jailer chroot base %s is writable by group or others (mode %04o): chmod go-w it", dir, fi.Mode().Perm())
491 }
492 st, ok := fi.Sys().(*syscall.Stat_t)
493 if !ok {
494 return fmt.Errorf("jailer chroot base %s: cannot read its owner", dir)
495 }
496 // The jailer needs root, so the runner is root in production; accepting its own
497 // user as well keeps the driver testable without root.
498 if int(st.Uid) != 0 && int(st.Uid) != euid {
499 return fmt.Errorf("jailer chroot base %s is owned by uid %d, not root: chown it", dir, st.Uid)
500 }
501 return nil
502}
503
504func (s *vmSandbox) vmConfig(spec sandbox.JobSpec, netw netpolicy.Net, f vmFiles) vmConfig {
505 cfg := s.d.cfg
506 // The order decides the guest device names (vda, vdb, ...): new drives go last.
507 drives := []drive{
508 {DriveID: "rootfs", PathOnHost: f.execDisk, IsRootDevice: true, IsReadOnly: true},
509 {DriveID: "base", PathOnHost: f.baseDisk, IsReadOnly: true},
510 {DriveID: "scratch", PathOnHost: s.lay.fc("scratch.ext4"), RateLimiter: diskLimiter(cfg.DiskBandwidthMiB, cfg.DiskIOPS)},
511 }
512 if f.externals != "" {
513 drives = append(drives, drive{DriveID: "externals", PathOnHost: f.externals, IsReadOnly: true})
514 }
515 return vmConfig{
516 Machine: machineConfig{VcpuCount: spec.CPUs, MemSizeMiB: spec.MemoryMiB},
517 Boot: bootSource{KernelImagePath: f.kernel, BootArgs: bootArgs(netw.Guest, netw.Gateway, time.Now().Unix(), f.externals != "")},
518 Drives: drives,
519 Net: networkInterface{
520 IfaceID: "eth0", HostDevName: s.tap, GuestMAC: macFor(s.slot),
521 RxRateLimiter: netLimiter(cfg.NetMbit), TxRateLimiter: netLimiter(cfg.NetMbit),
522 },
523 Vsock: vsockDevice{GuestCID: 3, UDSPath: s.lay.fc(udsName)},
524 }
525}
526
527// waitExecutor waits for the guest's executor to connect. Only ctx (the caller's
528// context, or Stop through the boot gate) and the boot timeout end the wait: a
529// boot cannot be aborted any other way.
530func (s *vmSandbox) waitExecutor(ctx context.Context, accepted <-chan acceptResult) (sandbox.Conn, error) {
531 timer := time.NewTimer(s.d.opt.BootTimeout)
532 defer timer.Stop()
533 select {
534 case r := <-accepted:
535 if r.err != nil {
536 return nil, fmt.Errorf("accept executor connection: %w", r.err)
537 }
538 s.conn = r.c
539 return r.c, nil
540 case <-s.exited:
541 return nil, errors.New("firecracker exited before the executor connected")
542 case <-timer.C:
543 return nil, fmt.Errorf("executor did not connect within %s", s.d.opt.BootTimeout)
544 case <-ctx.Done():
545 return nil, context.Cause(ctx)
546 }
547}
548
549type acceptResult struct {
550 c net.Conn
551 err error
552}
553
554// jailerArgs builds the jailer command line (after the binary name).
555//
556// - cgroup v2 with memory.max (guest memory + VMM overhead) and pids.max (a
557// VMM needs a handful of threads per vCPU): a runaway or compromised VMM
558// cannot take the host's memory or fork-bomb it;
559// - a new PID namespace: the VMM cannot see or signal host processes;
560// - no-file: the descriptor budget of the VMM.
561func jailerArgs(jailer, fc, id string, uid int, chrootBase string, spec sandbox.JobSpec, extra []string) []string {
562 memBytes := (int64(spec.MemoryMiB) + vmmOverheadMiB) << 20
563 pids := 32 + 8*spec.CPUs
564 args := []string{
565 "--id", id, "--exec-file", fc,
566 "--uid", strconv.Itoa(uid), "--gid", strconv.Itoa(uid),
567 "--chroot-base-dir", chrootBase,
568 "--cgroup-version", "2",
569 "--cgroup", "memory.max=" + strconv.FormatInt(memBytes, 10),
570 "--cgroup", "pids.max=" + strconv.Itoa(pids),
571 "--new-pid-ns",
572 "--resource-limit", "no-file=4096",
573 "--",
574 }
575 return append(args, extra...)
576}
577
578func (s *vmSandbox) startVMM(spec sandbox.JobSpec, apiSock string) error {
579 d := s.d
580 var cmd *exec.Cmd
581 if s.lay.jailed {
582 args := jailerArgs(d.jailer, d.fcPath, s.id, s.uid(), jailBase(d.cfg.StateDir), spec, d.opt.FirecrackerArgs)
583 cmd = exec.Command(d.jailer, args...)
584 } else {
585 args := append([]string{"--id", s.id, "--api-sock", apiSock}, d.opt.FirecrackerArgs...)
586 cmd = exec.Command(d.fcPath, args...)
587 }
588 s.console = newRing(16 << 10)
589 cmd.Stdout, cmd.Stderr = s.console, s.console
590 cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
591 if err := cmd.Start(); err != nil {
592 return fmt.Errorf("start firecracker: %w", err)
593 }
594 s.cmd = cmd
595 s.exited = make(chan struct{})
596 jailed := s.lay.jailed
597 go func() {
598 _ = cmd.Wait()
599 // With --new-pid-ns the jailer may exit as soon as it has started the
600 // VMM in its namespace: the VMM is then gone only when its own pid is.
601 if jailed {
602 s.awaitVMMGone()
603 }
604 close(s.exited)
605 }()
606 s.rec.PID = cmd.Process.Pid
607 return s.d.store.save(s.rec)
608}
609
610// awaitVMMGone blocks until no process of this VM's id is left.
611func (s *vmSandbox) awaitVMMGone() {
612 for {
613 alive := false
614 for pid, id := range vmmProcs(s.d.opt.ProcDir, s.d.scope()) {
615 if id == s.id && pidAlive(pid) {
616 alive = true
617 }
618 }
619 if !alive {
620 return
621 }
622 time.Sleep(50 * time.Millisecond)
623 }
624}
625
626func (s *vmSandbox) waitAPI(ctx context.Context, api *apiClient, sock string) error {
627 deadline := time.Now().Add(10 * time.Second)
628 for {
629 if err := api.ready(ctx); err == nil {
630 return nil
631 }
632 select {
633 case <-s.exited:
634 return errors.New("firecracker exited before its API was ready")
635 case <-ctx.Done():
636 return context.Cause(ctx)
637 case <-time.After(25 * time.Millisecond):
638 }
639 if time.Now().After(deadline) {
640 return fmt.Errorf("firecracker API socket %s not ready after 10s", sock)
641 }
642 }
643}
644
645// Stop implements sandbox.Sandbox.
646func (s *vmSandbox) Stop(ctx context.Context) error {
647 // Abort a boot in progress first: Start holds s.mu for its whole duration.
648 s.gate.Abort()
649 s.mu.Lock()
650 defer s.mu.Unlock()
651 if s.stopped {
652 return s.leakErr
653 }
654 if !s.started {
655 s.releaseLocked()
656 return nil
657 }
658 return s.cleanupLocked(context.WithoutCancel(ctx))
659}
660
661func (s *vmSandbox) releaseLocked() {
662 s.stopped = true
663 s.d.release(s.slot)
664}
665
666// vmmPIDs returns every host process of this VM: the launched command and
667// whatever the /proc scan finds with this id.
668func (s *vmSandbox) vmmPIDs() []int {
669 set := map[int]bool{}
670 if s.cmd != nil && s.cmd.Process != nil {
671 set[s.cmd.Process.Pid] = true
672 }
673 for pid, id := range vmmProcs(s.d.opt.ProcDir, s.d.scope()) {
674 if id == s.id {
675 set[pid] = true
676 }
677 }
678 var out []int
679 for pid := range set {
680 out = append(out, pid)
681 }
682 return out
683}
684
685// stillMine is the identity check handed to the kill: pid still carries this
686// VM's id and passes the scope (command line and root).
687func (s *vmSandbox) stillMine(pid int) func() bool {
688 return func() bool { return cmdlineID(s.d.opt.ProcDir, pid, s.d.scope()) == s.id }
689}
690
691// cleanupLocked removes everything the VM created, in dependency order. Steps
692// that need the VMM to be dead (network device, firewall rules, files, cgroup)
693// are skipped when it could not be killed: removing the TAP or the policy under
694// a VM that may still run would break the isolation it relies on, so the
695// sandbox is reported as leaked (sandbox.ErrLeaked), its slot stays taken, and
696// Reap finishes the job.
697func (s *vmSandbox) cleanupLocked(ctx context.Context) error {
698 log := zerowrap.FromCtx(ctx)
699 d := s.d
700 var errs []error
701 if s.conn != nil {
702 _ = s.conn.Close()
703 s.conn = nil
704 }
705 if s.listener != nil {
706 _ = s.listener.Close()
707 s.listener = nil
708 }
709 dead := true
710 // No kill(-pgid) of the launched command: once it has been waited for, its
711 // pid (and so its group id) may belong to anyone. Every process of this VM is
712 // found by the scan (id, markers and scope) and killed through a pidfd.
713 for _, pid := range s.vmmPIDs() {
714 if !d.opt.kill(pid, s.stillMine(pid), 5*time.Second) {
715 dead = false
716 errs = append(errs, fmt.Errorf("vmm process %d did not exit", pid))
717 }
718 }
719 if dead && s.exited != nil {
720 select {
721 case <-s.exited:
722 case <-time.After(5 * time.Second):
723 dead = false
724 errs = append(errs, errors.New("vmm exit was not observed"))
725 }
726 }
727 if dead {
728 if s.attached {
729 if err := d.net.Detach(ctx, s.tap, s.gateway); err != nil {
730 errs = append(errs, err)
731 }
732 if err := deleteTap(ctx, d.opt.Run, s.tap); err != nil {
733 errs = append(errs, err)
734 } else {
735 s.attached = false
736 }
737 }
738 if s.lay.dir != "" {
739 if err := removeAll(s.lay.dir); err != nil {
740 errs = append(errs, err)
741 }
742 }
743 if s.lay.jailed {
744 if err := removeCgroup(d.opt.CgroupRoot, filepath.Base(d.fcPath), s.id); err != nil {
745 errs = append(errs, err)
746 }
747 }
748 if len(errs) == 0 {
749 if err := d.store.remove(s.id); err != nil {
750 errs = append(errs, err)
751 }
752 }
753 }
754 if len(errs) == 0 {
755 s.releaseLocked()
756 return nil
757 }
758 s.stopped = true
759 d.markLeaked(s.slot)
760 s.leakErr = fmt.Errorf("%w: vm %s: %w", sandbox.ErrLeaked, s.id, errors.Join(errs...))
761 log.Error().Err(s.leakErr).Str("vm", s.id).Msg("micro-VM cleanup incomplete; call Reap to finish it")
762 return s.leakErr
763}
764
765// removeAll is os.RemoveAll that also reports a directory that is still there.
766func removeAll(dir string) error {
767 if err := os.RemoveAll(dir); err != nil {
768 return err
769 }
770 if _, err := os.Lstat(dir); err == nil {
771 return fmt.Errorf("%s still exists after removal", dir)
772 }
773 return nil
774}
775
776// removeCgroup removes the cgroup the jailer created for a VM
777// (<root>/<parent>/<id>, parent = the exec file name). The jailer never removes
778// it. A missing cgroup is fine; a non-empty one fails (rmdir semantics).
779func removeCgroup(root, parent, id string) error {
780 err := os.Remove(filepath.Join(root, parent, id))
781 if err == nil || errors.Is(err, os.ErrNotExist) {
782 return nil
783 }
784 return fmt.Errorf("remove cgroup %s: %w", filepath.Join(root, parent, id), err)
785}