1// SPDX-License-Identifier: MIT
2
3// Package vm is the micro-VM sandbox: one Firecracker VM per job.
4//
5// Per VM, on the host: a jailer chroot (or, without jailer, a plain directory)
6// holding the per-job sparse scratch disk and the vsock socket; a TAP device
7// with a /30; rules in the dedicated nftables table; the DNS forwarder bound on
8// the TAP's host address. In the guest: three virtio disks (read-only runner
9// executable as the boot root, read-only shared base image, per-job scratch)
10// and a vsock device. Everything is traced in a state-directory record before it
11// is created, so that Reap can remove it after a crash.
12package vm
13
14import (
15 "context"
16 "errors"
17 "fmt"
18 "io/fs"
19 "net"
20 "net/netip"
21 "os"
22 "os/exec"
23 "path/filepath"
24 "strconv"
25 "sync"
26 "syscall"
27 "time"
28
29 "github.com/bnema/zerowrap"
30
31 "bnema.dev/ermite-runner/internal/cmdrun"
32 "bnema.dev/ermite-runner/internal/netpolicy"
33 "bnema.dev/ermite-runner/internal/sandbox"
34 "bnema.dev/ermite-runner/internal/sandbox/disk"
35 "bnema.dev/ermite-runner/internal/sandbox/netsetup"
36 "bnema.dev/ermite-runner/internal/tcprelay"
37)
38
39// Options are the testable knobs of the driver.
40type Options struct {
41 Run cmdrun.Runner
42 // ProcDir is the /proc used to find orphan VMM processes.
43 ProcDir string
44 // CgroupRoot is the cgroup v2 mount, where the jailer creates one cgroup per
45 // VM (default /sys/fs/cgroup).
46 CgroupRoot string
47 // BootTimeout bounds boot + init + executor connection (default 90 s).
48 BootTimeout time.Duration
49 // Net overrides the network plumbing (tests).
50 Net netsetup.Plumbing
51 // FirecrackerArgs are extra arguments for Firecracker (tests use them to
52 // select the fake).
53 FirecrackerArgs []string
54 // FreeSpace overrides statfs for the free-space warning (tests).
55 FreeSpace func(string) (int64, error)
56 // kill overrides killAndWait (tests simulate a VMM that cannot be killed).
57 kill func(pid int, still func() bool, timeout time.Duration) bool
58}
59
60// vmmOverheadMiB is what the Firecracker process itself may use on top of the
61// guest memory (device emulation, the API server, the seccomp filter, stacks):
62// it is added to memory.max so that the guest, not the VMM, is what gets limited.
63const vmmOverheadMiB = 256
64
65// Driver implements sandbox.Driver for micro-VMs.
66type Driver struct {
67 cfg sandbox.Config
68 opt Options
69 fcPath string
70 jailer string
71 execDisk string
72 // extMu serialises the building of the externals disk with the linking of it
73 // into a sandbox (see shareExternals).
74 extMu sync.Mutex
75 store store
76 net netsetup.Plumbing
77 lock *sandbox.HostLock
78
79 mu sync.Mutex
80 slots *sandbox.Slots
81}
82
83// New builds the driver: takes the host-wide lock, resolves binaries, builds the
84// exec disk and enables IPv4 forwarding (a host-wide sysctl that stays on: see
85// the README). Run Reap before New sandboxes.
86func New(ctx context.Context, cfg sandbox.Config, opt Options) (*Driver, error) {
87 if opt.Run == nil {
88 opt.Run = cmdrun.Exec{}
89 }
90 if opt.ProcDir == "" {
91 opt.ProcDir = "/proc"
92 }
93 if opt.CgroupRoot == "" {
94 opt.CgroupRoot = "/sys/fs/cgroup"
95 }
96 if opt.BootTimeout == 0 {
97 opt.BootTimeout = 90 * time.Second
98 }
99 if opt.kill == nil {
100 opt.kill = killAndWait
101 }
102 d := &Driver{cfg: cfg, opt: opt, slots: sandbox.NewSlots(cfg.Capacity), store: store{dir: filepath.Join(cfg.StateDir, "vms")}}
103
104 var err error
105 if d.fcPath, err = resolveBin(cfg.Firecracker, "firecracker"); err != nil {
106 return nil, err
107 }
108 if !cfg.JailerOff {
109 if d.jailer, err = resolveBin(cfg.Jailer, "jailer"); err != nil {
110 return nil, err
111 }
112 }
113 exe, err := cfg.ExecutablePath()
114 if err != nil {
115 return nil, err
116 }
117 if err := sandbox.CheckStatic(exe); err != nil {
118 return nil, err
119 }
120 if err := os.MkdirAll(cfg.StateDir, 0o755); err != nil {
121 return nil, fmt.Errorf("state directory: %w", err)
122 }
123 if d.lock, err = sandbox.AcquireHostLock(cfg.LockFile); err != nil {
124 return nil, err
125 }
126 fail := func(err error) (*Driver, error) { _ = d.Close(); return nil, err }
127
128 // A path over sun_path fails later as a 10 s "API not ready" (or a vsock
129 // that never connects): refuse at start, in both modes.
130 if err := checkSocketPaths(cfg.StateDir, filepath.Base(d.fcPath), !cfg.JailerOff); err != nil {
131 return fail(err)
132 }
133 if !cfg.JailerOff {
134 // Jailer 1.17 canonicalizes --chroot-base-dir and fails when it does not
135 // exist ("Failed to canonicalize path"); older releases created it.
136 if err := ensureJailBase(jailBase(cfg.StateDir)); err != nil {
137 return fail(err)
138 }
139 for _, p := range []string{cfg.KernelPath(), cfg.BaseImagePath()} {
140 if err := checkWorldReadable(p); err != nil {
141 return fail(err)
142 }
143 }
144 }
145 sandbox.WarnIfLowSpace(ctx, cfg.StateDir, cfg.Capacity, cfg.JobDiskMiB, opt.FreeSpace)
146 if err := disk.CheckMkfs(ctx, opt.Run); err != nil {
147 return fail(err)
148 }
149 if d.execDisk, err = ensureExecDisk(ctx, opt.Run, exe, cfg.StateDir); err != nil {
150 return fail(err)
151 }
152 d.net = opt.Net
153 if d.net == nil {
154 n := netsetup.New(opt.Run, cfg.DNSPort, "")
155 n.DNS.Override = cfg.GuestForge.DNSName()
156 if t := cfg.GuestForge.RelayTarget(); t != "" {
157 n.Relay = tcprelay.New(t)
158 }
159 d.net = n
160 }
161 if cfg.GuestForge != nil {
162 l := zerowrap.FromCtx(ctx)
163 l.Warn().Msg(cfg.GuestForge.Warning())
164 }
165 if err := d.net.EnableForwarding(); err != nil {
166 return fail(err)
167 }
168 return d, nil
169}
170
171func resolveBin(configured, name string) (string, error) {
172 p := configured
173 if p == "" {
174 var err error
175 if p, err = exec.LookPath(name); err != nil {
176 return "", fmt.Errorf("%s not found in PATH (set ERMITE_RUNNER_%s)", name, map[string]string{"firecracker": "FIRECRACKER", "jailer": "JAILER"}[name])
177 }
178 }
179 abs, err := filepath.Abs(p)
180 if err != nil {
181 return "", err
182 }
183 if _, err := os.Stat(abs); err != nil {
184 return "", fmt.Errorf("%s: %w", name, err)
185 }
186 return abs, nil
187}
188
189// Mode implements sandbox.Driver.
190func (d *Driver) Mode() sandbox.Mode { return sandbox.VM }
191
192// Isolation implements sandbox.Driver.
193func (d *Driver) Isolation() sandbox.Isolation { return sandbox.IsolationVM }
194
195// Close releases the host lock and the DNS listeners.
196func (d *Driver) Close() error {
197 if d.net != nil {
198 d.net.Close()
199 }
200 return d.lock.Release()
201}
202
203// New implements sandbox.Driver.
204func (d *Driver) New() (sandbox.Sandbox, error) {
205 id, err := sandbox.NewID()
206 if err != nil {
207 return nil, err
208 }
209 d.mu.Lock()
210 defer d.mu.Unlock()
211 slot, err := d.slots.Acquire()
212 if err != nil {
213 return nil, err
214 }
215 return &vmSandbox{d: d, slot: slot, id: id}, nil
216}
217
218// scope narrows the /proc scan to this driver's own jail base.
219func (d *Driver) scope() scope { return newScope(d.cfg.StateDir) }
220
221func (d *Driver) release(slot int) {
222 d.mu.Lock()
223 d.slots.Release(slot)
224 d.mu.Unlock()
225}
226
227func (d *Driver) markLeaked(slot int) {
228 d.mu.Lock()
229 d.slots.MarkLeaked(slot)
230 d.mu.Unlock()
231}
232
233// vmSandbox is one VM, single use.
234type vmSandbox struct {
235 d *Driver
236 slot int
237 id string
238 gate sandbox.BootGate
239
240 mu sync.Mutex
241 started bool
242 stopped bool
243 // leakErr is the error of a cleanup that left something behind; later Stop
244 // calls return it again.
245 leakErr error
246
247 rec record
248 lay layout
249 cmd *exec.Cmd
250 exited chan struct{}
251 console *ring
252 listener net.Listener
253 cacheOn bool
254 conn net.Conn
255 tap string
256 gateway netip.Addr
257 attached bool
258}
259
260// ID implements sandbox.Sandbox.
261func (s *vmSandbox) ID() string { return s.id }
262
263func (s *vmSandbox) uid() int { return s.d.cfg.UIDBase + s.slot }
264
265// Start implements sandbox.Sandbox.
266func (s *vmSandbox) Start(ctx context.Context, spec sandbox.JobSpec) (sandbox.Conn, error) {
267 // A Stop from another goroutine aborts the boot through the gate.
268 bootCtx, err := s.gate.Begin(ctx)
269 if err != nil {
270 return nil, fmt.Errorf("vm sandbox: %w", err)
271 }
272 defer s.gate.End()
273
274 s.mu.Lock()
275 defer s.mu.Unlock()
276 if s.started || s.stopped {
277 return nil, errors.New("vm sandbox: Start called twice or after Stop")
278 }
279 s.started = true
280 log := zerowrap.FromCtx(ctx)
281 d := s.d
282 spec = d.cfg.Resolve(spec)
283 if d.cfg.GuestForge != nil {
284 // At every job start, as for privileged containers.
285 log.Warn().Str("job", spec.JobID).Str("vm", s.id).Msg(d.cfg.GuestForge.Warning())
286 }
287
288 jailed := !d.cfg.JailerOff
289 runUID := s.uid()
290 if !jailed {
291 runUID = os.Getuid()
292 }
293 s.lay = newLayout(d.cfg.StateDir, filepath.Base(d.fcPath), s.id, jailed, runUID)
294 s.rec = record{ID: s.id, Slot: s.slot, Job: spec.JobID, Dir: s.lay.dir}
295 // The record exists before the first resource, so any crash from here on is
296 // reapable.
297 if err := d.store.save(s.rec); err != nil {
298 s.releaseLocked()
299 return nil, fmt.Errorf("write state record: %w", err)
300 }
301
302 conn, err := s.boot(bootCtx, spec)
303 if err != nil {
304 console := ""
305 if s.console != nil {
306 console = s.console.String()
307 }
308 cerr := s.cleanupLocked(context.WithoutCancel(ctx))
309 if console != "" {
310 err = fmt.Errorf("%w\nVMM console tail:\n%s", err, console)
311 }
312 return nil, errors.Join(err, cerr)
313 }
314 log.Info().Str("vm", s.id).Int("slot", s.slot).Str("job", spec.JobID).Msg("micro-VM running")
315 return conn, nil
316}
317
318// boot runs the stages in order; each stage records what it created so that
319// cleanupLocked can undo exactly that.
320func (s *vmSandbox) boot(ctx context.Context, spec sandbox.JobSpec) (sandbox.Conn, error) {
321 netw, err := netpolicy.Subnet(s.d.cfg.Network, s.slot)
322 if err != nil {
323 return nil, err
324 }
325 if err := s.bootNetwork(ctx, netw); err != nil {
326 return nil, err
327 }
328 api, err := s.bootVMM(ctx, spec)
329 if err != nil {
330 return nil, err
331 }
332 files, err := s.bootFiles(ctx, spec)
333 if err != nil {
334 return nil, err
335 }
336 accepted, err := s.bootVsock()
337 if err != nil {
338 return nil, err
339 }
340 if err := s.bootCache(spec); err != nil {
341 return nil, err
342 }
343 if err := api.configure(ctx, s.vmConfig(spec, netw, files)); err != nil {
344 return nil, err
345 }
346 return s.waitExecutor(ctx, accepted)
347}
348
349// bootNetwork creates the TAP (it must exist when Firecracker opens it) and
350// installs the firewall policy and the DNS listener.
351func (s *vmSandbox) bootNetwork(ctx context.Context, netw netpolicy.Net) error {
352 d := s.d
353 s.tap, s.gateway = tapName(s.slot), netw.Gateway
354 s.rec.Tap, s.rec.Gate = s.tap, s.gateway.String()
355 if err := d.store.save(s.rec); err != nil {
356 return err
357 }
358 if err := createTap(ctx, d.opt.Run, s.tap, s.lay.uid, s.gateway); err != nil {
359 return err
360 }
361 s.attached = true // from here Detach is idempotent and always safe
362 return d.net.Attach(ctx, netpolicy.Guest{Iface: s.tap, Gateway: netw.Gateway, Addr: netw.Guest, ForgePort: d.cfg.GuestForge.TCPPort()})
363}
364
365// bootVMM starts the jailer (or Firecracker) and waits for its API.
366func (s *vmSandbox) bootVMM(ctx context.Context, spec sandbox.JobSpec) (*apiClient, error) {
367 apiSock := s.lay.host("api.sock")
368 if s.lay.jailed {
369 apiSock = s.lay.host("run/firecracker.socket")
370 } else if err := os.MkdirAll(s.lay.root, 0o700); err != nil {
371 return nil, err
372 }
373 if err := s.startVMM(spec, apiSock); err != nil {
374 return nil, err
375 }
376 api := newAPIClient(apiSock)
377 if err := s.waitAPI(ctx, api, apiSock); err != nil {
378 return nil, err
379 }
380 return api, nil
381}
382
383// vmFiles are the paths Firecracker is given.
384type vmFiles struct{ execDisk, baseDisk, kernel, externals string }
385
386// bootFiles makes the shared files available and creates the scratch disk.
387func (s *vmSandbox) bootFiles(ctx context.Context, spec sandbox.JobSpec) (vmFiles, error) {
388 d := s.d
389 var f vmFiles
390 var err error
391 if f.execDisk, err = s.lay.share(d.execDisk, "exec.ext4"); err != nil {
392 return f, err
393 }
394 if f.baseDisk, err = s.lay.share(d.cfg.BaseImagePath(), "base.ext4"); err != nil {
395 return f, err
396 }
397 if f.kernel, err = s.lay.share(d.cfg.KernelPath(), "kernel"); err != nil {
398 return f, err
399 }
400 f.externals = s.shareExternals(ctx)
401 owner := -1
402 if s.lay.jailed {
403 owner = s.lay.uid
404 }
405 return f, disk.CreateScratch(ctx, d.opt.Run, s.lay.host("scratch.ext4"), spec.DiskMiB, owner)
406}
407
408// shareExternals makes the read-only disk of the host's external tools available
409// to Firecracker; "" means the sandbox has none. A disk that cannot be built
410// does not stop the job: it starts without the tools and the failure is logged.
411func (s *vmSandbox) shareExternals(ctx context.Context) string {
412 d := s.d
413 set := d.cfg.ExternalsOf()
414 if len(set) == 0 {
415 return ""
416 }
417 log := zerowrap.FromCtx(ctx)
418 d.extMu.Lock()
419 defer d.extMu.Unlock()
420 path, err := ensureExternalsDisk(ctx, d.opt.Run, set, d.cfg.StateDir)
421 if err == nil && path != "" {
422 path, err = s.lay.share(path, "externals.ext4")
423 }
424 if err != nil {
425 log.Error().Err(err).Str("vm", s.id).Msg("external tools (Node.js) not attached to this job")
426 return ""
427 }
428 return path
429}
430
431// bootVsock listens on the host side of the vsock stream: Firecracker connects
432// to <uds_path>_<port> when the guest connects to the host CID.
433func (s *vmSandbox) bootVsock() (<-chan acceptResult, error) {
434 path := s.lay.host(udsName) + fmt.Sprintf("_%d", sandbox.VsockPort)
435 ln, err := net.Listen("unix", path)
436 if err != nil {
437 return nil, fmt.Errorf("listen vsock socket: %w", err)
438 }
439 s.listener = ln
440 if s.lay.jailed {
441 if err := os.Lchown(path, s.lay.uid, s.lay.uid); err != nil {
442 return nil, err
443 }
444 }
445 accepted := make(chan acceptResult, 1)
446 go func() { c, err := ln.Accept(); accepted <- acceptResult{c, err} }()
447 return accepted, nil
448}
449
450// cacheSocket is the host side of the cache streams of the VM: Firecracker
451// connects to <uds_path>_<CacheVsockPort> each time the guest relay connects to
452// the host on that vsock port. The path is inside this VM's own directory (its
453// chroot with the jailer), so no other VM reaches it.
454func (s *vmSandbox) cacheSocket() string {
455 return s.lay.host(udsName) + fmt.Sprintf("_%d", sandbox.CacheVsockPort)
456}
457
458// bootCache serves the cache on the VM's cache socket, owned by the jailed uid
459// as the executor's.
460func (s *vmSandbox) bootCache(spec sandbox.JobSpec) error {
461 if s.d.cfg.Cache == nil {
462 return nil
463 }
464 err := s.d.cfg.ServeCache(s.id, s.cacheSocket(), spec, s.d.cfg.NetMbit, func(p string) error {
465 if s.lay.jailed {
466 return os.Lchown(p, s.lay.uid, s.lay.uid)
467 }
468 return nil
469 })
470 if err != nil {
471 return fmt.Errorf("serve the cache socket: %w", err)
472 }
473 s.cacheOn = true
474 return nil
475}
476
477const udsName = "v.sock"
478
479// maxSockPath is the longest Unix socket path Linux accepts (108 with the NUL).
480const maxSockPath = 107
481
482// socketPaths lists the host paths of the unix sockets of a VM: the API socket
483// and the vsock listener (the guest's UDS path plus "_<port>"). With the jailer
484// they are deep in the chroot; without it they sit in <state>/vm/<id>.
485func socketPaths(stateDir, execName string, jailed bool) []string {
486 lay := newLayout(stateDir, execName, "er00000000", jailed, 0)
487 api := lay.host("api.sock")
488 if jailed {
489 api = lay.host("run/firecracker.socket")
490 }
491 return []string{api, lay.host(udsName) + fmt.Sprintf("_%d", sandbox.VsockPort), lay.host(udsName) + fmt.Sprintf("_%d", sandbox.CacheVsockPort)}
492}
493
494// checkSocketPaths refuses a state directory that makes a VM socket path longer
495// than a unix socket address.
496func checkSocketPaths(stateDir, execName string, jailed bool) error {
497 for _, p := range socketPaths(stateDir, execName, jailed) {
498 if len(p) > maxSockPath {
499 return fmt.Errorf("socket path %q is %d bytes, over the %d-byte limit of a Unix socket: use a shorter %s", p, len(p), maxSockPath, sandbox.EnvStateDir)
500 }
501 }
502 return nil
503}
504
505// ensureJailBase creates the jailer's chroot base and refuses one that others
506// could use to redirect the jailer: anything but a real directory (a symlink is
507// not followed), owned by neither root nor the runner's own user, or writable by
508// group or others. An existing directory is checked, never chmod-ed.
509func ensureJailBase(dir string) error {
510 if err := os.MkdirAll(dir, 0o700); err != nil {
511 return fmt.Errorf("jailer chroot base: %w", err)
512 }
513 fi, err := os.Lstat(dir)
514 if err != nil {
515 return fmt.Errorf("jailer chroot base: %w", err)
516 }
517 return checkJailBaseInfo(dir, fi, os.Geteuid())
518}
519
520func checkJailBaseInfo(dir string, fi fs.FileInfo, euid int) error {
521 if !fi.IsDir() {
522 return fmt.Errorf("jailer chroot base %s is not a directory (%v): remove it", dir, fi.Mode().Type())
523 }
524 if fi.Mode().Perm()&0o022 != 0 {
525 return fmt.Errorf("jailer chroot base %s is writable by group or others (mode %04o): chmod go-w it", dir, fi.Mode().Perm())
526 }
527 st, ok := fi.Sys().(*syscall.Stat_t)
528 if !ok {
529 return fmt.Errorf("jailer chroot base %s: cannot read its owner", dir)
530 }
531 // The jailer needs root, so the runner is root in production; accepting its own
532 // user as well keeps the driver testable without root.
533 if int(st.Uid) != 0 && int(st.Uid) != euid {
534 return fmt.Errorf("jailer chroot base %s is owned by uid %d, not root: chown it", dir, st.Uid)
535 }
536 return nil
537}
538
539func (s *vmSandbox) vmConfig(spec sandbox.JobSpec, netw netpolicy.Net, f vmFiles) vmConfig {
540 cfg := s.d.cfg
541 // The order decides the guest device names (vda, vdb, ...): new drives go last.
542 drives := []drive{
543 {DriveID: "rootfs", PathOnHost: f.execDisk, IsRootDevice: true, IsReadOnly: true},
544 {DriveID: "base", PathOnHost: f.baseDisk, IsReadOnly: true},
545 {DriveID: "scratch", PathOnHost: s.lay.fc("scratch.ext4"), RateLimiter: diskLimiter(cfg.DiskBandwidthMiB, cfg.DiskIOPS)},
546 }
547 if f.externals != "" {
548 drives = append(drives, drive{DriveID: "externals", PathOnHost: f.externals, IsReadOnly: true})
549 }
550 return vmConfig{
551 Machine: machineConfig{VcpuCount: spec.CPUs, MemSizeMiB: spec.MemoryMiB},
552 Boot: bootSource{KernelImagePath: f.kernel, BootArgs: bootArgs(netw.Guest, netw.Gateway, time.Now().Unix(), f.externals != "")},
553 Drives: drives,
554 Net: networkInterface{
555 IfaceID: "eth0", HostDevName: s.tap, GuestMAC: macFor(s.slot),
556 RxRateLimiter: netLimiter(cfg.NetMbit), TxRateLimiter: netLimiter(cfg.NetMbit),
557 },
558 Vsock: vsockDevice{GuestCID: 3, UDSPath: s.lay.fc(udsName)},
559 }
560}
561
562// waitExecutor waits for the guest's executor to connect. Only ctx (the caller's
563// context, or Stop through the boot gate) and the boot timeout end the wait: a
564// boot cannot be aborted any other way.
565func (s *vmSandbox) waitExecutor(ctx context.Context, accepted <-chan acceptResult) (sandbox.Conn, error) {
566 timer := time.NewTimer(s.d.opt.BootTimeout)
567 defer timer.Stop()
568 select {
569 case r := <-accepted:
570 if r.err != nil {
571 return nil, fmt.Errorf("accept executor connection: %w", r.err)
572 }
573 s.conn = r.c
574 return r.c, nil
575 case <-s.exited:
576 return nil, errors.New("firecracker exited before the executor connected")
577 case <-timer.C:
578 return nil, fmt.Errorf("executor did not connect within %s", s.d.opt.BootTimeout)
579 case <-ctx.Done():
580 return nil, context.Cause(ctx)
581 }
582}
583
584type acceptResult struct {
585 c net.Conn
586 err error
587}
588
589// jailerArgs builds the jailer command line (after the binary name).
590//
591// - cgroup v2 with memory.max (guest memory + VMM overhead) and pids.max (a
592// VMM needs a handful of threads per vCPU): a runaway or compromised VMM
593// cannot take the host's memory or fork-bomb it;
594// - a new PID namespace: the VMM cannot see or signal host processes;
595// - no-file: the descriptor budget of the VMM.
596func jailerArgs(jailer, fc, id string, uid int, chrootBase string, spec sandbox.JobSpec, extra []string) []string {
597 memBytes := (int64(spec.MemoryMiB) + vmmOverheadMiB) << 20
598 pids := 32 + 8*spec.CPUs
599 args := []string{
600 "--id", id, "--exec-file", fc,
601 "--uid", strconv.Itoa(uid), "--gid", strconv.Itoa(uid),
602 "--chroot-base-dir", chrootBase,
603 "--cgroup-version", "2",
604 "--cgroup", "memory.max=" + strconv.FormatInt(memBytes, 10),
605 "--cgroup", "pids.max=" + strconv.Itoa(pids),
606 "--new-pid-ns",
607 "--resource-limit", "no-file=4096",
608 "--",
609 }
610 return append(args, extra...)
611}
612
613func (s *vmSandbox) startVMM(spec sandbox.JobSpec, apiSock string) error {
614 d := s.d
615 var cmd *exec.Cmd
616 if s.lay.jailed {
617 args := jailerArgs(d.jailer, d.fcPath, s.id, s.uid(), jailBase(d.cfg.StateDir), spec, d.opt.FirecrackerArgs)
618 cmd = exec.Command(d.jailer, args...)
619 } else {
620 args := append([]string{"--id", s.id, "--api-sock", apiSock}, d.opt.FirecrackerArgs...)
621 cmd = exec.Command(d.fcPath, args...)
622 }
623 s.console = newRing(16 << 10)
624 cmd.Stdout, cmd.Stderr = s.console, s.console
625 cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
626 if err := cmd.Start(); err != nil {
627 return fmt.Errorf("start firecracker: %w", err)
628 }
629 s.cmd = cmd
630 s.exited = make(chan struct{})
631 jailed := s.lay.jailed
632 go func() {
633 _ = cmd.Wait()
634 // With --new-pid-ns the jailer may exit as soon as it has started the
635 // VMM in its namespace: the VMM is then gone only when its own pid is.
636 if jailed {
637 s.awaitVMMGone()
638 }
639 close(s.exited)
640 }()
641 s.rec.PID = cmd.Process.Pid
642 return s.d.store.save(s.rec)
643}
644
645// awaitVMMGone blocks until no process of this VM's id is left.
646func (s *vmSandbox) awaitVMMGone() {
647 for {
648 alive := false
649 for pid, id := range vmmProcs(s.d.opt.ProcDir, s.d.scope()) {
650 if id == s.id && pidAlive(pid) {
651 alive = true
652 }
653 }
654 if !alive {
655 return
656 }
657 time.Sleep(50 * time.Millisecond)
658 }
659}
660
661func (s *vmSandbox) waitAPI(ctx context.Context, api *apiClient, sock string) error {
662 deadline := time.Now().Add(10 * time.Second)
663 for {
664 if err := api.ready(ctx); err == nil {
665 return nil
666 }
667 select {
668 case <-s.exited:
669 return errors.New("firecracker exited before its API was ready")
670 case <-ctx.Done():
671 return context.Cause(ctx)
672 case <-time.After(25 * time.Millisecond):
673 }
674 if time.Now().After(deadline) {
675 return fmt.Errorf("firecracker API socket %s not ready after 10s", sock)
676 }
677 }
678}
679
680// Stop implements sandbox.Sandbox.
681func (s *vmSandbox) Stop(ctx context.Context) error {
682 // Abort a boot in progress first: Start holds s.mu for its whole duration.
683 s.gate.Abort()
684 s.mu.Lock()
685 defer s.mu.Unlock()
686 if s.stopped {
687 return s.leakErr
688 }
689 if !s.started {
690 s.releaseLocked()
691 return nil
692 }
693 return s.cleanupLocked(context.WithoutCancel(ctx))
694}
695
696func (s *vmSandbox) releaseLocked() {
697 s.stopped = true
698 s.d.release(s.slot)
699}
700
701// vmmPIDs returns every host process of this VM: the launched command and
702// whatever the /proc scan finds with this id.
703func (s *vmSandbox) vmmPIDs() []int {
704 set := map[int]bool{}
705 if s.cmd != nil && s.cmd.Process != nil {
706 set[s.cmd.Process.Pid] = true
707 }
708 for pid, id := range vmmProcs(s.d.opt.ProcDir, s.d.scope()) {
709 if id == s.id {
710 set[pid] = true
711 }
712 }
713 var out []int
714 for pid := range set {
715 out = append(out, pid)
716 }
717 return out
718}
719
720// stillMine is the identity check handed to the kill: pid still carries this
721// VM's id and passes the scope (command line and root).
722func (s *vmSandbox) stillMine(pid int) func() bool {
723 return func() bool { return cmdlineID(s.d.opt.ProcDir, pid, s.d.scope()) == s.id }
724}
725
726// cleanupLocked removes everything the VM created, in dependency order. Steps
727// that need the VMM to be dead (network device, firewall rules, files, cgroup)
728// are skipped when it could not be killed: removing the TAP or the policy under
729// a VM that may still run would break the isolation it relies on, so the
730// sandbox is reported as leaked (sandbox.ErrLeaked), its slot stays taken, and
731// Reap finishes the job.
732func (s *vmSandbox) cleanupLocked(ctx context.Context) error {
733 log := zerowrap.FromCtx(ctx)
734 d := s.d
735 var errs []error
736 if s.conn != nil {
737 _ = s.conn.Close()
738 s.conn = nil
739 }
740 if s.listener != nil {
741 _ = s.listener.Close()
742 s.listener = nil
743 }
744 if s.cacheOn {
745 d.cfg.StopCache(s.id)
746 s.cacheOn = false
747 }
748 dead := true
749 // No kill(-pgid) of the launched command: once it has been waited for, its
750 // pid (and so its group id) may belong to anyone. Every process of this VM is
751 // found by the scan (id, markers and scope) and killed through a pidfd.
752 for _, pid := range s.vmmPIDs() {
753 if !d.opt.kill(pid, s.stillMine(pid), 5*time.Second) {
754 dead = false
755 errs = append(errs, fmt.Errorf("vmm process %d did not exit", pid))
756 }
757 }
758 if dead && s.exited != nil {
759 select {
760 case <-s.exited:
761 case <-time.After(5 * time.Second):
762 dead = false
763 errs = append(errs, errors.New("vmm exit was not observed"))
764 }
765 }
766 if dead {
767 if s.attached {
768 if err := d.net.Detach(ctx, s.tap, s.gateway); err != nil {
769 errs = append(errs, err)
770 }
771 if err := deleteTap(ctx, d.opt.Run, s.tap); err != nil {
772 errs = append(errs, err)
773 } else {
774 s.attached = false
775 }
776 }
777 if s.lay.dir != "" {
778 if err := removeAll(s.lay.dir); err != nil {
779 errs = append(errs, err)
780 }
781 }
782 if s.lay.jailed {
783 if err := removeCgroup(d.opt.CgroupRoot, filepath.Base(d.fcPath), s.id); err != nil {
784 errs = append(errs, err)
785 }
786 }
787 if len(errs) == 0 {
788 if err := d.store.remove(s.id); err != nil {
789 errs = append(errs, err)
790 }
791 }
792 }
793 if len(errs) == 0 {
794 s.releaseLocked()
795 return nil
796 }
797 s.stopped = true
798 d.markLeaked(s.slot)
799 s.leakErr = fmt.Errorf("%w: vm %s: %w", sandbox.ErrLeaked, s.id, errors.Join(errs...))
800 log.Error().Err(s.leakErr).Str("vm", s.id).Msg("micro-VM cleanup incomplete; call Reap to finish it")
801 return s.leakErr
802}
803
804// removeAll is os.RemoveAll that also reports a directory that is still there.
805func removeAll(dir string) error {
806 if err := os.RemoveAll(dir); err != nil {
807 return err
808 }
809 if _, err := os.Lstat(dir); err == nil {
810 return fmt.Errorf("%s still exists after removal", dir)
811 }
812 return nil
813}
814
815// removeCgroup removes the cgroup the jailer created for a VM
816// (<root>/<parent>/<id>, parent = the exec file name). The jailer never removes
817// it. A missing cgroup is fine; a non-empty one fails (rmdir semantics).
818func removeCgroup(root, parent, id string) error {
819 err := os.Remove(filepath.Join(root, parent, id))
820 if err == nil || errors.Is(err, os.ErrNotExist) {
821 return nil
822 }
823 return fmt.Errorf("remove cgroup %s: %w", filepath.Join(root, parent, id), err)
824}