1// SPDX-License-Identifier: MIT
2
3// Package vm is the micro-VM sandbox: one Firecracker VM per job.
4//
5// Per VM, on the host: a jailer chroot (or, without jailer, a plain directory)
6// holding the per-job sparse scratch disk and the vsock socket; a TAP device
7// with a /30; rules in the dedicated nftables table; the DNS forwarder bound on
8// the TAP's host address. In the guest: three virtio disks (read-only runner
9// executable as the boot root, read-only shared base image, per-job scratch)
10// and a vsock device. Everything is traced in a state-directory record before it
11// is created, so that Reap can remove it after a crash.
12package vm
13
14import (
15 "context"
16 "errors"
17 "fmt"
18 "io/fs"
19 "net"
20 "net/netip"
21 "os"
22 "os/exec"
23 "path/filepath"
24 "strconv"
25 "sync"
26 "syscall"
27 "time"
28
29 "github.com/bnema/zerowrap"
30
31 "bnema.dev/ermite-runner/internal/cmdrun"
32 "bnema.dev/ermite-runner/internal/netpolicy"
33 "bnema.dev/ermite-runner/internal/sandbox"
34 "bnema.dev/ermite-runner/internal/sandbox/disk"
35 "bnema.dev/ermite-runner/internal/sandbox/netsetup"
36)
37
38// Options are the testable knobs of the driver.
39type Options struct {
40 Run cmdrun.Runner
41 // ProcDir is the /proc used to find orphan VMM processes.
42 ProcDir string
43 // CgroupRoot is the cgroup v2 mount, where the jailer creates one cgroup per
44 // VM (default /sys/fs/cgroup).
45 CgroupRoot string
46 // BootTimeout bounds boot + init + executor connection (default 90 s).
47 BootTimeout time.Duration
48 // Net overrides the network plumbing (tests).
49 Net netsetup.Plumbing
50 // FirecrackerArgs are extra arguments for Firecracker (tests use them to
51 // select the fake).
52 FirecrackerArgs []string
53 // FreeSpace overrides statfs for the free-space warning (tests).
54 FreeSpace func(string) (int64, error)
55 // kill overrides killAndWait (tests simulate a VMM that cannot be killed).
56 kill func(pid int, still func() bool, timeout time.Duration) bool
57}
58
59// vmmOverheadMiB is what the Firecracker process itself may use on top of the
60// guest memory (device emulation, the API server, the seccomp filter, stacks):
61// it is added to memory.max so that the guest, not the VMM, is what gets limited.
62const vmmOverheadMiB = 256
63
64// Driver implements sandbox.Driver for micro-VMs.
65type Driver struct {
66 cfg sandbox.Config
67 opt Options
68 fcPath string
69 jailer string
70 execDisk string
71 // extMu serialises the building of the externals disk with the linking of it
72 // into a sandbox (see shareExternals).
73 extMu sync.Mutex
74 store store
75 net netsetup.Plumbing
76 lock *sandbox.HostLock
77
78 mu sync.Mutex
79 slots *sandbox.Slots
80}
81
82// New builds the driver: takes the host-wide lock, resolves binaries, builds the
83// exec disk and enables IPv4 forwarding (a host-wide sysctl that stays on: see
84// the README). Run Reap before New sandboxes.
85func New(ctx context.Context, cfg sandbox.Config, opt Options) (*Driver, error) {
86 if opt.Run == nil {
87 opt.Run = cmdrun.Exec{}
88 }
89 if opt.ProcDir == "" {
90 opt.ProcDir = "/proc"
91 }
92 if opt.CgroupRoot == "" {
93 opt.CgroupRoot = "/sys/fs/cgroup"
94 }
95 if opt.BootTimeout == 0 {
96 opt.BootTimeout = 90 * time.Second
97 }
98 if opt.kill == nil {
99 opt.kill = killAndWait
100 }
101 d := &Driver{cfg: cfg, opt: opt, slots: sandbox.NewSlots(cfg.Capacity), store: store{dir: filepath.Join(cfg.StateDir, "vms")}}
102
103 var err error
104 if d.fcPath, err = resolveBin(cfg.Firecracker, "firecracker"); err != nil {
105 return nil, err
106 }
107 if !cfg.JailerOff {
108 if d.jailer, err = resolveBin(cfg.Jailer, "jailer"); err != nil {
109 return nil, err
110 }
111 }
112 exe, err := cfg.ExecutablePath()
113 if err != nil {
114 return nil, err
115 }
116 if err := sandbox.CheckStatic(exe); err != nil {
117 return nil, err
118 }
119 if err := os.MkdirAll(cfg.StateDir, 0o755); err != nil {
120 return nil, fmt.Errorf("state directory: %w", err)
121 }
122 if d.lock, err = sandbox.AcquireHostLock(cfg.LockFile); err != nil {
123 return nil, err
124 }
125 fail := func(err error) (*Driver, error) { _ = d.Close(); return nil, err }
126
127 // A path over sun_path fails later as a 10 s "API not ready" (or a vsock
128 // that never connects): refuse at start, in both modes.
129 if err := checkSocketPaths(cfg.StateDir, filepath.Base(d.fcPath), !cfg.JailerOff); err != nil {
130 return fail(err)
131 }
132 if !cfg.JailerOff {
133 // Jailer 1.17 canonicalizes --chroot-base-dir and fails when it does not
134 // exist ("Failed to canonicalize path"); older releases created it.
135 if err := ensureJailBase(jailBase(cfg.StateDir)); err != nil {
136 return fail(err)
137 }
138 for _, p := range []string{cfg.KernelPath(), cfg.BaseImagePath()} {
139 if err := checkWorldReadable(p); err != nil {
140 return fail(err)
141 }
142 }
143 }
144 sandbox.WarnIfLowSpace(ctx, cfg.StateDir, cfg.Capacity, cfg.JobDiskMiB, opt.FreeSpace)
145 if err := disk.CheckMkfs(ctx, opt.Run); err != nil {
146 return fail(err)
147 }
148 if d.execDisk, err = ensureExecDisk(ctx, opt.Run, exe, cfg.StateDir); err != nil {
149 return fail(err)
150 }
151 d.net = opt.Net
152 if d.net == nil {
153 n := netsetup.New(opt.Run, cfg.DNSPort, "")
154 n.DNS.Override = cfg.GuestForge.DNSName()
155 d.net = n
156 }
157 if cfg.GuestForge != nil {
158 l := zerowrap.FromCtx(ctx)
159 l.Warn().Msg(cfg.GuestForge.Warning())
160 }
161 if err := d.net.EnableForwarding(); err != nil {
162 return fail(err)
163 }
164 return d, nil
165}
166
167func resolveBin(configured, name string) (string, error) {
168 p := configured
169 if p == "" {
170 var err error
171 if p, err = exec.LookPath(name); err != nil {
172 return "", fmt.Errorf("%s not found in PATH (set ERMITE_RUNNER_%s)", name, map[string]string{"firecracker": "FIRECRACKER", "jailer": "JAILER"}[name])
173 }
174 }
175 abs, err := filepath.Abs(p)
176 if err != nil {
177 return "", err
178 }
179 if _, err := os.Stat(abs); err != nil {
180 return "", fmt.Errorf("%s: %w", name, err)
181 }
182 return abs, nil
183}
184
185// Mode implements sandbox.Driver.
186func (d *Driver) Mode() sandbox.Mode { return sandbox.VM }
187
188// Isolation implements sandbox.Driver.
189func (d *Driver) Isolation() sandbox.Isolation { return sandbox.IsolationVM }
190
191// Close releases the host lock and the DNS listeners.
192func (d *Driver) Close() error {
193 if d.net != nil {
194 d.net.Close()
195 }
196 return d.lock.Release()
197}
198
199// New implements sandbox.Driver.
200func (d *Driver) New() (sandbox.Sandbox, error) {
201 id, err := sandbox.NewID()
202 if err != nil {
203 return nil, err
204 }
205 d.mu.Lock()
206 defer d.mu.Unlock()
207 slot, err := d.slots.Acquire()
208 if err != nil {
209 return nil, err
210 }
211 return &vmSandbox{d: d, slot: slot, id: id}, nil
212}
213
214// scope narrows the /proc scan to this driver's own jail base.
215func (d *Driver) scope() scope { return newScope(d.cfg.StateDir) }
216
217func (d *Driver) release(slot int) {
218 d.mu.Lock()
219 d.slots.Release(slot)
220 d.mu.Unlock()
221}
222
223func (d *Driver) markLeaked(slot int) {
224 d.mu.Lock()
225 d.slots.MarkLeaked(slot)
226 d.mu.Unlock()
227}
228
229// vmSandbox is one VM, single use.
230type vmSandbox struct {
231 d *Driver
232 slot int
233 id string
234 gate sandbox.BootGate
235
236 mu sync.Mutex
237 started bool
238 stopped bool
239 // leakErr is the error of a cleanup that left something behind; later Stop
240 // calls return it again.
241 leakErr error
242
243 rec record
244 lay layout
245 cmd *exec.Cmd
246 exited chan struct{}
247 console *ring
248 listener net.Listener
249 cacheOn bool
250 conn net.Conn
251 tap string
252 gateway netip.Addr
253 attached bool
254}
255
256// ID implements sandbox.Sandbox.
257func (s *vmSandbox) ID() string { return s.id }
258
259func (s *vmSandbox) uid() int { return s.d.cfg.UIDBase + s.slot }
260
261// Start implements sandbox.Sandbox.
262func (s *vmSandbox) Start(ctx context.Context, spec sandbox.JobSpec) (sandbox.Conn, error) {
263 // A Stop from another goroutine aborts the boot through the gate.
264 bootCtx, err := s.gate.Begin(ctx)
265 if err != nil {
266 return nil, fmt.Errorf("vm sandbox: %w", err)
267 }
268 defer s.gate.End()
269
270 s.mu.Lock()
271 defer s.mu.Unlock()
272 if s.started || s.stopped {
273 return nil, errors.New("vm sandbox: Start called twice or after Stop")
274 }
275 s.started = true
276 log := zerowrap.FromCtx(ctx)
277 d := s.d
278 spec = d.cfg.Resolve(spec)
279 if d.cfg.GuestForge != nil {
280 // At every job start, as for privileged containers.
281 log.Warn().Str("job", spec.JobID).Str("vm", s.id).Msg(d.cfg.GuestForge.Warning())
282 }
283
284 jailed := !d.cfg.JailerOff
285 runUID := s.uid()
286 if !jailed {
287 runUID = os.Getuid()
288 }
289 s.lay = newLayout(d.cfg.StateDir, filepath.Base(d.fcPath), s.id, jailed, runUID)
290 s.rec = record{ID: s.id, Slot: s.slot, Job: spec.JobID, Dir: s.lay.dir}
291 // The record exists before the first resource, so any crash from here on is
292 // reapable.
293 if err := d.store.save(s.rec); err != nil {
294 s.releaseLocked()
295 return nil, fmt.Errorf("write state record: %w", err)
296 }
297
298 conn, err := s.boot(bootCtx, spec)
299 if err != nil {
300 console := ""
301 if s.console != nil {
302 console = s.console.String()
303 }
304 cerr := s.cleanupLocked(context.WithoutCancel(ctx))
305 if console != "" {
306 err = fmt.Errorf("%w\nVMM console tail:\n%s", err, console)
307 }
308 return nil, errors.Join(err, cerr)
309 }
310 log.Info().Str("vm", s.id).Int("slot", s.slot).Str("job", spec.JobID).Msg("micro-VM running")
311 return conn, nil
312}
313
314// boot runs the stages in order; each stage records what it created so that
315// cleanupLocked can undo exactly that.
316func (s *vmSandbox) boot(ctx context.Context, spec sandbox.JobSpec) (sandbox.Conn, error) {
317 netw, err := netpolicy.Subnet(s.d.cfg.Network, s.slot)
318 if err != nil {
319 return nil, err
320 }
321 if err := s.bootNetwork(ctx, netw); err != nil {
322 return nil, err
323 }
324 api, err := s.bootVMM(ctx, spec)
325 if err != nil {
326 return nil, err
327 }
328 files, err := s.bootFiles(ctx, spec)
329 if err != nil {
330 return nil, err
331 }
332 accepted, err := s.bootVsock()
333 if err != nil {
334 return nil, err
335 }
336 if err := s.bootCache(spec); err != nil {
337 return nil, err
338 }
339 if err := api.configure(ctx, s.vmConfig(spec, netw, files)); err != nil {
340 return nil, err
341 }
342 return s.waitExecutor(ctx, accepted)
343}
344
345// bootNetwork creates the TAP (it must exist when Firecracker opens it) and
346// installs the firewall policy and the DNS listener.
347func (s *vmSandbox) bootNetwork(ctx context.Context, netw netpolicy.Net) error {
348 d := s.d
349 s.tap, s.gateway = tapName(s.slot), netw.Gateway
350 s.rec.Tap, s.rec.Gate = s.tap, s.gateway.String()
351 if err := d.store.save(s.rec); err != nil {
352 return err
353 }
354 if err := createTap(ctx, d.opt.Run, s.tap, s.lay.uid, s.gateway); err != nil {
355 return err
356 }
357 s.attached = true // from here Detach is idempotent and always safe
358 return d.net.Attach(ctx, netpolicy.Guest{Iface: s.tap, Gateway: netw.Gateway, Addr: netw.Guest, ForgePort: d.cfg.GuestForge.TCPPort()})
359}
360
361// bootVMM starts the jailer (or Firecracker) and waits for its API.
362func (s *vmSandbox) bootVMM(ctx context.Context, spec sandbox.JobSpec) (*apiClient, error) {
363 apiSock := s.lay.host("api.sock")
364 if s.lay.jailed {
365 apiSock = s.lay.host("run/firecracker.socket")
366 } else if err := os.MkdirAll(s.lay.root, 0o700); err != nil {
367 return nil, err
368 }
369 if err := s.startVMM(spec, apiSock); err != nil {
370 return nil, err
371 }
372 api := newAPIClient(apiSock)
373 if err := s.waitAPI(ctx, api, apiSock); err != nil {
374 return nil, err
375 }
376 return api, nil
377}
378
379// vmFiles are the paths Firecracker is given.
380type vmFiles struct{ execDisk, baseDisk, kernel, externals string }
381
382// bootFiles makes the shared files available and creates the scratch disk.
383func (s *vmSandbox) bootFiles(ctx context.Context, spec sandbox.JobSpec) (vmFiles, error) {
384 d := s.d
385 var f vmFiles
386 var err error
387 if f.execDisk, err = s.lay.share(d.execDisk, "exec.ext4"); err != nil {
388 return f, err
389 }
390 if f.baseDisk, err = s.lay.share(d.cfg.BaseImagePath(), "base.ext4"); err != nil {
391 return f, err
392 }
393 if f.kernel, err = s.lay.share(d.cfg.KernelPath(), "kernel"); err != nil {
394 return f, err
395 }
396 f.externals = s.shareExternals(ctx)
397 owner := -1
398 if s.lay.jailed {
399 owner = s.lay.uid
400 }
401 return f, disk.CreateScratch(ctx, d.opt.Run, s.lay.host("scratch.ext4"), spec.DiskMiB, owner)
402}
403
404// shareExternals makes the read-only disk of the host's external tools available
405// to Firecracker; "" means the sandbox has none. A disk that cannot be built
406// does not stop the job: it starts without the tools and the failure is logged.
407func (s *vmSandbox) shareExternals(ctx context.Context) string {
408 d := s.d
409 set := d.cfg.ExternalsOf()
410 if len(set) == 0 {
411 return ""
412 }
413 log := zerowrap.FromCtx(ctx)
414 d.extMu.Lock()
415 defer d.extMu.Unlock()
416 path, err := ensureExternalsDisk(ctx, d.opt.Run, set, d.cfg.StateDir)
417 if err == nil && path != "" {
418 path, err = s.lay.share(path, "externals.ext4")
419 }
420 if err != nil {
421 log.Error().Err(err).Str("vm", s.id).Msg("external tools (Node.js) not attached to this job")
422 return ""
423 }
424 return path
425}
426
427// bootVsock listens on the host side of the vsock stream: Firecracker connects
428// to <uds_path>_<port> when the guest connects to the host CID.
429func (s *vmSandbox) bootVsock() (<-chan acceptResult, error) {
430 path := s.lay.host(udsName) + fmt.Sprintf("_%d", sandbox.VsockPort)
431 ln, err := net.Listen("unix", path)
432 if err != nil {
433 return nil, fmt.Errorf("listen vsock socket: %w", err)
434 }
435 s.listener = ln
436 if s.lay.jailed {
437 if err := os.Lchown(path, s.lay.uid, s.lay.uid); err != nil {
438 return nil, err
439 }
440 }
441 accepted := make(chan acceptResult, 1)
442 go func() { c, err := ln.Accept(); accepted <- acceptResult{c, err} }()
443 return accepted, nil
444}
445
446// cacheSocket is the host side of the cache streams of the VM: Firecracker
447// connects to <uds_path>_<CacheVsockPort> each time the guest relay connects to
448// the host on that vsock port. The path is inside this VM's own directory (its
449// chroot with the jailer), so no other VM reaches it.
450func (s *vmSandbox) cacheSocket() string {
451 return s.lay.host(udsName) + fmt.Sprintf("_%d", sandbox.CacheVsockPort)
452}
453
454// bootCache serves the cache on the VM's cache socket, owned by the jailed uid
455// as the executor's.
456func (s *vmSandbox) bootCache(spec sandbox.JobSpec) error {
457 if s.d.cfg.Cache == nil {
458 return nil
459 }
460 err := s.d.cfg.ServeCache(s.id, s.cacheSocket(), spec.JobID, s.d.cfg.NetMbit, func(p string) error {
461 if s.lay.jailed {
462 return os.Lchown(p, s.lay.uid, s.lay.uid)
463 }
464 return nil
465 })
466 if err != nil {
467 return fmt.Errorf("serve the cache socket: %w", err)
468 }
469 s.cacheOn = true
470 return nil
471}
472
473const udsName = "v.sock"
474
475// maxSockPath is the longest Unix socket path Linux accepts (108 with the NUL).
476const maxSockPath = 107
477
478// socketPaths lists the host paths of the unix sockets of a VM: the API socket
479// and the vsock listener (the guest's UDS path plus "_<port>"). With the jailer
480// they are deep in the chroot; without it they sit in <state>/vm/<id>.
481func socketPaths(stateDir, execName string, jailed bool) []string {
482 lay := newLayout(stateDir, execName, "er00000000", jailed, 0)
483 api := lay.host("api.sock")
484 if jailed {
485 api = lay.host("run/firecracker.socket")
486 }
487 return []string{api, lay.host(udsName) + fmt.Sprintf("_%d", sandbox.VsockPort), lay.host(udsName) + fmt.Sprintf("_%d", sandbox.CacheVsockPort)}
488}
489
490// checkSocketPaths refuses a state directory that makes a VM socket path longer
491// than a unix socket address.
492func checkSocketPaths(stateDir, execName string, jailed bool) error {
493 for _, p := range socketPaths(stateDir, execName, jailed) {
494 if len(p) > maxSockPath {
495 return fmt.Errorf("socket path %q is %d bytes, over the %d-byte limit of a Unix socket: use a shorter %s", p, len(p), maxSockPath, sandbox.EnvStateDir)
496 }
497 }
498 return nil
499}
500
501// ensureJailBase creates the jailer's chroot base and refuses one that others
502// could use to redirect the jailer: anything but a real directory (a symlink is
503// not followed), owned by neither root nor the runner's own user, or writable by
504// group or others. An existing directory is checked, never chmod-ed.
505func ensureJailBase(dir string) error {
506 if err := os.MkdirAll(dir, 0o700); err != nil {
507 return fmt.Errorf("jailer chroot base: %w", err)
508 }
509 fi, err := os.Lstat(dir)
510 if err != nil {
511 return fmt.Errorf("jailer chroot base: %w", err)
512 }
513 return checkJailBaseInfo(dir, fi, os.Geteuid())
514}
515
516func checkJailBaseInfo(dir string, fi fs.FileInfo, euid int) error {
517 if !fi.IsDir() {
518 return fmt.Errorf("jailer chroot base %s is not a directory (%v): remove it", dir, fi.Mode().Type())
519 }
520 if fi.Mode().Perm()&0o022 != 0 {
521 return fmt.Errorf("jailer chroot base %s is writable by group or others (mode %04o): chmod go-w it", dir, fi.Mode().Perm())
522 }
523 st, ok := fi.Sys().(*syscall.Stat_t)
524 if !ok {
525 return fmt.Errorf("jailer chroot base %s: cannot read its owner", dir)
526 }
527 // The jailer needs root, so the runner is root in production; accepting its own
528 // user as well keeps the driver testable without root.
529 if int(st.Uid) != 0 && int(st.Uid) != euid {
530 return fmt.Errorf("jailer chroot base %s is owned by uid %d, not root: chown it", dir, st.Uid)
531 }
532 return nil
533}
534
535func (s *vmSandbox) vmConfig(spec sandbox.JobSpec, netw netpolicy.Net, f vmFiles) vmConfig {
536 cfg := s.d.cfg
537 // The order decides the guest device names (vda, vdb, ...): new drives go last.
538 drives := []drive{
539 {DriveID: "rootfs", PathOnHost: f.execDisk, IsRootDevice: true, IsReadOnly: true},
540 {DriveID: "base", PathOnHost: f.baseDisk, IsReadOnly: true},
541 {DriveID: "scratch", PathOnHost: s.lay.fc("scratch.ext4"), RateLimiter: diskLimiter(cfg.DiskBandwidthMiB, cfg.DiskIOPS)},
542 }
543 if f.externals != "" {
544 drives = append(drives, drive{DriveID: "externals", PathOnHost: f.externals, IsReadOnly: true})
545 }
546 return vmConfig{
547 Machine: machineConfig{VcpuCount: spec.CPUs, MemSizeMiB: spec.MemoryMiB},
548 Boot: bootSource{KernelImagePath: f.kernel, BootArgs: bootArgs(netw.Guest, netw.Gateway, time.Now().Unix(), f.externals != "")},
549 Drives: drives,
550 Net: networkInterface{
551 IfaceID: "eth0", HostDevName: s.tap, GuestMAC: macFor(s.slot),
552 RxRateLimiter: netLimiter(cfg.NetMbit), TxRateLimiter: netLimiter(cfg.NetMbit),
553 },
554 Vsock: vsockDevice{GuestCID: 3, UDSPath: s.lay.fc(udsName)},
555 }
556}
557
558// waitExecutor waits for the guest's executor to connect. Only ctx (the caller's
559// context, or Stop through the boot gate) and the boot timeout end the wait: a
560// boot cannot be aborted any other way.
561func (s *vmSandbox) waitExecutor(ctx context.Context, accepted <-chan acceptResult) (sandbox.Conn, error) {
562 timer := time.NewTimer(s.d.opt.BootTimeout)
563 defer timer.Stop()
564 select {
565 case r := <-accepted:
566 if r.err != nil {
567 return nil, fmt.Errorf("accept executor connection: %w", r.err)
568 }
569 s.conn = r.c
570 return r.c, nil
571 case <-s.exited:
572 return nil, errors.New("firecracker exited before the executor connected")
573 case <-timer.C:
574 return nil, fmt.Errorf("executor did not connect within %s", s.d.opt.BootTimeout)
575 case <-ctx.Done():
576 return nil, context.Cause(ctx)
577 }
578}
579
580type acceptResult struct {
581 c net.Conn
582 err error
583}
584
585// jailerArgs builds the jailer command line (after the binary name).
586//
587// - cgroup v2 with memory.max (guest memory + VMM overhead) and pids.max (a
588// VMM needs a handful of threads per vCPU): a runaway or compromised VMM
589// cannot take the host's memory or fork-bomb it;
590// - a new PID namespace: the VMM cannot see or signal host processes;
591// - no-file: the descriptor budget of the VMM.
592func jailerArgs(jailer, fc, id string, uid int, chrootBase string, spec sandbox.JobSpec, extra []string) []string {
593 memBytes := (int64(spec.MemoryMiB) + vmmOverheadMiB) << 20
594 pids := 32 + 8*spec.CPUs
595 args := []string{
596 "--id", id, "--exec-file", fc,
597 "--uid", strconv.Itoa(uid), "--gid", strconv.Itoa(uid),
598 "--chroot-base-dir", chrootBase,
599 "--cgroup-version", "2",
600 "--cgroup", "memory.max=" + strconv.FormatInt(memBytes, 10),
601 "--cgroup", "pids.max=" + strconv.Itoa(pids),
602 "--new-pid-ns",
603 "--resource-limit", "no-file=4096",
604 "--",
605 }
606 return append(args, extra...)
607}
608
609func (s *vmSandbox) startVMM(spec sandbox.JobSpec, apiSock string) error {
610 d := s.d
611 var cmd *exec.Cmd
612 if s.lay.jailed {
613 args := jailerArgs(d.jailer, d.fcPath, s.id, s.uid(), jailBase(d.cfg.StateDir), spec, d.opt.FirecrackerArgs)
614 cmd = exec.Command(d.jailer, args...)
615 } else {
616 args := append([]string{"--id", s.id, "--api-sock", apiSock}, d.opt.FirecrackerArgs...)
617 cmd = exec.Command(d.fcPath, args...)
618 }
619 s.console = newRing(16 << 10)
620 cmd.Stdout, cmd.Stderr = s.console, s.console
621 cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
622 if err := cmd.Start(); err != nil {
623 return fmt.Errorf("start firecracker: %w", err)
624 }
625 s.cmd = cmd
626 s.exited = make(chan struct{})
627 jailed := s.lay.jailed
628 go func() {
629 _ = cmd.Wait()
630 // With --new-pid-ns the jailer may exit as soon as it has started the
631 // VMM in its namespace: the VMM is then gone only when its own pid is.
632 if jailed {
633 s.awaitVMMGone()
634 }
635 close(s.exited)
636 }()
637 s.rec.PID = cmd.Process.Pid
638 return s.d.store.save(s.rec)
639}
640
641// awaitVMMGone blocks until no process of this VM's id is left.
642func (s *vmSandbox) awaitVMMGone() {
643 for {
644 alive := false
645 for pid, id := range vmmProcs(s.d.opt.ProcDir, s.d.scope()) {
646 if id == s.id && pidAlive(pid) {
647 alive = true
648 }
649 }
650 if !alive {
651 return
652 }
653 time.Sleep(50 * time.Millisecond)
654 }
655}
656
657func (s *vmSandbox) waitAPI(ctx context.Context, api *apiClient, sock string) error {
658 deadline := time.Now().Add(10 * time.Second)
659 for {
660 if err := api.ready(ctx); err == nil {
661 return nil
662 }
663 select {
664 case <-s.exited:
665 return errors.New("firecracker exited before its API was ready")
666 case <-ctx.Done():
667 return context.Cause(ctx)
668 case <-time.After(25 * time.Millisecond):
669 }
670 if time.Now().After(deadline) {
671 return fmt.Errorf("firecracker API socket %s not ready after 10s", sock)
672 }
673 }
674}
675
676// Stop implements sandbox.Sandbox.
677func (s *vmSandbox) Stop(ctx context.Context) error {
678 // Abort a boot in progress first: Start holds s.mu for its whole duration.
679 s.gate.Abort()
680 s.mu.Lock()
681 defer s.mu.Unlock()
682 if s.stopped {
683 return s.leakErr
684 }
685 if !s.started {
686 s.releaseLocked()
687 return nil
688 }
689 return s.cleanupLocked(context.WithoutCancel(ctx))
690}
691
692func (s *vmSandbox) releaseLocked() {
693 s.stopped = true
694 s.d.release(s.slot)
695}
696
697// vmmPIDs returns every host process of this VM: the launched command and
698// whatever the /proc scan finds with this id.
699func (s *vmSandbox) vmmPIDs() []int {
700 set := map[int]bool{}
701 if s.cmd != nil && s.cmd.Process != nil {
702 set[s.cmd.Process.Pid] = true
703 }
704 for pid, id := range vmmProcs(s.d.opt.ProcDir, s.d.scope()) {
705 if id == s.id {
706 set[pid] = true
707 }
708 }
709 var out []int
710 for pid := range set {
711 out = append(out, pid)
712 }
713 return out
714}
715
716// stillMine is the identity check handed to the kill: pid still carries this
717// VM's id and passes the scope (command line and root).
718func (s *vmSandbox) stillMine(pid int) func() bool {
719 return func() bool { return cmdlineID(s.d.opt.ProcDir, pid, s.d.scope()) == s.id }
720}
721
722// cleanupLocked removes everything the VM created, in dependency order. Steps
723// that need the VMM to be dead (network device, firewall rules, files, cgroup)
724// are skipped when it could not be killed: removing the TAP or the policy under
725// a VM that may still run would break the isolation it relies on, so the
726// sandbox is reported as leaked (sandbox.ErrLeaked), its slot stays taken, and
727// Reap finishes the job.
728func (s *vmSandbox) cleanupLocked(ctx context.Context) error {
729 log := zerowrap.FromCtx(ctx)
730 d := s.d
731 var errs []error
732 if s.conn != nil {
733 _ = s.conn.Close()
734 s.conn = nil
735 }
736 if s.listener != nil {
737 _ = s.listener.Close()
738 s.listener = nil
739 }
740 if s.cacheOn {
741 d.cfg.StopCache(s.id)
742 s.cacheOn = false
743 }
744 dead := true
745 // No kill(-pgid) of the launched command: once it has been waited for, its
746 // pid (and so its group id) may belong to anyone. Every process of this VM is
747 // found by the scan (id, markers and scope) and killed through a pidfd.
748 for _, pid := range s.vmmPIDs() {
749 if !d.opt.kill(pid, s.stillMine(pid), 5*time.Second) {
750 dead = false
751 errs = append(errs, fmt.Errorf("vmm process %d did not exit", pid))
752 }
753 }
754 if dead && s.exited != nil {
755 select {
756 case <-s.exited:
757 case <-time.After(5 * time.Second):
758 dead = false
759 errs = append(errs, errors.New("vmm exit was not observed"))
760 }
761 }
762 if dead {
763 if s.attached {
764 if err := d.net.Detach(ctx, s.tap, s.gateway); err != nil {
765 errs = append(errs, err)
766 }
767 if err := deleteTap(ctx, d.opt.Run, s.tap); err != nil {
768 errs = append(errs, err)
769 } else {
770 s.attached = false
771 }
772 }
773 if s.lay.dir != "" {
774 if err := removeAll(s.lay.dir); err != nil {
775 errs = append(errs, err)
776 }
777 }
778 if s.lay.jailed {
779 if err := removeCgroup(d.opt.CgroupRoot, filepath.Base(d.fcPath), s.id); err != nil {
780 errs = append(errs, err)
781 }
782 }
783 if len(errs) == 0 {
784 if err := d.store.remove(s.id); err != nil {
785 errs = append(errs, err)
786 }
787 }
788 }
789 if len(errs) == 0 {
790 s.releaseLocked()
791 return nil
792 }
793 s.stopped = true
794 d.markLeaked(s.slot)
795 s.leakErr = fmt.Errorf("%w: vm %s: %w", sandbox.ErrLeaked, s.id, errors.Join(errs...))
796 log.Error().Err(s.leakErr).Str("vm", s.id).Msg("micro-VM cleanup incomplete; call Reap to finish it")
797 return s.leakErr
798}
799
800// removeAll is os.RemoveAll that also reports a directory that is still there.
801func removeAll(dir string) error {
802 if err := os.RemoveAll(dir); err != nil {
803 return err
804 }
805 if _, err := os.Lstat(dir); err == nil {
806 return fmt.Errorf("%s still exists after removal", dir)
807 }
808 return nil
809}
810
811// removeCgroup removes the cgroup the jailer created for a VM
812// (<root>/<parent>/<id>, parent = the exec file name). The jailer never removes
813// it. A missing cgroup is fine; a non-empty one fails (rmdir semantics).
814func removeCgroup(root, parent, id string) error {
815 err := os.Remove(filepath.Join(root, parent, id))
816 if err == nil || errors.Is(err, os.ErrNotExist) {
817 return nil
818 }
819 return fmt.Errorf("remove cgroup %s: %w", filepath.Join(root, parent, id), err)
820}