1// SPDX-License-Identifier: MIT
2
3// Package vm is the micro-VM sandbox: one Firecracker VM per job.
4//
5// Per VM, on the host: a jailer chroot (or, without jailer, a plain directory)
6// holding the per-job sparse scratch disk and the vsock socket; a TAP device
7// with a /30; rules in the dedicated nftables table; the DNS forwarder bound on
8// the TAP's host address. In the guest: three virtio disks (read-only runner
9// executable as the boot root, read-only shared base image, per-job scratch)
10// and a vsock device. Everything is traced in a state-directory record before it
11// is created, so that Reap can remove it after a crash.
12package vm
13
14import (
15 "context"
16 "errors"
17 "fmt"
18 "io/fs"
19 "maps"
20 "net"
21 "net/netip"
22 "os"
23 "os/exec"
24 "path/filepath"
25 "strconv"
26 "sync"
27 "syscall"
28 "time"
29
30 "github.com/bnema/zerowrap"
31
32 "bnema.dev/ermite-runner/internal/cmdrun"
33 "bnema.dev/ermite-runner/internal/netpolicy"
34 "bnema.dev/ermite-runner/internal/sandbox"
35 "bnema.dev/ermite-runner/internal/sandbox/disk"
36 "bnema.dev/ermite-runner/internal/sandbox/netsetup"
37 "bnema.dev/ermite-runner/internal/tcprelay"
38)
39
40// Options are the testable knobs of the driver.
41type Options struct {
42 Run cmdrun.Runner
43 // ProcDir is the /proc used to find orphan VMM processes.
44 ProcDir string
45 // CgroupRoot is the cgroup v2 mount, where the jailer creates one cgroup per
46 // VM (default /sys/fs/cgroup).
47 CgroupRoot string
48 // BootTimeout bounds boot + init + executor connection (default 90 s).
49 BootTimeout time.Duration
50 // Net overrides the network plumbing (tests).
51 Net netsetup.Plumbing
52 // FirecrackerArgs are extra arguments for Firecracker (tests use them to
53 // select the fake).
54 FirecrackerArgs []string
55 // FreeSpace overrides statfs for the free-space warning (tests).
56 FreeSpace func(string) (int64, error)
57 // kill overrides killAndWait (tests simulate a VMM that cannot be killed).
58 kill func(pid int, still func() bool, timeout time.Duration) bool
59}
60
61// vmmOverheadMiB is what the Firecracker process itself may use on top of the
62// guest memory (device emulation, the API server, the seccomp filter, stacks):
63// it is added to memory.max so that the guest, not the VMM, is what gets limited.
64const vmmOverheadMiB = 256
65
66// Driver implements sandbox.Driver for micro-VMs.
67type Driver struct {
68 cfg sandbox.Config
69 opt Options
70 fcPath string
71 jailer string
72 execDisk string
73 // extMu serialises the building of the externals disk with the linking of it
74 // into a sandbox (see shareExternals).
75 extMu sync.Mutex
76 store store
77 net netsetup.Plumbing
78 lock *sandbox.HostLock
79
80 mu sync.Mutex
81 slots *sandbox.Slots
82}
83
84// New builds the driver: takes the host-wide lock, resolves binaries, builds the
85// exec disk and enables IPv4 forwarding (a host-wide sysctl that stays on: see
86// the README). Run Reap before New sandboxes.
87func New(ctx context.Context, cfg sandbox.Config, opt Options) (*Driver, error) {
88 if opt.Run == nil {
89 opt.Run = cmdrun.Exec{}
90 }
91 if opt.ProcDir == "" {
92 opt.ProcDir = "/proc"
93 }
94 if opt.CgroupRoot == "" {
95 opt.CgroupRoot = "/sys/fs/cgroup"
96 }
97 if opt.BootTimeout == 0 {
98 opt.BootTimeout = 90 * time.Second
99 }
100 if opt.kill == nil {
101 opt.kill = killAndWait
102 }
103 d := &Driver{cfg: cfg, opt: opt, slots: sandbox.NewSlots(cfg.Capacity), store: store{dir: filepath.Join(cfg.StateDir, "vms")}}
104
105 var err error
106 if d.fcPath, err = resolveBin(cfg.Firecracker, "firecracker"); err != nil {
107 return nil, err
108 }
109 if !cfg.JailerOff {
110 if d.jailer, err = resolveBin(cfg.Jailer, "jailer"); err != nil {
111 return nil, err
112 }
113 }
114 exe, err := cfg.ExecutablePath()
115 if err != nil {
116 return nil, err
117 }
118 if err := sandbox.CheckStatic(exe); err != nil {
119 return nil, err
120 }
121 if err := os.MkdirAll(cfg.StateDir, 0o755); err != nil {
122 return nil, fmt.Errorf("state directory: %w", err)
123 }
124 if d.lock, err = sandbox.AcquireHostLock(cfg.LockFile); err != nil {
125 return nil, err
126 }
127 fail := func(err error) (*Driver, error) { _ = d.Close(); return nil, err }
128
129 // A path over sun_path fails later as a 10 s "API not ready" (or a vsock
130 // that never connects): refuse at start, in both modes.
131 if err := checkSocketPaths(cfg.StateDir, filepath.Base(d.fcPath), !cfg.JailerOff); err != nil {
132 return fail(err)
133 }
134 if !cfg.JailerOff {
135 // Jailer 1.17 canonicalizes --chroot-base-dir and fails when it does not
136 // exist ("Failed to canonicalize path"); older releases created it.
137 if err := ensureJailBase(jailBase(cfg.StateDir)); err != nil {
138 return fail(err)
139 }
140 for _, p := range []string{cfg.KernelPath(), cfg.BaseImagePath()} {
141 if err := checkWorldReadable(p); err != nil {
142 return fail(err)
143 }
144 }
145 }
146 sandbox.WarnIfLowSpace(ctx, cfg.StateDir, cfg.Capacity, cfg.JobDiskMiB, opt.FreeSpace)
147 if err := disk.CheckMkfs(ctx, opt.Run); err != nil {
148 return fail(err)
149 }
150 if d.execDisk, err = ensureExecDisk(ctx, opt.Run, exe, cfg.StateDir); err != nil {
151 return fail(err)
152 }
153 d.net = opt.Net
154 if d.net == nil {
155 n := netsetup.New(opt.Run, cfg.DNSPort, "")
156 n.DNS.Override = cfg.GuestForge.DNSName()
157 if t := cfg.GuestForge.RelayTarget(); t != "" {
158 n.Relay = tcprelay.New(t)
159 }
160 d.net = n
161 }
162 if cfg.GuestForge != nil {
163 l := zerowrap.FromCtx(ctx)
164 l.Warn().Msg(cfg.GuestForge.Warning())
165 }
166 if err := d.net.EnableForwarding(); err != nil {
167 return fail(err)
168 }
169 return d, nil
170}
171
172func resolveBin(configured, name string) (string, error) {
173 p := configured
174 if p == "" {
175 var err error
176 if p, err = exec.LookPath(name); err != nil {
177 return "", fmt.Errorf("%s not found in PATH (set ERMITE_RUNNER_%s)", name, map[string]string{"firecracker": "FIRECRACKER", "jailer": "JAILER"}[name])
178 }
179 }
180 abs, err := filepath.Abs(p)
181 if err != nil {
182 return "", err
183 }
184 if _, err := os.Stat(abs); err != nil {
185 return "", fmt.Errorf("%s: %w", name, err)
186 }
187 return abs, nil
188}
189
190// Mode implements sandbox.Driver.
191func (d *Driver) Mode() sandbox.Mode { return sandbox.VM }
192
193// Isolation implements sandbox.Driver.
194func (d *Driver) Isolation() sandbox.Isolation { return sandbox.IsolationVM }
195
196// Close releases the host lock and the DNS listeners.
197func (d *Driver) Close() error {
198 if d.net != nil {
199 d.net.Close()
200 }
201 return d.lock.Release()
202}
203
204// New implements sandbox.Driver.
205func (d *Driver) New() (sandbox.Sandbox, error) {
206 id, err := sandbox.NewID()
207 if err != nil {
208 return nil, err
209 }
210 d.mu.Lock()
211 defer d.mu.Unlock()
212 slot, err := d.slots.Acquire()
213 if err != nil {
214 return nil, err
215 }
216 return &vmSandbox{d: d, slot: slot, id: id}, nil
217}
218
219// scope narrows the /proc scan to this driver's own jail base.
220func (d *Driver) scope() scope { return newScope(d.cfg.StateDir) }
221
222func (d *Driver) release(slot int) {
223 d.mu.Lock()
224 d.slots.Release(slot)
225 d.mu.Unlock()
226}
227
228func (d *Driver) markLeaked(slot int) {
229 d.mu.Lock()
230 d.slots.MarkLeaked(slot)
231 d.mu.Unlock()
232}
233
234// vmSandbox is one VM, single use.
235type vmSandbox struct {
236 d *Driver
237 slot int
238 id string
239 gate sandbox.BootGate
240 timings map[string]int64
241
242 mu sync.Mutex
243 started bool
244 stopped bool
245 // leakErr is the error of a cleanup that left something behind; later Stop
246 // calls return it again.
247 leakErr error
248
249 rec record
250 lay layout
251 cmd *exec.Cmd
252 exited chan struct{}
253 console *ring
254 listener net.Listener
255 cacheOn bool
256 conn net.Conn
257 tap string
258 gateway netip.Addr
259 attached bool
260}
261
262// StartTimings returns a copy of the phase durations of a successful Start.
263// It is valid once Start has returned, which writes them.
264func (s *vmSandbox) StartTimings() map[string]int64 { return maps.Clone(s.timings) }
265
266// ID implements sandbox.Sandbox.
267func (s *vmSandbox) ID() string { return s.id }
268
269func (s *vmSandbox) uid() int { return s.d.cfg.UIDBase + s.slot }
270
271// Start implements sandbox.Sandbox.
272func (s *vmSandbox) Start(ctx context.Context, spec sandbox.JobSpec) (sandbox.Conn, error) {
273 // A Stop from another goroutine aborts the boot through the gate.
274 bootCtx, err := s.gate.Begin(ctx)
275 if err != nil {
276 return nil, fmt.Errorf("vm sandbox: %w", err)
277 }
278 defer s.gate.End()
279
280 s.mu.Lock()
281 defer s.mu.Unlock()
282 if s.started || s.stopped {
283 return nil, errors.New("vm sandbox: Start called twice or after Stop")
284 }
285 s.started = true
286 log := zerowrap.FromCtx(ctx)
287 d := s.d
288 spec = d.cfg.Resolve(spec)
289 if d.cfg.GuestForge != nil {
290 // At every job start, as for privileged containers.
291 log.Warn().Str("job", spec.JobID).Str("vm", s.id).Msg(d.cfg.GuestForge.Warning())
292 }
293
294 jailed := !d.cfg.JailerOff
295 runUID := s.uid()
296 if !jailed {
297 runUID = os.Getuid()
298 }
299 s.lay = newLayout(d.cfg.StateDir, filepath.Base(d.fcPath), s.id, jailed, runUID)
300 s.rec = record{ID: s.id, Slot: s.slot, Job: spec.JobID, Dir: s.lay.dir}
301 // The record exists before the first resource, so any crash from here on is
302 // reapable.
303 if err := d.store.save(s.rec); err != nil {
304 s.releaseLocked()
305 return nil, fmt.Errorf("write state record: %w", err)
306 }
307
308 start := time.Now()
309 s.timings = make(map[string]int64)
310 conn, err := s.boot(bootCtx, spec)
311 if err != nil {
312 s.timings = nil
313 console := ""
314 if s.console != nil {
315 console = s.console.String()
316 }
317 cerr := s.cleanupLocked(context.WithoutCancel(ctx))
318 if console != "" {
319 err = fmt.Errorf("%w\nVMM console tail:\n%s", err, console)
320 }
321 return nil, errors.Join(err, cerr)
322 }
323 fields := map[string]any{"total_ms": time.Since(start).Milliseconds()}
324 for k, v := range s.timings {
325 fields[k] = v
326 }
327 log.Info().Str("vm", s.id).Int("slot", s.slot).Str("job", spec.JobID).Fields(fields).Msg("micro-VM running")
328 return conn, nil
329}
330
331// boot runs the stages in order; each stage records what it created so that
332// cleanupLocked can undo exactly that.
333func (s *vmSandbox) boot(ctx context.Context, spec sandbox.JobSpec) (sandbox.Conn, error) {
334 netw, err := netpolicy.Subnet(s.d.cfg.Network, s.slot)
335 if err != nil {
336 return nil, err
337 }
338 start := time.Now()
339 if err := s.bootNetwork(ctx, netw); err != nil {
340 return nil, err
341 }
342 s.timings["net_ms"] = time.Since(start).Milliseconds() - s.timings["tap_ms"]
343 start = time.Now()
344 api, err := s.bootVMM(ctx, spec)
345 if err != nil {
346 return nil, err
347 }
348 s.timings["vmm_ms"] = time.Since(start).Milliseconds()
349 start = time.Now()
350 files, err := s.bootFiles(ctx, spec)
351 if err != nil {
352 return nil, err
353 }
354 s.timings["files_ms"] = time.Since(start).Milliseconds() - s.timings["scratch_ms"]
355 start = time.Now()
356 accepted, err := s.bootVsock()
357 if err != nil {
358 return nil, err
359 }
360 if err := s.bootCache(spec); err != nil {
361 return nil, err
362 }
363 s.timings["vsock_ms"] = time.Since(start).Milliseconds()
364 start = time.Now()
365 if err := api.configure(ctx, s.vmConfig(spec, netw, files)); err != nil {
366 return nil, err
367 }
368 s.timings["configure_ms"] = time.Since(start).Milliseconds()
369 start = time.Now()
370 conn, err := s.waitExecutor(ctx, accepted)
371 if err == nil {
372 s.timings["connect_ms"] = time.Since(start).Milliseconds()
373 }
374 return conn, err
375}
376
377// bootNetwork creates the TAP (it must exist when Firecracker opens it) and
378// installs the firewall policy and the DNS listener.
379func (s *vmSandbox) bootNetwork(ctx context.Context, netw netpolicy.Net) error {
380 d := s.d
381 s.tap, s.gateway = tapName(s.slot), netw.Gateway
382 s.rec.Tap, s.rec.Gate = s.tap, s.gateway.String()
383 if err := d.store.save(s.rec); err != nil {
384 return err
385 }
386 start := time.Now()
387 if err := createTap(ctx, d.opt.Run, s.tap, s.lay.uid, s.gateway); err != nil {
388 return err
389 }
390 s.timings["tap_ms"] = time.Since(start).Milliseconds()
391 s.attached = true // from here Detach is idempotent and always safe
392 return d.net.Attach(ctx, netpolicy.Guest{Iface: s.tap, Gateway: netw.Gateway, Addr: netw.Guest, ForgePort: d.cfg.GuestForge.TCPPort()})
393}
394
395// bootVMM starts the jailer (or Firecracker) and waits for its API.
396func (s *vmSandbox) bootVMM(ctx context.Context, spec sandbox.JobSpec) (*apiClient, error) {
397 apiSock := s.lay.host("api.sock")
398 if s.lay.jailed {
399 apiSock = s.lay.host("run/firecracker.socket")
400 } else if err := os.MkdirAll(s.lay.root, 0o700); err != nil {
401 return nil, err
402 }
403 if err := s.startVMM(spec, apiSock); err != nil {
404 return nil, err
405 }
406 api := newAPIClient(apiSock)
407 if err := s.waitAPI(ctx, api, apiSock); err != nil {
408 return nil, err
409 }
410 return api, nil
411}
412
413// vmFiles are the paths Firecracker is given.
414type vmFiles struct{ execDisk, baseDisk, kernel, externals string }
415
416// bootFiles makes the shared files available and creates the scratch disk.
417func (s *vmSandbox) bootFiles(ctx context.Context, spec sandbox.JobSpec) (vmFiles, error) {
418 d := s.d
419 var f vmFiles
420 var err error
421 if f.execDisk, err = s.lay.share(d.execDisk, "exec.ext4"); err != nil {
422 return f, err
423 }
424 if f.baseDisk, err = s.lay.share(d.cfg.BaseImagePath(), "base.ext4"); err != nil {
425 return f, err
426 }
427 if f.kernel, err = s.lay.share(d.cfg.KernelPath(), "kernel"); err != nil {
428 return f, err
429 }
430 f.externals = s.shareExternals(ctx)
431 owner := -1
432 if s.lay.jailed {
433 owner = s.lay.uid
434 }
435 start := time.Now()
436 err = disk.CreateScratch(ctx, d.opt.Run, s.lay.host("scratch.ext4"), spec.DiskMiB, owner)
437 if err == nil {
438 s.timings["scratch_ms"] = time.Since(start).Milliseconds()
439 }
440 return f, err
441}
442
443// shareExternals makes the read-only disk of the host's external tools available
444// to Firecracker; "" means the sandbox has none. A disk that cannot be built
445// does not stop the job: it starts without the tools and the failure is logged.
446func (s *vmSandbox) shareExternals(ctx context.Context) string {
447 d := s.d
448 set := d.cfg.ExternalsOf()
449 if len(set) == 0 {
450 return ""
451 }
452 log := zerowrap.FromCtx(ctx)
453 d.extMu.Lock()
454 defer d.extMu.Unlock()
455 path, err := ensureExternalsDisk(ctx, d.opt.Run, set, d.cfg.StateDir)
456 if err == nil && path != "" {
457 path, err = s.lay.share(path, "externals.ext4")
458 }
459 if err != nil {
460 log.Error().Err(err).Str("vm", s.id).Msg("external tools (Node.js) not attached to this job")
461 return ""
462 }
463 return path
464}
465
466// bootVsock listens on the host side of the vsock stream: Firecracker connects
467// to <uds_path>_<port> when the guest connects to the host CID.
468func (s *vmSandbox) bootVsock() (<-chan acceptResult, error) {
469 path := s.lay.host(udsName) + fmt.Sprintf("_%d", sandbox.VsockPort)
470 ln, err := net.Listen("unix", path)
471 if err != nil {
472 return nil, fmt.Errorf("listen vsock socket: %w", err)
473 }
474 s.listener = ln
475 if s.lay.jailed {
476 if err := os.Lchown(path, s.lay.uid, s.lay.uid); err != nil {
477 return nil, err
478 }
479 }
480 accepted := make(chan acceptResult, 1)
481 go func() { c, err := ln.Accept(); accepted <- acceptResult{c, err} }()
482 return accepted, nil
483}
484
485// cacheSocket is the host side of the cache streams of the VM: Firecracker
486// connects to <uds_path>_<CacheVsockPort> each time the guest relay connects to
487// the host on that vsock port. The path is inside this VM's own directory (its
488// chroot with the jailer), so no other VM reaches it.
489func (s *vmSandbox) cacheSocket() string {
490 return s.lay.host(udsName) + fmt.Sprintf("_%d", sandbox.CacheVsockPort)
491}
492
493// bootCache serves the cache on the VM's cache socket, owned by the jailed uid
494// as the executor's.
495func (s *vmSandbox) bootCache(spec sandbox.JobSpec) error {
496 if s.d.cfg.Cache == nil {
497 return nil
498 }
499 err := s.d.cfg.ServeCache(s.id, s.cacheSocket(), spec, s.d.cfg.NetMbit, func(p string) error {
500 if s.lay.jailed {
501 return os.Lchown(p, s.lay.uid, s.lay.uid)
502 }
503 return nil
504 })
505 if err != nil {
506 return fmt.Errorf("serve the cache socket: %w", err)
507 }
508 s.cacheOn = true
509 return nil
510}
511
512const udsName = "v.sock"
513
514// maxSockPath is the longest Unix socket path Linux accepts (108 with the NUL).
515const maxSockPath = 107
516
517// socketPaths lists the host paths of the unix sockets of a VM: the API socket
518// and the vsock listener (the guest's UDS path plus "_<port>"). With the jailer
519// they are deep in the chroot; without it they sit in <state>/vm/<id>.
520func socketPaths(stateDir, execName string, jailed bool) []string {
521 lay := newLayout(stateDir, execName, "er00000000", jailed, 0)
522 api := lay.host("api.sock")
523 if jailed {
524 api = lay.host("run/firecracker.socket")
525 }
526 return []string{api, lay.host(udsName) + fmt.Sprintf("_%d", sandbox.VsockPort), lay.host(udsName) + fmt.Sprintf("_%d", sandbox.CacheVsockPort)}
527}
528
529// checkSocketPaths refuses a state directory that makes a VM socket path longer
530// than a unix socket address.
531func checkSocketPaths(stateDir, execName string, jailed bool) error {
532 for _, p := range socketPaths(stateDir, execName, jailed) {
533 if len(p) > maxSockPath {
534 return fmt.Errorf("socket path %q is %d bytes, over the %d-byte limit of a Unix socket: use a shorter %s", p, len(p), maxSockPath, sandbox.EnvStateDir)
535 }
536 }
537 return nil
538}
539
540// ensureJailBase creates the jailer's chroot base and refuses one that others
541// could use to redirect the jailer: anything but a real directory (a symlink is
542// not followed), owned by neither root nor the runner's own user, or writable by
543// group or others. An existing directory is checked, never chmod-ed.
544func ensureJailBase(dir string) error {
545 if err := os.MkdirAll(dir, 0o700); err != nil {
546 return fmt.Errorf("jailer chroot base: %w", err)
547 }
548 fi, err := os.Lstat(dir)
549 if err != nil {
550 return fmt.Errorf("jailer chroot base: %w", err)
551 }
552 return checkJailBaseInfo(dir, fi, os.Geteuid())
553}
554
555func checkJailBaseInfo(dir string, fi fs.FileInfo, euid int) error {
556 if !fi.IsDir() {
557 return fmt.Errorf("jailer chroot base %s is not a directory (%v): remove it", dir, fi.Mode().Type())
558 }
559 if fi.Mode().Perm()&0o022 != 0 {
560 return fmt.Errorf("jailer chroot base %s is writable by group or others (mode %04o): chmod go-w it", dir, fi.Mode().Perm())
561 }
562 st, ok := fi.Sys().(*syscall.Stat_t)
563 if !ok {
564 return fmt.Errorf("jailer chroot base %s: cannot read its owner", dir)
565 }
566 // The jailer needs root, so the runner is root in production; accepting its own
567 // user as well keeps the driver testable without root.
568 if int(st.Uid) != 0 && int(st.Uid) != euid {
569 return fmt.Errorf("jailer chroot base %s is owned by uid %d, not root: chown it", dir, st.Uid)
570 }
571 return nil
572}
573
574func (s *vmSandbox) vmConfig(spec sandbox.JobSpec, netw netpolicy.Net, f vmFiles) vmConfig {
575 cfg := s.d.cfg
576 // The order decides the guest device names (vda, vdb, ...): new drives go last.
577 drives := []drive{
578 {DriveID: "rootfs", PathOnHost: f.execDisk, IsRootDevice: true, IsReadOnly: true},
579 {DriveID: "base", PathOnHost: f.baseDisk, IsReadOnly: true},
580 {DriveID: "scratch", PathOnHost: s.lay.fc("scratch.ext4"), RateLimiter: diskLimiter(cfg.DiskBandwidthMiB, cfg.DiskIOPS)},
581 }
582 if f.externals != "" {
583 drives = append(drives, drive{DriveID: "externals", PathOnHost: f.externals, IsReadOnly: true})
584 }
585 return vmConfig{
586 Machine: machineConfig{VcpuCount: spec.CPUs, MemSizeMiB: spec.MemoryMiB},
587 Boot: bootSource{KernelImagePath: f.kernel, BootArgs: bootArgs(netw.Guest, netw.Gateway, time.Now().Unix(), f.externals != "")},
588 Drives: drives,
589 Net: networkInterface{
590 IfaceID: "eth0", HostDevName: s.tap, GuestMAC: macFor(s.slot),
591 RxRateLimiter: netLimiter(cfg.NetMbit), TxRateLimiter: netLimiter(cfg.NetMbit),
592 },
593 Vsock: vsockDevice{GuestCID: 3, UDSPath: s.lay.fc(udsName)},
594 }
595}
596
597// waitExecutor waits for the guest's executor to connect. Only ctx (the caller's
598// context, or Stop through the boot gate) and the boot timeout end the wait: a
599// boot cannot be aborted any other way.
600func (s *vmSandbox) waitExecutor(ctx context.Context, accepted <-chan acceptResult) (sandbox.Conn, error) {
601 timer := time.NewTimer(s.d.opt.BootTimeout)
602 defer timer.Stop()
603 select {
604 case r := <-accepted:
605 if r.err != nil {
606 return nil, fmt.Errorf("accept executor connection: %w", r.err)
607 }
608 s.conn = r.c
609 return r.c, nil
610 case <-s.exited:
611 return nil, errors.New("firecracker exited before the executor connected")
612 case <-timer.C:
613 return nil, fmt.Errorf("executor did not connect within %s", s.d.opt.BootTimeout)
614 case <-ctx.Done():
615 return nil, context.Cause(ctx)
616 }
617}
618
619type acceptResult struct {
620 c net.Conn
621 err error
622}
623
624// jailerArgs builds the jailer command line (after the binary name).
625//
626// - cgroup v2 with memory.max (guest memory + VMM overhead) and pids.max (a
627// VMM needs a handful of threads per vCPU): a runaway or compromised VMM
628// cannot take the host's memory or fork-bomb it;
629// - a new PID namespace: the VMM cannot see or signal host processes;
630// - no-file: the descriptor budget of the VMM.
631func jailerArgs(jailer, fc, id string, uid int, chrootBase string, spec sandbox.JobSpec, extra []string) []string {
632 memBytes := (int64(spec.MemoryMiB) + vmmOverheadMiB) << 20
633 pids := 32 + 8*spec.CPUs
634 args := []string{
635 "--id", id, "--exec-file", fc,
636 "--uid", strconv.Itoa(uid), "--gid", strconv.Itoa(uid),
637 "--chroot-base-dir", chrootBase,
638 "--cgroup-version", "2",
639 "--cgroup", "memory.max=" + strconv.FormatInt(memBytes, 10),
640 "--cgroup", "pids.max=" + strconv.Itoa(pids),
641 "--new-pid-ns",
642 "--resource-limit", "no-file=4096",
643 "--",
644 }
645 return append(args, extra...)
646}
647
648func (s *vmSandbox) startVMM(spec sandbox.JobSpec, apiSock string) error {
649 d := s.d
650 var cmd *exec.Cmd
651 if s.lay.jailed {
652 args := jailerArgs(d.jailer, d.fcPath, s.id, s.uid(), jailBase(d.cfg.StateDir), spec, d.opt.FirecrackerArgs)
653 cmd = exec.Command(d.jailer, args...)
654 } else {
655 args := append([]string{"--id", s.id, "--api-sock", apiSock}, d.opt.FirecrackerArgs...)
656 cmd = exec.Command(d.fcPath, args...)
657 }
658 s.console = newRing(16 << 10)
659 cmd.Stdout, cmd.Stderr = s.console, s.console
660 cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
661 if err := cmd.Start(); err != nil {
662 return fmt.Errorf("start firecracker: %w", err)
663 }
664 s.cmd = cmd
665 s.exited = make(chan struct{})
666 jailed := s.lay.jailed
667 go func() {
668 _ = cmd.Wait()
669 // With --new-pid-ns the jailer may exit as soon as it has started the
670 // VMM in its namespace: the VMM is then gone only when its own pid is.
671 if jailed {
672 s.awaitVMMGone()
673 }
674 close(s.exited)
675 }()
676 s.rec.PID = cmd.Process.Pid
677 return s.d.store.save(s.rec)
678}
679
680// awaitVMMGone blocks until no process of this VM's id is left.
681func (s *vmSandbox) awaitVMMGone() {
682 for {
683 alive := false
684 for pid, id := range vmmProcs(s.d.opt.ProcDir, s.d.scope()) {
685 if id == s.id && pidAlive(pid) {
686 alive = true
687 }
688 }
689 if !alive {
690 return
691 }
692 time.Sleep(50 * time.Millisecond)
693 }
694}
695
696func (s *vmSandbox) waitAPI(ctx context.Context, api *apiClient, sock string) error {
697 deadline := time.Now().Add(10 * time.Second)
698 for {
699 if err := api.ready(ctx); err == nil {
700 return nil
701 }
702 select {
703 case <-s.exited:
704 return errors.New("firecracker exited before its API was ready")
705 case <-ctx.Done():
706 return context.Cause(ctx)
707 case <-time.After(25 * time.Millisecond):
708 }
709 if time.Now().After(deadline) {
710 return fmt.Errorf("firecracker API socket %s not ready after 10s", sock)
711 }
712 }
713}
714
715// Stop implements sandbox.Sandbox.
716func (s *vmSandbox) Stop(ctx context.Context) error {
717 // Abort a boot in progress first: Start holds s.mu for its whole duration.
718 s.gate.Abort()
719 s.mu.Lock()
720 defer s.mu.Unlock()
721 if s.stopped {
722 return s.leakErr
723 }
724 if !s.started {
725 s.releaseLocked()
726 return nil
727 }
728 return s.cleanupLocked(context.WithoutCancel(ctx))
729}
730
731func (s *vmSandbox) releaseLocked() {
732 s.stopped = true
733 s.d.release(s.slot)
734}
735
736// vmmPIDs returns every host process of this VM: the launched command and
737// whatever the /proc scan finds with this id.
738func (s *vmSandbox) vmmPIDs() []int {
739 set := map[int]bool{}
740 if s.cmd != nil && s.cmd.Process != nil {
741 set[s.cmd.Process.Pid] = true
742 }
743 for pid, id := range vmmProcs(s.d.opt.ProcDir, s.d.scope()) {
744 if id == s.id {
745 set[pid] = true
746 }
747 }
748 var out []int
749 for pid := range set {
750 out = append(out, pid)
751 }
752 return out
753}
754
755// stillMine is the identity check handed to the kill: pid still carries this
756// VM's id and passes the scope (command line and root).
757func (s *vmSandbox) stillMine(pid int) func() bool {
758 return func() bool { return cmdlineID(s.d.opt.ProcDir, pid, s.d.scope()) == s.id }
759}
760
761// cleanupLocked removes everything the VM created, in dependency order. Steps
762// that need the VMM to be dead (network device, firewall rules, files, cgroup)
763// are skipped when it could not be killed: removing the TAP or the policy under
764// a VM that may still run would break the isolation it relies on, so the
765// sandbox is reported as leaked (sandbox.ErrLeaked), its slot stays taken, and
766// Reap finishes the job.
767func (s *vmSandbox) cleanupLocked(ctx context.Context) error {
768 log := zerowrap.FromCtx(ctx)
769 d := s.d
770 var errs []error
771 if s.conn != nil {
772 _ = s.conn.Close()
773 s.conn = nil
774 }
775 if s.listener != nil {
776 _ = s.listener.Close()
777 s.listener = nil
778 }
779 if s.cacheOn {
780 d.cfg.StopCache(s.id)
781 s.cacheOn = false
782 }
783 dead := true
784 // No kill(-pgid) of the launched command: once it has been waited for, its
785 // pid (and so its group id) may belong to anyone. Every process of this VM is
786 // found by the scan (id, markers and scope) and killed through a pidfd.
787 for _, pid := range s.vmmPIDs() {
788 if !d.opt.kill(pid, s.stillMine(pid), 5*time.Second) {
789 dead = false
790 errs = append(errs, fmt.Errorf("vmm process %d did not exit", pid))
791 }
792 }
793 if dead && s.exited != nil {
794 select {
795 case <-s.exited:
796 case <-time.After(5 * time.Second):
797 dead = false
798 errs = append(errs, errors.New("vmm exit was not observed"))
799 }
800 }
801 if dead {
802 if s.attached {
803 if err := d.net.Detach(ctx, s.tap, s.gateway); err != nil {
804 errs = append(errs, err)
805 }
806 if err := deleteTap(ctx, d.opt.Run, s.tap); err != nil {
807 errs = append(errs, err)
808 } else {
809 s.attached = false
810 }
811 }
812 if s.lay.dir != "" {
813 if err := removeAll(s.lay.dir); err != nil {
814 errs = append(errs, err)
815 }
816 }
817 if s.lay.jailed {
818 if err := removeCgroup(d.opt.CgroupRoot, filepath.Base(d.fcPath), s.id); err != nil {
819 errs = append(errs, err)
820 }
821 }
822 if len(errs) == 0 {
823 if err := d.store.remove(s.id); err != nil {
824 errs = append(errs, err)
825 }
826 }
827 }
828 if len(errs) == 0 {
829 s.releaseLocked()
830 return nil
831 }
832 s.stopped = true
833 d.markLeaked(s.slot)
834 s.leakErr = fmt.Errorf("%w: vm %s: %w", sandbox.ErrLeaked, s.id, errors.Join(errs...))
835 log.Error().Err(s.leakErr).Str("vm", s.id).Msg("micro-VM cleanup incomplete; call Reap to finish it")
836 return s.leakErr
837}
838
839// removeAll is os.RemoveAll that also reports a directory that is still there.
840func removeAll(dir string) error {
841 if err := os.RemoveAll(dir); err != nil {
842 return err
843 }
844 if _, err := os.Lstat(dir); err == nil {
845 return fmt.Errorf("%s still exists after removal", dir)
846 }
847 return nil
848}
849
850// removeCgroup removes the cgroup the jailer created for a VM
851// (<root>/<parent>/<id>, parent = the exec file name). The jailer never removes
852// it. A missing cgroup is fine; a non-empty one fails (rmdir semantics).
853func removeCgroup(root, parent, id string) error {
854 err := os.Remove(filepath.Join(root, parent, id))
855 if err == nil || errors.Is(err, os.ErrNotExist) {
856 return nil
857 }
858 return fmt.Errorf("remove cgroup %s: %w", filepath.Join(root, parent, id), err)
859}