detached 82488a8

feat(sandbox/vm): add the Firecracker micro-VM driver with jailer, TAP and crash reaping

commit
82488a82e63ebc7d1206cf0b1a000c85fac635d7
author
brice · · 2026-10-12 15:35 UTC
parent
d85c6ab
signature
GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email
10 files · +1967 -0
Unified
UnifiedSplit
A internal/sandbox/vm/api.go +165 -0
@@ -0,0 +1,165 @@
1+// SPDX-License-Identifier: MIT
2+
3+package vm
4+
5+import (
6+ "bytes"
7+ "context"
8+ "encoding/json"
9+ "fmt"
10+ "io"
11+ "net"
12+ "net/http"
13+ "time"
14+)
15+
16+// Firecracker is controlled through its HTTP API on a unix socket, with
17+// net/http, rather than through firecracker-go-sdk: the SDK's last release
18+// (v1.0.0, 2022) predates Firecracker 1.17 and drags in CNI, go-openapi and
19+// netlink for what is a handful of PUT requests. The structures below are the
20+// subset of the API this driver uses; Firecracker rejects unknown or invalid
21+// fields with a fault message, which surfaces in the returned error.
22+
23+type machineConfig struct {
24+ VcpuCount int `json:"vcpu_count"`
25+ MemSizeMiB int `json:"mem_size_mib"`
26+ SMT bool `json:"smt"`
27+}
28+
29+type bootSource struct {
30+ KernelImagePath string `json:"kernel_image_path"`
31+ BootArgs string `json:"boot_args"`
32+}
33+
34+type drive struct {
35+ DriveID string `json:"drive_id"`
36+ PathOnHost string `json:"path_on_host"`
37+ IsRootDevice bool `json:"is_root_device"`
38+ IsReadOnly bool `json:"is_read_only"`
39+}
40+
41+type networkInterface struct {
42+ IfaceID string `json:"iface_id"`
43+ HostDevName string `json:"host_dev_name"`
44+ GuestMAC string `json:"guest_mac"`
45+}
46+
47+type vsockDevice struct {
48+ GuestCID uint32 `json:"guest_cid"`
49+ UDSPath string `json:"uds_path"`
50+}
51+
52+type action struct {
53+ ActionType string `json:"action_type"`
54+}
55+
56+// apiClient talks to one Firecracker instance.
57+type apiClient struct {
58+ http *http.Client
59+}
60+
61+func newAPIClient(socket string) *apiClient {
62+ return &apiClient{http: &http.Client{
63+ Timeout: 10 * time.Second,
64+ Transport: &http.Transport{
65+ DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
66+ var d net.Dialer
67+ return d.DialContext(ctx, "unix", socket)
68+ },
69+ DisableKeepAlives: true,
70+ },
71+ }}
72+}
73+
74+// put sends a PUT with a JSON body and expects 204 (or 200).
75+func (c *apiClient) put(ctx context.Context, path string, body any) error {
76+ b, err := json.Marshal(body)
77+ if err != nil {
78+ return err
79+ }
80+ req, err := http.NewRequestWithContext(ctx, http.MethodPut, "http://firecracker"+path, bytes.NewReader(b))
81+ if err != nil {
82+ return err
83+ }
84+ req.Header.Set("Content-Type", "application/json")
85+ resp, err := c.http.Do(req)
86+ if err != nil {
87+ return fmt.Errorf("firecracker PUT %s: %w", path, err)
88+ }
89+ defer func() { _ = resp.Body.Close() }()
90+ if resp.StatusCode/100 != 2 {
91+ msg, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
92+ return fmt.Errorf("firecracker PUT %s: %s: %s", path, resp.Status, bytes.TrimSpace(msg))
93+ }
94+ return nil
95+}
96+
97+// ready reports whether the API answers GET /.
98+func (c *apiClient) ready(ctx context.Context) error {
99+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://firecracker/", nil)
100+ if err != nil {
101+ return err
102+ }
103+ resp, err := c.http.Do(req)
104+ if err != nil {
105+ return err
106+ }
107+ defer func() { _ = resp.Body.Close() }()
108+ _, _ = io.Copy(io.Discard, resp.Body)
109+ if resp.StatusCode/100 != 2 {
110+ return fmt.Errorf("firecracker GET /: %s", resp.Status)
111+ }
112+ return nil
113+}
114+
115+// vmConfig is everything configured before boot.
116+type vmConfig struct {
117+ Machine machineConfig
118+ Boot bootSource
119+ Drives []drive
120+ Net networkInterface
121+ Vsock vsockDevice
122+}
123+
124+// configure applies the pre-boot configuration and boots the VM. The order is
125+// fixed (and covered by a unit test) because Firecracker rejects some changes
126+// after boot and drive order decides the guest device names vda, vdb, vdc.
127+func (c *apiClient) configure(ctx context.Context, cfg vmConfig) error {
128+ steps := []struct {
129+ path string
130+ body any
131+ }{
132+ {"/machine-config", cfg.Machine},
133+ {"/boot-source", cfg.Boot},
134+ }
135+ for _, d := range cfg.Drives {
136+ steps = append(steps, struct {
137+ path string
138+ body any
139+ }{"/drives/" + d.DriveID, d})
140+ }
141+ steps = append(steps,
142+ struct {
143+ path string
144+ body any
145+ }{"/network-interfaces/" + cfg.Net.IfaceID, cfg.Net},
146+ struct {
147+ path string
148+ body any
149+ }{"/vsock", cfg.Vsock},
150+ struct {
151+ path string
152+ body any
153+ }{"/entropy", struct{}{}},
154+ struct {
155+ path string
156+ body any
157+ }{"/actions", action{ActionType: "InstanceStart"}},
158+ )
159+ for _, s := range steps {
160+ if err := c.put(ctx, s.path, s.body); err != nil {
161+ return err
162+ }
163+ }
164+ return nil
165+}
@@ -0,0 +1,165 @@
1+// SPDX-License-Identifier: MIT
2+
3+package vm
4+
5+import (
6+ "bytes"
7+ "context"
8+ "encoding/json"
9+ "fmt"
10+ "io"
11+ "net"
12+ "net/http"
13+ "time"
14+)
15+
16+// Firecracker is controlled through its HTTP API on a unix socket, with
17+// net/http, rather than through firecracker-go-sdk: the SDK's last release
18+// (v1.0.0, 2022) predates Firecracker 1.17 and drags in CNI, go-openapi and
19+// netlink for what is a handful of PUT requests. The structures below are the
20+// subset of the API this driver uses; Firecracker rejects unknown or invalid
21+// fields with a fault message, which surfaces in the returned error.
22+
23+type machineConfig struct {
24+ VcpuCount int `json:"vcpu_count"`
25+ MemSizeMiB int `json:"mem_size_mib"`
26+ SMT bool `json:"smt"`
27+}
28+
29+type bootSource struct {
30+ KernelImagePath string `json:"kernel_image_path"`
31+ BootArgs string `json:"boot_args"`
32+}
33+
34+type drive struct {
35+ DriveID string `json:"drive_id"`
36+ PathOnHost string `json:"path_on_host"`
37+ IsRootDevice bool `json:"is_root_device"`
38+ IsReadOnly bool `json:"is_read_only"`
39+}
40+
41+type networkInterface struct {
42+ IfaceID string `json:"iface_id"`
43+ HostDevName string `json:"host_dev_name"`
44+ GuestMAC string `json:"guest_mac"`
45+}
46+
47+type vsockDevice struct {
48+ GuestCID uint32 `json:"guest_cid"`
49+ UDSPath string `json:"uds_path"`
50+}
51+
52+type action struct {
53+ ActionType string `json:"action_type"`
54+}
55+
56+// apiClient talks to one Firecracker instance.
57+type apiClient struct {
58+ http *http.Client
59+}
60+
61+func newAPIClient(socket string) *apiClient {
62+ return &apiClient{http: &http.Client{
63+ Timeout: 10 * time.Second,
64+ Transport: &http.Transport{
65+ DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
66+ var d net.Dialer
67+ return d.DialContext(ctx, "unix", socket)
68+ },
69+ DisableKeepAlives: true,
70+ },
71+ }}
72+}
73+
74+// put sends a PUT with a JSON body and expects 204 (or 200).
75+func (c *apiClient) put(ctx context.Context, path string, body any) error {
76+ b, err := json.Marshal(body)
77+ if err != nil {
78+ return err
79+ }
80+ req, err := http.NewRequestWithContext(ctx, http.MethodPut, "http://firecracker"+path, bytes.NewReader(b))
81+ if err != nil {
82+ return err
83+ }
84+ req.Header.Set("Content-Type", "application/json")
85+ resp, err := c.http.Do(req)
86+ if err != nil {
87+ return fmt.Errorf("firecracker PUT %s: %w", path, err)
88+ }
89+ defer func() { _ = resp.Body.Close() }()
90+ if resp.StatusCode/100 != 2 {
91+ msg, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
92+ return fmt.Errorf("firecracker PUT %s: %s: %s", path, resp.Status, bytes.TrimSpace(msg))
93+ }
94+ return nil
95+}
96+
97+// ready reports whether the API answers GET /.
98+func (c *apiClient) ready(ctx context.Context) error {
99+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://firecracker/", nil)
100+ if err != nil {
101+ return err
102+ }
103+ resp, err := c.http.Do(req)
104+ if err != nil {
105+ return err
106+ }
107+ defer func() { _ = resp.Body.Close() }()
108+ _, _ = io.Copy(io.Discard, resp.Body)
109+ if resp.StatusCode/100 != 2 {
110+ return fmt.Errorf("firecracker GET /: %s", resp.Status)
111+ }
112+ return nil
113+}
114+
115+// vmConfig is everything configured before boot.
116+type vmConfig struct {
117+ Machine machineConfig
118+ Boot bootSource
119+ Drives []drive
120+ Net networkInterface
121+ Vsock vsockDevice
122+}
123+
124+// configure applies the pre-boot configuration and boots the VM. The order is
125+// fixed (and covered by a unit test) because Firecracker rejects some changes
126+// after boot and drive order decides the guest device names vda, vdb, vdc.
127+func (c *apiClient) configure(ctx context.Context, cfg vmConfig) error {
128+ steps := []struct {
129+ path string
130+ body any
131+ }{
132+ {"/machine-config", cfg.Machine},
133+ {"/boot-source", cfg.Boot},
134+ }
135+ for _, d := range cfg.Drives {
136+ steps = append(steps, struct {
137+ path string
138+ body any
139+ }{"/drives/" + d.DriveID, d})
140+ }
141+ steps = append(steps,
142+ struct {
143+ path string
144+ body any
145+ }{"/network-interfaces/" + cfg.Net.IfaceID, cfg.Net},
146+ struct {
147+ path string
148+ body any
149+ }{"/vsock", cfg.Vsock},
150+ struct {
151+ path string
152+ body any
153+ }{"/entropy", struct{}{}},
154+ struct {
155+ path string
156+ body any
157+ }{"/actions", action{ActionType: "InstanceStart"}},
158+ )
159+ for _, s := range steps {
160+ if err := c.put(ctx, s.path, s.body); err != nil {
161+ return err
162+ }
163+ }
164+ return nil
165+}
A internal/sandbox/vm/bootargs.go +42 -0
@@ -0,0 +1,42 @@
1+// SPDX-License-Identifier: MIT
2+
3+package vm
4+
5+import (
6+ "fmt"
7+ "net/netip"
8+ "strconv"
9+ "strings"
10+
11+ "bnema.dev/ermite-runner/internal/sandbox"
12+)
13+
14+// bootArgs builds the guest kernel command line.
15+//
16+// - console=ttyS0 reboot=k panic=1: serial console; a reboot (what init does
17+// at the end) or a panic ends the VMM instead of hanging;
18+// - ipv6.disable=1: the guest has no IPv6 (the host policy drops it too);
19+// - root=/dev/vda ro: the exec disk; init=/ermite-runner -- init: the runner
20+// executable is PID 1 and runs its init subcommand;
21+// - ip=…: static address, gateway and DNS from the kernel, no DHCP client;
22+// - ermite.*: read by init from /proc/cmdline (connect target, gateway, time).
23+func bootArgs(guest, gateway netip.Addr, unixSeconds int64) string {
24+ args := []string{
25+ "console=ttyS0", "reboot=k", "panic=1", "pci=off", "acpi=off", "8250.nr_uarts=1",
26+ "ipv6.disable=1", "random.trust_cpu=on", "i8042.noaux", "i8042.nomux", "i8042.nopnp", "i8042.dumbkbd",
27+ "root=/dev/vda", "ro", "rootfstype=ext4",
28+ "init=/" + sandbox.GuestExecName,
29+ fmt.Sprintf("ip=%s::%s:255.255.255.252:ermite-job:eth0:off:%s", guest, gateway, gateway),
30+ sandbox.CmdlineSandbox + "=" + string(sandbox.VM),
31+ sandbox.CmdlineConnect + "=vsock:" + strconv.Itoa(sandbox.VsockPort),
32+ sandbox.CmdlineGateway + "=" + gateway.String(),
33+ "ermite.time=" + strconv.FormatInt(unixSeconds, 10),
34+ "--", "init",
35+ }
36+ return strings.Join(args, " ")
37+}
38+
39+// macFor derives a locally administered MAC from the slot.
40+func macFor(slot int) string {
41+ return fmt.Sprintf("06:00:ac:10:%02x:%02x", (slot>>8)&0xff, slot&0xff)
42+}
@@ -0,0 +1,42 @@
1+// SPDX-License-Identifier: MIT
2+
3+package vm
4+
5+import (
6+ "fmt"
7+ "net/netip"
8+ "strconv"
9+ "strings"
10+
11+ "bnema.dev/ermite-runner/internal/sandbox"
12+)
13+
14+// bootArgs builds the guest kernel command line.
15+//
16+// - console=ttyS0 reboot=k panic=1: serial console; a reboot (what init does
17+// at the end) or a panic ends the VMM instead of hanging;
18+// - ipv6.disable=1: the guest has no IPv6 (the host policy drops it too);
19+// - root=/dev/vda ro: the exec disk; init=/ermite-runner -- init: the runner
20+// executable is PID 1 and runs its init subcommand;
21+// - ip=…: static address, gateway and DNS from the kernel, no DHCP client;
22+// - ermite.*: read by init from /proc/cmdline (connect target, gateway, time).
23+func bootArgs(guest, gateway netip.Addr, unixSeconds int64) string {
24+ args := []string{
25+ "console=ttyS0", "reboot=k", "panic=1", "pci=off", "acpi=off", "8250.nr_uarts=1",
26+ "ipv6.disable=1", "random.trust_cpu=on", "i8042.noaux", "i8042.nomux", "i8042.nopnp", "i8042.dumbkbd",
27+ "root=/dev/vda", "ro", "rootfstype=ext4",
28+ "init=/" + sandbox.GuestExecName,
29+ fmt.Sprintf("ip=%s::%s:255.255.255.252:ermite-job:eth0:off:%s", guest, gateway, gateway),
30+ sandbox.CmdlineSandbox + "=" + string(sandbox.VM),
31+ sandbox.CmdlineConnect + "=vsock:" + strconv.Itoa(sandbox.VsockPort),
32+ sandbox.CmdlineGateway + "=" + gateway.String(),
33+ "ermite.time=" + strconv.FormatInt(unixSeconds, 10),
34+ "--", "init",
35+ }
36+ return strings.Join(args, " ")
37+}
38+
39+// macFor derives a locally administered MAC from the slot.
40+func macFor(slot int) string {
41+ return fmt.Sprintf("06:00:ac:10:%02x:%02x", (slot>>8)&0xff, slot&0xff)
42+}
A internal/sandbox/vm/driver.go +495 -0
@@ -0,0 +1,495 @@
1+// SPDX-License-Identifier: MIT
2+
3+// Package vm is the micro-VM sandbox: one Firecracker VM per job.
4+//
5+// Per VM, on the host: a jailer chroot (or, without jailer, a plain directory)
6+// holding the per-job sparse scratch disk and the vsock socket; a TAP device
7+// with a /30; rules in the dedicated nftables table; the DNS forwarder bound on
8+// the TAP's host address. In the guest: three virtio disks (read-only runner
9+// executable as the boot root, read-only shared base image, per-job scratch)
10+// and a vsock device. Everything is traced in a state-directory record before it
11+// is created, so that Reap can remove it after a crash.
12+package vm
13+
14+import (
15+ "context"
16+ "errors"
17+ "fmt"
18+ "net"
19+ "net/netip"
20+ "os"
21+ "os/exec"
22+ "path/filepath"
23+ "sync"
24+ "syscall"
25+ "time"
26+
27+ "github.com/bnema/zerowrap"
28+ "golang.org/x/sys/unix"
29+
30+ "bnema.dev/ermite-runner/internal/cmdrun"
31+ "bnema.dev/ermite-runner/internal/netpolicy"
32+ "bnema.dev/ermite-runner/internal/sandbox"
33+ "bnema.dev/ermite-runner/internal/sandbox/netsetup"
34+)
35+
36+// Options are the testable knobs of the driver.
37+type Options struct {
38+ Run cmdrun.Runner
39+ // ProcDir is the /proc used to find orphan Firecracker processes.
40+ ProcDir string
41+ // BootTimeout bounds boot + init + executor connection (default 90 s).
42+ BootTimeout time.Duration
43+ // Net overrides the network plumbing (tests).
44+ Net netsetup.Plumbing
45+ // FirecrackerArgs are extra arguments for Firecracker (tests use them to
46+ // select the fake).
47+ FirecrackerArgs []string
48+}
49+
50+// Driver implements sandbox.Driver for micro-VMs.
51+type Driver struct {
52+ cfg sandbox.Config
53+ opt Options
54+ fcPath string
55+ jailer string
56+ execDisk string
57+ store store
58+ net netsetup.Plumbing
59+ lock *os.File
60+
61+ mu sync.Mutex
62+ slots *sandbox.Slots
63+}
64+
65+// New builds the driver: resolves binaries, builds the exec disk, takes the
66+// state-directory lock and enables IPv4 forwarding. Run Reap before New sandboxes.
67+func New(ctx context.Context, cfg sandbox.Config, opt Options) (*Driver, error) {
68+ if opt.Run == nil {
69+ opt.Run = cmdrun.Exec{}
70+ }
71+ if opt.ProcDir == "" {
72+ opt.ProcDir = "/proc"
73+ }
74+ if opt.BootTimeout == 0 {
75+ opt.BootTimeout = 90 * time.Second
76+ }
77+ d := &Driver{cfg: cfg, opt: opt, slots: sandbox.NewSlots(cfg.Capacity), store: store{dir: filepath.Join(cfg.StateDir, "vms")}}
78+
79+ var err error
80+ if d.fcPath, err = resolveBin(cfg.Firecracker, "firecracker"); err != nil {
81+ return nil, err
82+ }
83+ if !cfg.JailerOff {
84+ if d.jailer, err = resolveBin(cfg.Jailer, "jailer"); err != nil {
85+ return nil, err
86+ }
87+ }
88+ exe, err := cfg.ExecutablePath()
89+ if err != nil {
90+ return nil, err
91+ }
92+ if err := sandbox.CheckStatic(exe); err != nil {
93+ return nil, err
94+ }
95+ if err := os.MkdirAll(cfg.StateDir, 0o755); err != nil {
96+ return nil, fmt.Errorf("state directory: %w", err)
97+ }
98+ if err := d.acquireLock(); err != nil {
99+ return nil, err
100+ }
101+ fail := func(err error) (*Driver, error) { _ = d.Close(); return nil, err }
102+
103+ if !cfg.JailerOff {
104+ for _, p := range []string{cfg.KernelPath(), cfg.BaseImagePath()} {
105+ if err := checkWorldReadable(p); err != nil {
106+ return fail(err)
107+ }
108+ }
109+ }
110+ if d.execDisk, err = ensureExecDisk(ctx, opt.Run, exe, cfg.StateDir); err != nil {
111+ return fail(err)
112+ }
113+ d.net = opt.Net
114+ if d.net == nil {
115+ d.net = netsetup.New(opt.Run, cfg.DNSPort, "")
116+ }
117+ if err := d.net.EnableForwarding(); err != nil {
118+ return fail(err)
119+ }
120+ return d, nil
121+}
122+
123+func resolveBin(configured, name string) (string, error) {
124+ p := configured
125+ if p == "" {
126+ var err error
127+ if p, err = exec.LookPath(name); err != nil {
128+ return "", fmt.Errorf("%s not found in PATH (set ERMITE_RUNNER_%s)", name, map[string]string{"firecracker": "FIRECRACKER", "jailer": "JAILER"}[name])
129+ }
130+ }
131+ abs, err := filepath.Abs(p)
132+ if err != nil {
133+ return "", err
134+ }
135+ if _, err := os.Stat(abs); err != nil {
136+ return "", fmt.Errorf("%s: %w", name, err)
137+ }
138+ return abs, nil
139+}
140+
141+func (d *Driver) acquireLock() error {
142+ f, err := os.OpenFile(filepath.Join(d.cfg.StateDir, "vm.lock"), os.O_CREATE|os.O_RDWR, 0o600)
143+ if err != nil {
144+ return err
145+ }
146+ if err := unix.Flock(int(f.Fd()), unix.LOCK_EX|unix.LOCK_NB); err != nil {
147+ _ = f.Close()
148+ return fmt.Errorf("another ermite-runner already uses state directory %s (reaping would destroy its VMs): %w", d.cfg.StateDir, err)
149+ }
150+ d.lock = f
151+ return nil
152+}
153+
154+// Mode implements sandbox.Driver.
155+func (d *Driver) Mode() sandbox.Mode { return sandbox.VM }
156+
157+// Close releases the lock and the DNS listeners.
158+func (d *Driver) Close() error {
159+ if d.net != nil {
160+ d.net.Close()
161+ }
162+ if d.lock != nil {
163+ err := d.lock.Close()
164+ d.lock = nil
165+ return err
166+ }
167+ return nil
168+}
169+
170+// New implements sandbox.Driver.
171+func (d *Driver) New() (sandbox.Sandbox, error) {
172+ d.mu.Lock()
173+ defer d.mu.Unlock()
174+ slot, err := d.slots.Acquire()
175+ if err != nil {
176+ return nil, err
177+ }
178+ return &vmSandbox{d: d, slot: slot}, nil
179+}
180+
181+func (d *Driver) release(slot int) {
182+ d.mu.Lock()
183+ d.slots.Release(slot)
184+ d.mu.Unlock()
185+}
186+
187+// vmSandbox is one VM, single use.
188+type vmSandbox struct {
189+ d *Driver
190+ slot int
191+
192+ mu sync.Mutex
193+ started bool
194+ stopped bool
195+ rec record
196+ lay layout
197+ cmd *exec.Cmd
198+ exited chan struct{}
199+ console *ring
200+ listener net.Listener
201+ conn net.Conn
202+ tap string
203+ gateway netip.Addr
204+ attached bool
205+}
206+
207+func (s *vmSandbox) uid() int { return s.d.cfg.UIDBase + s.slot }
208+
209+// Start implements sandbox.Sandbox.
210+func (s *vmSandbox) Start(ctx context.Context, spec sandbox.JobSpec) (sandbox.Conn, error) {
211+ s.mu.Lock()
212+ defer s.mu.Unlock()
213+ if s.started || s.stopped {
214+ return nil, errors.New("vm sandbox: Start called twice or after Stop")
215+ }
216+ s.started = true
217+ log := zerowrap.FromCtx(ctx)
218+ d := s.d
219+ spec = d.cfg.Resolve(spec)
220+
221+ id, err := sandbox.NewID()
222+ if err != nil {
223+ s.releaseLocked()
224+ return nil, err
225+ }
226+ jailed := !d.cfg.JailerOff
227+ runUID := s.uid()
228+ if !jailed {
229+ runUID = os.Getuid()
230+ }
231+ s.lay = newLayout(d.cfg.StateDir, filepath.Base(d.fcPath), id, jailed, runUID)
232+ s.rec = record{ID: id, Slot: s.slot, Job: spec.JobID, Dir: s.lay.dir}
233+ // The record exists before the first resource, so any crash from here on is
234+ // reapable.
235+ if err := d.store.save(s.rec); err != nil {
236+ s.releaseLocked()
237+ return nil, fmt.Errorf("write state record: %w", err)
238+ }
239+
240+ conn, err := s.boot(ctx, spec, id)
241+ if err != nil {
242+ console := ""
243+ if s.console != nil {
244+ console = s.console.String()
245+ }
246+ cerr := s.cleanupLocked(context.WithoutCancel(ctx))
247+ if console != "" {
248+ err = fmt.Errorf("%w\nVMM console tail:\n%s", err, console)
249+ }
250+ return nil, errors.Join(err, cerr)
251+ }
252+ log.Info().Str("vm", id).Int("slot", s.slot).Str("job", spec.JobID).Msg("micro-VM running")
253+ return conn, nil
254+}
255+
256+func (s *vmSandbox) boot(ctx context.Context, spec sandbox.JobSpec, id string) (sandbox.Conn, error) {
257+ d := s.d
258+ netw, err := netpolicy.Subnet(d.cfg.Network, s.slot)
259+ if err != nil {
260+ return nil, err
261+ }
262+
263+ // Network first: the TAP must exist when Firecracker opens it.
264+ s.tap, s.gateway = tapName(s.slot), netw.Gateway
265+ s.rec.Tap, s.rec.Gate = s.tap, s.gateway.String()
266+ if err := d.store.save(s.rec); err != nil {
267+ return nil, err
268+ }
269+ if err := createTap(ctx, d.opt.Run, s.tap, s.lay.uid, s.gateway); err != nil {
270+ return nil, err
271+ }
272+ s.attached = true // from here Detach is idempotent and always safe
273+ if err := d.net.Attach(ctx, netpolicy.Guest{Iface: s.tap, Gateway: netw.Gateway, Addr: netw.Guest}); err != nil {
274+ return nil, err
275+ }
276+
277+ // The VMM process.
278+ apiSock := s.lay.host("api.sock")
279+ if s.lay.jailed {
280+ apiSock = s.lay.host("run/firecracker.socket")
281+ } else if err := os.MkdirAll(s.lay.root, 0o700); err != nil {
282+ return nil, err
283+ }
284+ if err := s.startVMM(id, apiSock); err != nil {
285+ return nil, err
286+ }
287+ api := newAPIClient(apiSock)
288+ if err := s.waitAPI(ctx, api, apiSock); err != nil {
289+ return nil, err
290+ }
291+
292+ // Files: the process only opens them when configured below.
293+ execDisk, err := s.lay.share(d.execDisk, "exec.ext4")
294+ if err != nil {
295+ return nil, err
296+ }
297+ baseDisk, err := s.lay.share(d.cfg.BaseImagePath(), "base.ext4")
298+ if err != nil {
299+ return nil, err
300+ }
301+ kernel, err := s.lay.share(d.cfg.KernelPath(), "kernel")
302+ if err != nil {
303+ return nil, err
304+ }
305+ if err := makeScratch(ctx, d.opt.Run, s.lay.host("scratch.ext4"), spec.DiskMiB); err != nil {
306+ return nil, err
307+ }
308+ if err := s.lay.own("scratch.ext4"); err != nil {
309+ return nil, err
310+ }
311+
312+ // The host side of the vsock stream: Firecracker connects to
313+ // <uds_path>_<port> when the guest connects to the host CID.
314+ udsName := "v.sock"
315+ ln, err := net.Listen("unix", s.lay.host(udsName)+fmt.Sprintf("_%d", sandbox.VsockPort))
316+ if err != nil {
317+ return nil, fmt.Errorf("listen vsock socket: %w", err)
318+ }
319+ s.listener = ln
320+ if s.lay.jailed {
321+ if err := os.Chown(s.lay.host(udsName+fmt.Sprintf("_%d", sandbox.VsockPort)), s.lay.uid, s.lay.uid); err != nil {
322+ return nil, err
323+ }
324+ }
325+ accepted := make(chan acceptResult, 1)
326+ go func() { c, err := ln.Accept(); accepted <- acceptResult{c, err} }()
327+
328+ err = api.configure(ctx, vmConfig{
329+ Machine: machineConfig{VcpuCount: spec.CPUs, MemSizeMiB: spec.MemoryMiB},
330+ Boot: bootSource{KernelImagePath: kernel, BootArgs: bootArgs(netw.Guest, netw.Gateway, time.Now().Unix())},
331+ Drives: []drive{
332+ {DriveID: "rootfs", PathOnHost: execDisk, IsRootDevice: true, IsReadOnly: true},
333+ {DriveID: "base", PathOnHost: baseDisk, IsReadOnly: true},
334+ {DriveID: "scratch", PathOnHost: s.lay.fc("scratch.ext4")},
335+ },
336+ Net: networkInterface{IfaceID: "eth0", HostDevName: s.tap, GuestMAC: macFor(s.slot)},
337+ Vsock: vsockDevice{GuestCID: 3, UDSPath: s.lay.fc(udsName)},
338+ })
339+ if err != nil {
340+ return nil, err
341+ }
342+
343+ timer := time.NewTimer(d.opt.BootTimeout)
344+ defer timer.Stop()
345+ select {
346+ case r := <-accepted:
347+ if r.err != nil {
348+ return nil, fmt.Errorf("accept executor connection: %w", r.err)
349+ }
350+ s.conn = r.c
351+ return r.c, nil
352+ case <-s.exited:
353+ return nil, errors.New("firecracker exited before the executor connected")
354+ case <-timer.C:
355+ return nil, fmt.Errorf("executor did not connect within %s", d.opt.BootTimeout)
356+ case <-ctx.Done():
357+ return nil, ctx.Err()
358+ }
359+}
360+
361+type acceptResult struct {
362+ c net.Conn
363+ err error
364+}
365+
366+func (s *vmSandbox) startVMM(id, apiSock string) error {
367+ d := s.d
368+ var cmd *exec.Cmd
369+ if s.lay.jailed {
370+ args := []string{"--id", id, "--exec-file", d.fcPath,
371+ "--uid", fmt.Sprint(s.uid()), "--gid", fmt.Sprint(s.uid()),
372+ "--chroot-base-dir", jailBase(d.cfg.StateDir),
373+ "--resource-limit", "no-file=4096",
374+ "--"}
375+ cmd = exec.Command(d.jailer, append(args, d.opt.FirecrackerArgs...)...)
376+ } else {
377+ args := append([]string{"--id", id, "--api-sock", apiSock}, d.opt.FirecrackerArgs...)
378+ cmd = exec.Command(d.fcPath, args...)
379+ }
380+ s.console = newRing(16 << 10)
381+ cmd.Stdout, cmd.Stderr = s.console, s.console
382+ cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
383+ if err := cmd.Start(); err != nil {
384+ return fmt.Errorf("start firecracker: %w", err)
385+ }
386+ s.cmd = cmd
387+ s.exited = make(chan struct{})
388+ go func() { _ = cmd.Wait(); close(s.exited) }()
389+ s.rec.PID = cmd.Process.Pid
390+ return s.d.store.save(s.rec)
391+}
392+
393+func (s *vmSandbox) waitAPI(ctx context.Context, api *apiClient, sock string) error {
394+ deadline := time.Now().Add(10 * time.Second)
395+ for {
396+ if err := api.ready(ctx); err == nil {
397+ return nil
398+ }
399+ select {
400+ case <-s.exited:
401+ return errors.New("firecracker exited before its API was ready")
402+ case <-ctx.Done():
403+ return ctx.Err()
404+ case <-time.After(25 * time.Millisecond):
405+ }
406+ if time.Now().After(deadline) {
407+ return fmt.Errorf("firecracker API socket %s not ready after 10s", sock)
408+ }
409+ }
410+}
411+
412+// Stop implements sandbox.Sandbox.
413+func (s *vmSandbox) Stop(ctx context.Context) error {
414+ s.mu.Lock()
415+ defer s.mu.Unlock()
416+ if s.stopped {
417+ return nil
418+ }
419+ if !s.started {
420+ s.stopped = true
421+ s.releaseLocked()
422+ return nil
423+ }
424+ return s.cleanupLocked(context.WithoutCancel(ctx))
425+}
426+
427+func (s *vmSandbox) releaseLocked() {
428+ s.stopped = true
429+ s.d.release(s.slot)
430+}
431+
432+// cleanupLocked removes everything the VM created, in dependency order, and
433+// keeps going after an error. The state record and the slot are released only
434+// when everything is gone, so that a half-failed cleanup is retried by Reap
435+// rather than forgotten and the slot (hence TAP name) is not reused meanwhile.
436+func (s *vmSandbox) cleanupLocked(ctx context.Context) error {
437+ log := zerowrap.FromCtx(ctx)
438+ d := s.d
439+ var errs []error
440+ if s.conn != nil {
441+ _ = s.conn.Close()
442+ s.conn = nil
443+ }
444+ if s.listener != nil {
445+ _ = s.listener.Close()
446+ s.listener = nil
447+ }
448+ if s.cmd != nil && s.cmd.Process != nil {
449+ killAndWait(s.cmd.Process.Pid, 5*time.Second)
450+ select {
451+ case <-s.exited:
452+ case <-time.After(5 * time.Second):
453+ errs = append(errs, fmt.Errorf("firecracker pid %d did not exit", s.cmd.Process.Pid))
454+ }
455+ }
456+ if s.attached {
457+ if err := d.net.Detach(ctx, s.tap, s.gateway); err != nil {
458+ errs = append(errs, err)
459+ }
460+ if err := deleteTap(ctx, d.opt.Run, s.tap); err != nil {
461+ errs = append(errs, err)
462+ } else {
463+ s.attached = false
464+ }
465+ }
466+ if s.lay.dir != "" {
467+ if err := removeAll(s.lay.dir); err != nil {
468+ errs = append(errs, err)
469+ }
470+ }
471+ if len(errs) == 0 {
472+ if err := d.store.remove(s.rec.ID); err != nil {
473+ errs = append(errs, err)
474+ }
475+ }
476+ if len(errs) == 0 {
477+ s.releaseLocked()
478+ return nil
479+ }
480+ s.stopped = true // do not retry from here; Reap at the next start will
481+ err := errors.Join(errs...)
482+ log.Error().Err(err).Str("vm", s.rec.ID).Msg("micro-VM cleanup incomplete; the next runner start will retry")
483+ return err
484+}
485+
486+// removeAll is os.RemoveAll that also reports a directory that is still there.
487+func removeAll(dir string) error {
488+ if err := os.RemoveAll(dir); err != nil {
489+ return err
490+ }
491+ if _, err := os.Lstat(dir); err == nil {
492+ return fmt.Errorf("%s still exists after removal", dir)
493+ }
494+ return nil
495+}
@@ -0,0 +1,495 @@
1+// SPDX-License-Identifier: MIT
2+
3+// Package vm is the micro-VM sandbox: one Firecracker VM per job.
4+//
5+// Per VM, on the host: a jailer chroot (or, without jailer, a plain directory)
6+// holding the per-job sparse scratch disk and the vsock socket; a TAP device
7+// with a /30; rules in the dedicated nftables table; the DNS forwarder bound on
8+// the TAP's host address. In the guest: three virtio disks (read-only runner
9+// executable as the boot root, read-only shared base image, per-job scratch)
10+// and a vsock device. Everything is traced in a state-directory record before it
11+// is created, so that Reap can remove it after a crash.
12+package vm
13+
14+import (
15+ "context"
16+ "errors"
17+ "fmt"
18+ "net"
19+ "net/netip"
20+ "os"
21+ "os/exec"
22+ "path/filepath"
23+ "sync"
24+ "syscall"
25+ "time"
26+
27+ "github.com/bnema/zerowrap"
28+ "golang.org/x/sys/unix"
29+
30+ "bnema.dev/ermite-runner/internal/cmdrun"
31+ "bnema.dev/ermite-runner/internal/netpolicy"
32+ "bnema.dev/ermite-runner/internal/sandbox"
33+ "bnema.dev/ermite-runner/internal/sandbox/netsetup"
34+)
35+
36+// Options are the testable knobs of the driver.
37+type Options struct {
38+ Run cmdrun.Runner
39+ // ProcDir is the /proc used to find orphan Firecracker processes.
40+ ProcDir string
41+ // BootTimeout bounds boot + init + executor connection (default 90 s).
42+ BootTimeout time.Duration
43+ // Net overrides the network plumbing (tests).
44+ Net netsetup.Plumbing
45+ // FirecrackerArgs are extra arguments for Firecracker (tests use them to
46+ // select the fake).
47+ FirecrackerArgs []string
48+}
49+
50+// Driver implements sandbox.Driver for micro-VMs.
51+type Driver struct {
52+ cfg sandbox.Config
53+ opt Options
54+ fcPath string
55+ jailer string
56+ execDisk string
57+ store store
58+ net netsetup.Plumbing
59+ lock *os.File
60+
61+ mu sync.Mutex
62+ slots *sandbox.Slots
63+}
64+
65+// New builds the driver: resolves binaries, builds the exec disk, takes the
66+// state-directory lock and enables IPv4 forwarding. Run Reap before New sandboxes.
67+func New(ctx context.Context, cfg sandbox.Config, opt Options) (*Driver, error) {
68+ if opt.Run == nil {
69+ opt.Run = cmdrun.Exec{}
70+ }
71+ if opt.ProcDir == "" {
72+ opt.ProcDir = "/proc"
73+ }
74+ if opt.BootTimeout == 0 {
75+ opt.BootTimeout = 90 * time.Second
76+ }
77+ d := &Driver{cfg: cfg, opt: opt, slots: sandbox.NewSlots(cfg.Capacity), store: store{dir: filepath.Join(cfg.StateDir, "vms")}}
78+
79+ var err error
80+ if d.fcPath, err = resolveBin(cfg.Firecracker, "firecracker"); err != nil {
81+ return nil, err
82+ }
83+ if !cfg.JailerOff {
84+ if d.jailer, err = resolveBin(cfg.Jailer, "jailer"); err != nil {
85+ return nil, err
86+ }
87+ }
88+ exe, err := cfg.ExecutablePath()
89+ if err != nil {
90+ return nil, err
91+ }
92+ if err := sandbox.CheckStatic(exe); err != nil {
93+ return nil, err
94+ }
95+ if err := os.MkdirAll(cfg.StateDir, 0o755); err != nil {
96+ return nil, fmt.Errorf("state directory: %w", err)
97+ }
98+ if err := d.acquireLock(); err != nil {
99+ return nil, err
100+ }
101+ fail := func(err error) (*Driver, error) { _ = d.Close(); return nil, err }
102+
103+ if !cfg.JailerOff {
104+ for _, p := range []string{cfg.KernelPath(), cfg.BaseImagePath()} {
105+ if err := checkWorldReadable(p); err != nil {
106+ return fail(err)
107+ }
108+ }
109+ }
110+ if d.execDisk, err = ensureExecDisk(ctx, opt.Run, exe, cfg.StateDir); err != nil {
111+ return fail(err)
112+ }
113+ d.net = opt.Net
114+ if d.net == nil {
115+ d.net = netsetup.New(opt.Run, cfg.DNSPort, "")
116+ }
117+ if err := d.net.EnableForwarding(); err != nil {
118+ return fail(err)
119+ }
120+ return d, nil
121+}
122+
123+func resolveBin(configured, name string) (string, error) {
124+ p := configured
125+ if p == "" {
126+ var err error
127+ if p, err = exec.LookPath(name); err != nil {
128+ return "", fmt.Errorf("%s not found in PATH (set ERMITE_RUNNER_%s)", name, map[string]string{"firecracker": "FIRECRACKER", "jailer": "JAILER"}[name])
129+ }
130+ }
131+ abs, err := filepath.Abs(p)
132+ if err != nil {
133+ return "", err
134+ }
135+ if _, err := os.Stat(abs); err != nil {
136+ return "", fmt.Errorf("%s: %w", name, err)
137+ }
138+ return abs, nil
139+}
140+
141+func (d *Driver) acquireLock() error {
142+ f, err := os.OpenFile(filepath.Join(d.cfg.StateDir, "vm.lock"), os.O_CREATE|os.O_RDWR, 0o600)
143+ if err != nil {
144+ return err
145+ }
146+ if err := unix.Flock(int(f.Fd()), unix.LOCK_EX|unix.LOCK_NB); err != nil {
147+ _ = f.Close()
148+ return fmt.Errorf("another ermite-runner already uses state directory %s (reaping would destroy its VMs): %w", d.cfg.StateDir, err)
149+ }
150+ d.lock = f
151+ return nil
152+}
153+
154+// Mode implements sandbox.Driver.
155+func (d *Driver) Mode() sandbox.Mode { return sandbox.VM }
156+
157+// Close releases the lock and the DNS listeners.
158+func (d *Driver) Close() error {
159+ if d.net != nil {
160+ d.net.Close()
161+ }
162+ if d.lock != nil {
163+ err := d.lock.Close()
164+ d.lock = nil
165+ return err
166+ }
167+ return nil
168+}
169+
170+// New implements sandbox.Driver.
171+func (d *Driver) New() (sandbox.Sandbox, error) {
172+ d.mu.Lock()
173+ defer d.mu.Unlock()
174+ slot, err := d.slots.Acquire()
175+ if err != nil {
176+ return nil, err
177+ }
178+ return &vmSandbox{d: d, slot: slot}, nil
179+}
180+
181+func (d *Driver) release(slot int) {
182+ d.mu.Lock()
183+ d.slots.Release(slot)
184+ d.mu.Unlock()
185+}
186+
187+// vmSandbox is one VM, single use.
188+type vmSandbox struct {
189+ d *Driver
190+ slot int
191+
192+ mu sync.Mutex
193+ started bool
194+ stopped bool
195+ rec record
196+ lay layout
197+ cmd *exec.Cmd
198+ exited chan struct{}
199+ console *ring
200+ listener net.Listener
201+ conn net.Conn
202+ tap string
203+ gateway netip.Addr
204+ attached bool
205+}
206+
207+func (s *vmSandbox) uid() int { return s.d.cfg.UIDBase + s.slot }
208+
209+// Start implements sandbox.Sandbox.
210+func (s *vmSandbox) Start(ctx context.Context, spec sandbox.JobSpec) (sandbox.Conn, error) {
211+ s.mu.Lock()
212+ defer s.mu.Unlock()
213+ if s.started || s.stopped {
214+ return nil, errors.New("vm sandbox: Start called twice or after Stop")
215+ }
216+ s.started = true
217+ log := zerowrap.FromCtx(ctx)
218+ d := s.d
219+ spec = d.cfg.Resolve(spec)
220+
221+ id, err := sandbox.NewID()
222+ if err != nil {
223+ s.releaseLocked()
224+ return nil, err
225+ }
226+ jailed := !d.cfg.JailerOff
227+ runUID := s.uid()
228+ if !jailed {
229+ runUID = os.Getuid()
230+ }
231+ s.lay = newLayout(d.cfg.StateDir, filepath.Base(d.fcPath), id, jailed, runUID)
232+ s.rec = record{ID: id, Slot: s.slot, Job: spec.JobID, Dir: s.lay.dir}
233+ // The record exists before the first resource, so any crash from here on is
234+ // reapable.
235+ if err := d.store.save(s.rec); err != nil {
236+ s.releaseLocked()
237+ return nil, fmt.Errorf("write state record: %w", err)
238+ }
239+
240+ conn, err := s.boot(ctx, spec, id)
241+ if err != nil {
242+ console := ""
243+ if s.console != nil {
244+ console = s.console.String()
245+ }
246+ cerr := s.cleanupLocked(context.WithoutCancel(ctx))
247+ if console != "" {
248+ err = fmt.Errorf("%w\nVMM console tail:\n%s", err, console)
249+ }
250+ return nil, errors.Join(err, cerr)
251+ }
252+ log.Info().Str("vm", id).Int("slot", s.slot).Str("job", spec.JobID).Msg("micro-VM running")
253+ return conn, nil
254+}
255+
256+func (s *vmSandbox) boot(ctx context.Context, spec sandbox.JobSpec, id string) (sandbox.Conn, error) {
257+ d := s.d
258+ netw, err := netpolicy.Subnet(d.cfg.Network, s.slot)
259+ if err != nil {
260+ return nil, err
261+ }
262+
263+ // Network first: the TAP must exist when Firecracker opens it.
264+ s.tap, s.gateway = tapName(s.slot), netw.Gateway
265+ s.rec.Tap, s.rec.Gate = s.tap, s.gateway.String()
266+ if err := d.store.save(s.rec); err != nil {
267+ return nil, err
268+ }
269+ if err := createTap(ctx, d.opt.Run, s.tap, s.lay.uid, s.gateway); err != nil {
270+ return nil, err
271+ }
272+ s.attached = true // from here Detach is idempotent and always safe
273+ if err := d.net.Attach(ctx, netpolicy.Guest{Iface: s.tap, Gateway: netw.Gateway, Addr: netw.Guest}); err != nil {
274+ return nil, err
275+ }
276+
277+ // The VMM process.
278+ apiSock := s.lay.host("api.sock")
279+ if s.lay.jailed {
280+ apiSock = s.lay.host("run/firecracker.socket")
281+ } else if err := os.MkdirAll(s.lay.root, 0o700); err != nil {
282+ return nil, err
283+ }
284+ if err := s.startVMM(id, apiSock); err != nil {
285+ return nil, err
286+ }
287+ api := newAPIClient(apiSock)
288+ if err := s.waitAPI(ctx, api, apiSock); err != nil {
289+ return nil, err
290+ }
291+
292+ // Files: the process only opens them when configured below.
293+ execDisk, err := s.lay.share(d.execDisk, "exec.ext4")
294+ if err != nil {
295+ return nil, err
296+ }
297+ baseDisk, err := s.lay.share(d.cfg.BaseImagePath(), "base.ext4")
298+ if err != nil {
299+ return nil, err
300+ }
301+ kernel, err := s.lay.share(d.cfg.KernelPath(), "kernel")
302+ if err != nil {
303+ return nil, err
304+ }
305+ if err := makeScratch(ctx, d.opt.Run, s.lay.host("scratch.ext4"), spec.DiskMiB); err != nil {
306+ return nil, err
307+ }
308+ if err := s.lay.own("scratch.ext4"); err != nil {
309+ return nil, err
310+ }
311+
312+ // The host side of the vsock stream: Firecracker connects to
313+ // <uds_path>_<port> when the guest connects to the host CID.
314+ udsName := "v.sock"
315+ ln, err := net.Listen("unix", s.lay.host(udsName)+fmt.Sprintf("_%d", sandbox.VsockPort))
316+ if err != nil {
317+ return nil, fmt.Errorf("listen vsock socket: %w", err)
318+ }
319+ s.listener = ln
320+ if s.lay.jailed {
321+ if err := os.Chown(s.lay.host(udsName+fmt.Sprintf("_%d", sandbox.VsockPort)), s.lay.uid, s.lay.uid); err != nil {
322+ return nil, err
323+ }
324+ }
325+ accepted := make(chan acceptResult, 1)
326+ go func() { c, err := ln.Accept(); accepted <- acceptResult{c, err} }()
327+
328+ err = api.configure(ctx, vmConfig{
329+ Machine: machineConfig{VcpuCount: spec.CPUs, MemSizeMiB: spec.MemoryMiB},
330+ Boot: bootSource{KernelImagePath: kernel, BootArgs: bootArgs(netw.Guest, netw.Gateway, time.Now().Unix())},
331+ Drives: []drive{
332+ {DriveID: "rootfs", PathOnHost: execDisk, IsRootDevice: true, IsReadOnly: true},
333+ {DriveID: "base", PathOnHost: baseDisk, IsReadOnly: true},
334+ {DriveID: "scratch", PathOnHost: s.lay.fc("scratch.ext4")},
335+ },
336+ Net: networkInterface{IfaceID: "eth0", HostDevName: s.tap, GuestMAC: macFor(s.slot)},
337+ Vsock: vsockDevice{GuestCID: 3, UDSPath: s.lay.fc(udsName)},
338+ })
339+ if err != nil {
340+ return nil, err
341+ }
342+
343+ timer := time.NewTimer(d.opt.BootTimeout)
344+ defer timer.Stop()
345+ select {
346+ case r := <-accepted:
347+ if r.err != nil {
348+ return nil, fmt.Errorf("accept executor connection: %w", r.err)
349+ }
350+ s.conn = r.c
351+ return r.c, nil
352+ case <-s.exited:
353+ return nil, errors.New("firecracker exited before the executor connected")
354+ case <-timer.C:
355+ return nil, fmt.Errorf("executor did not connect within %s", d.opt.BootTimeout)
356+ case <-ctx.Done():
357+ return nil, ctx.Err()
358+ }
359+}
360+
361+type acceptResult struct {
362+ c net.Conn
363+ err error
364+}
365+
366+func (s *vmSandbox) startVMM(id, apiSock string) error {
367+ d := s.d
368+ var cmd *exec.Cmd
369+ if s.lay.jailed {
370+ args := []string{"--id", id, "--exec-file", d.fcPath,
371+ "--uid", fmt.Sprint(s.uid()), "--gid", fmt.Sprint(s.uid()),
372+ "--chroot-base-dir", jailBase(d.cfg.StateDir),
373+ "--resource-limit", "no-file=4096",
374+ "--"}
375+ cmd = exec.Command(d.jailer, append(args, d.opt.FirecrackerArgs...)...)
376+ } else {
377+ args := append([]string{"--id", id, "--api-sock", apiSock}, d.opt.FirecrackerArgs...)
378+ cmd = exec.Command(d.fcPath, args...)
379+ }
380+ s.console = newRing(16 << 10)
381+ cmd.Stdout, cmd.Stderr = s.console, s.console
382+ cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
383+ if err := cmd.Start(); err != nil {
384+ return fmt.Errorf("start firecracker: %w", err)
385+ }
386+ s.cmd = cmd
387+ s.exited = make(chan struct{})
388+ go func() { _ = cmd.Wait(); close(s.exited) }()
389+ s.rec.PID = cmd.Process.Pid
390+ return s.d.store.save(s.rec)
391+}
392+
393+func (s *vmSandbox) waitAPI(ctx context.Context, api *apiClient, sock string) error {
394+ deadline := time.Now().Add(10 * time.Second)
395+ for {
396+ if err := api.ready(ctx); err == nil {
397+ return nil
398+ }
399+ select {
400+ case <-s.exited:
401+ return errors.New("firecracker exited before its API was ready")
402+ case <-ctx.Done():
403+ return ctx.Err()
404+ case <-time.After(25 * time.Millisecond):
405+ }
406+ if time.Now().After(deadline) {
407+ return fmt.Errorf("firecracker API socket %s not ready after 10s", sock)
408+ }
409+ }
410+}
411+
412+// Stop implements sandbox.Sandbox.
413+func (s *vmSandbox) Stop(ctx context.Context) error {
414+ s.mu.Lock()
415+ defer s.mu.Unlock()
416+ if s.stopped {
417+ return nil
418+ }
419+ if !s.started {
420+ s.stopped = true
421+ s.releaseLocked()
422+ return nil
423+ }
424+ return s.cleanupLocked(context.WithoutCancel(ctx))
425+}
426+
427+func (s *vmSandbox) releaseLocked() {
428+ s.stopped = true
429+ s.d.release(s.slot)
430+}
431+
432+// cleanupLocked removes everything the VM created, in dependency order, and
433+// keeps going after an error. The state record and the slot are released only
434+// when everything is gone, so that a half-failed cleanup is retried by Reap
435+// rather than forgotten and the slot (hence TAP name) is not reused meanwhile.
436+func (s *vmSandbox) cleanupLocked(ctx context.Context) error {
437+ log := zerowrap.FromCtx(ctx)
438+ d := s.d
439+ var errs []error
440+ if s.conn != nil {
441+ _ = s.conn.Close()
442+ s.conn = nil
443+ }
444+ if s.listener != nil {
445+ _ = s.listener.Close()
446+ s.listener = nil
447+ }
448+ if s.cmd != nil && s.cmd.Process != nil {
449+ killAndWait(s.cmd.Process.Pid, 5*time.Second)
450+ select {
451+ case <-s.exited:
452+ case <-time.After(5 * time.Second):
453+ errs = append(errs, fmt.Errorf("firecracker pid %d did not exit", s.cmd.Process.Pid))
454+ }
455+ }
456+ if s.attached {
457+ if err := d.net.Detach(ctx, s.tap, s.gateway); err != nil {
458+ errs = append(errs, err)
459+ }
460+ if err := deleteTap(ctx, d.opt.Run, s.tap); err != nil {
461+ errs = append(errs, err)
462+ } else {
463+ s.attached = false
464+ }
465+ }
466+ if s.lay.dir != "" {
467+ if err := removeAll(s.lay.dir); err != nil {
468+ errs = append(errs, err)
469+ }
470+ }
471+ if len(errs) == 0 {
472+ if err := d.store.remove(s.rec.ID); err != nil {
473+ errs = append(errs, err)
474+ }
475+ }
476+ if len(errs) == 0 {
477+ s.releaseLocked()
478+ return nil
479+ }
480+ s.stopped = true // do not retry from here; Reap at the next start will
481+ err := errors.Join(errs...)
482+ log.Error().Err(err).Str("vm", s.rec.ID).Msg("micro-VM cleanup incomplete; the next runner start will retry")
483+ return err
484+}
485+
486+// removeAll is os.RemoveAll that also reports a directory that is still there.
487+func removeAll(dir string) error {
488+ if err := os.RemoveAll(dir); err != nil {
489+ return err
490+ }
491+ if _, err := os.Lstat(dir); err == nil {
492+ return fmt.Errorf("%s still exists after removal", dir)
493+ }
494+ return nil
495+}
A internal/sandbox/vm/images.go +119 -0
@@ -0,0 +1,119 @@
1+// SPDX-License-Identifier: MIT
2+
3+package vm
4+
5+import (
6+ "context"
7+ "crypto/sha256"
8+ "encoding/hex"
9+ "fmt"
10+ "io"
11+ "os"
12+ "path/filepath"
13+
14+ "bnema.dev/ermite-runner/internal/cmdrun"
15+ "bnema.dev/ermite-runner/internal/sandbox"
16+)
17+
18+// ensureExecDisk builds (or reuses) the small read-only ext4 disk that carries
19+// the runner's own executable: the guest root during boot, and /opt/ermite
20+// afterwards. It is named after the executable's hash, so a runner upgrade
21+// builds a new one and no image rebuild is needed; older ones are removed.
22+// mkfs.ext4 -d needs no root.
23+func ensureExecDisk(ctx context.Context, run cmdrun.Runner, exe, stateDir string) (string, error) {
24+ sum, size, err := hashFile(exe)
25+ if err != nil {
26+ return "", fmt.Errorf("hash executable: %w", err)
27+ }
28+ cache := filepath.Join(stateDir, "cache")
29+ if err := os.MkdirAll(cache, 0o755); err != nil {
30+ return "", err
31+ }
32+ name := "execdisk-" + sum[:16] + ".ext4"
33+ dest := filepath.Join(cache, name)
34+ if _, err := os.Stat(dest); err == nil {
35+ return dest, nil
36+ }
37+ // Drop disks of other executable versions.
38+ if olds, _ := filepath.Glob(filepath.Join(cache, "execdisk-*")); len(olds) > 0 {
39+ for _, o := range olds {
40+ _ = os.Remove(o)
41+ }
42+ }
43+
44+ tmp, err := os.MkdirTemp(cache, "execroot-")
45+ if err != nil {
46+ return "", err
47+ }
48+ defer func() { _ = os.RemoveAll(tmp) }()
49+ for _, d := range []string{"proc", "sys", "dev", "mnt/base", "mnt/scratch", "mnt/newroot"} {
50+ if err := os.MkdirAll(filepath.Join(tmp, d), 0o755); err != nil {
51+ return "", err
52+ }
53+ }
54+ if err := copyFile(exe, filepath.Join(tmp, sandbox.GuestExecName)); err != nil {
55+ return "", fmt.Errorf("copy executable: %w", err)
56+ }
57+ if err := os.Chmod(filepath.Join(tmp, sandbox.GuestExecName), 0o755); err != nil {
58+ return "", err
59+ }
60+
61+ part := dest + ".part"
62+ _ = os.Remove(part)
63+ f, err := os.OpenFile(part, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
64+ if err != nil {
65+ return "", err
66+ }
67+ // The executable, ext4 overhead and some room; the disk is read-only.
68+ mib := size/(1<<20)*115/100 + 16
69+ if err := f.Truncate(mib << 20); err != nil {
70+ _ = f.Close()
71+ return "", err
72+ }
73+ _ = f.Close()
74+ if _, err := run.Run(ctx, nil, "mkfs.ext4", "-q", "-F", "-L", "ermite-exec",
75+ "-O", "^has_journal", "-E", "root_owner=0:0", "-d", tmp, part); err != nil {
76+ _ = os.Remove(part)
77+ return "", fmt.Errorf("build exec disk: %w", err)
78+ }
79+ if err := os.Rename(part, dest); err != nil {
80+ return "", err
81+ }
82+ return dest, nil
83+}
84+
85+// makeScratch creates the per-job sparse scratch disk of sizeMiB and formats it.
86+func makeScratch(ctx context.Context, run cmdrun.Runner, path string, sizeMiB int) error {
87+ f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
88+ if err != nil {
89+ return err
90+ }
91+ if err := f.Truncate(int64(sizeMiB) << 20); err != nil {
92+ _ = f.Close()
93+ return err
94+ }
95+ if err := f.Close(); err != nil {
96+ return err
97+ }
98+ // No journal and no reserved blocks: it is a throw-away disk. Lazy table
99+ // initialisation keeps creation instant and the file sparse.
100+ if _, err := run.Run(ctx, nil, "mkfs.ext4", "-q", "-F", "-L", "ermite-scratch", "-m", "0",
101+ "-O", "^has_journal,^resize_inode", "-E", "lazy_itable_init=1,nodiscard", path); err != nil {
102+ return fmt.Errorf("format scratch disk: %w", err)
103+ }
104+ return nil
105+}
106+
107+func hashFile(path string) (sum string, size int64, err error) {
108+ f, err := os.Open(path)
109+ if err != nil {
110+ return "", 0, err
111+ }
112+ defer func() { _ = f.Close() }()
113+ h := sha256.New()
114+ n, err := io.Copy(h, f)
115+ if err != nil {
116+ return "", 0, err
117+ }
118+ return hex.EncodeToString(h.Sum(nil)), n, nil
119+}
@@ -0,0 +1,119 @@
1+// SPDX-License-Identifier: MIT
2+
3+package vm
4+
5+import (
6+ "context"
7+ "crypto/sha256"
8+ "encoding/hex"
9+ "fmt"
10+ "io"
11+ "os"
12+ "path/filepath"
13+
14+ "bnema.dev/ermite-runner/internal/cmdrun"
15+ "bnema.dev/ermite-runner/internal/sandbox"
16+)
17+
18+// ensureExecDisk builds (or reuses) the small read-only ext4 disk that carries
19+// the runner's own executable: the guest root during boot, and /opt/ermite
20+// afterwards. It is named after the executable's hash, so a runner upgrade
21+// builds a new one and no image rebuild is needed; older ones are removed.
22+// mkfs.ext4 -d needs no root.
23+func ensureExecDisk(ctx context.Context, run cmdrun.Runner, exe, stateDir string) (string, error) {
24+ sum, size, err := hashFile(exe)
25+ if err != nil {
26+ return "", fmt.Errorf("hash executable: %w", err)
27+ }
28+ cache := filepath.Join(stateDir, "cache")
29+ if err := os.MkdirAll(cache, 0o755); err != nil {
30+ return "", err
31+ }
32+ name := "execdisk-" + sum[:16] + ".ext4"
33+ dest := filepath.Join(cache, name)
34+ if _, err := os.Stat(dest); err == nil {
35+ return dest, nil
36+ }
37+ // Drop disks of other executable versions.
38+ if olds, _ := filepath.Glob(filepath.Join(cache, "execdisk-*")); len(olds) > 0 {
39+ for _, o := range olds {
40+ _ = os.Remove(o)
41+ }
42+ }
43+
44+ tmp, err := os.MkdirTemp(cache, "execroot-")
45+ if err != nil {
46+ return "", err
47+ }
48+ defer func() { _ = os.RemoveAll(tmp) }()
49+ for _, d := range []string{"proc", "sys", "dev", "mnt/base", "mnt/scratch", "mnt/newroot"} {
50+ if err := os.MkdirAll(filepath.Join(tmp, d), 0o755); err != nil {
51+ return "", err
52+ }
53+ }
54+ if err := copyFile(exe, filepath.Join(tmp, sandbox.GuestExecName)); err != nil {
55+ return "", fmt.Errorf("copy executable: %w", err)
56+ }
57+ if err := os.Chmod(filepath.Join(tmp, sandbox.GuestExecName), 0o755); err != nil {
58+ return "", err
59+ }
60+
61+ part := dest + ".part"
62+ _ = os.Remove(part)
63+ f, err := os.OpenFile(part, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
64+ if err != nil {
65+ return "", err
66+ }
67+ // The executable, ext4 overhead and some room; the disk is read-only.
68+ mib := size/(1<<20)*115/100 + 16
69+ if err := f.Truncate(mib << 20); err != nil {
70+ _ = f.Close()
71+ return "", err
72+ }
73+ _ = f.Close()
74+ if _, err := run.Run(ctx, nil, "mkfs.ext4", "-q", "-F", "-L", "ermite-exec",
75+ "-O", "^has_journal", "-E", "root_owner=0:0", "-d", tmp, part); err != nil {
76+ _ = os.Remove(part)
77+ return "", fmt.Errorf("build exec disk: %w", err)
78+ }
79+ if err := os.Rename(part, dest); err != nil {
80+ return "", err
81+ }
82+ return dest, nil
83+}
84+
85+// makeScratch creates the per-job sparse scratch disk of sizeMiB and formats it.
86+func makeScratch(ctx context.Context, run cmdrun.Runner, path string, sizeMiB int) error {
87+ f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
88+ if err != nil {
89+ return err
90+ }
91+ if err := f.Truncate(int64(sizeMiB) << 20); err != nil {
92+ _ = f.Close()
93+ return err
94+ }
95+ if err := f.Close(); err != nil {
96+ return err
97+ }
98+ // No journal and no reserved blocks: it is a throw-away disk. Lazy table
99+ // initialisation keeps creation instant and the file sparse.
100+ if _, err := run.Run(ctx, nil, "mkfs.ext4", "-q", "-F", "-L", "ermite-scratch", "-m", "0",
101+ "-O", "^has_journal,^resize_inode", "-E", "lazy_itable_init=1,nodiscard", path); err != nil {
102+ return fmt.Errorf("format scratch disk: %w", err)
103+ }
104+ return nil
105+}
106+
107+func hashFile(path string) (sum string, size int64, err error) {
108+ f, err := os.Open(path)
109+ if err != nil {
110+ return "", 0, err
111+ }
112+ defer func() { _ = f.Close() }()
113+ h := sha256.New()
114+ n, err := io.Copy(h, f)
115+ if err != nil {
116+ return "", 0, err
117+ }
118+ return hex.EncodeToString(h.Sum(nil)), n, nil
119+}
A internal/sandbox/vm/layout.go +112 -0
@@ -0,0 +1,112 @@
1+// SPDX-License-Identifier: MIT
2+
3+package vm
4+
5+import (
6+ "errors"
7+ "fmt"
8+ "io"
9+ "os"
10+ "path/filepath"
11+ "syscall"
12+)
13+
14+// layout is where the files of one VM live and how Firecracker names them.
15+//
16+// With the jailer, Firecracker sees only its chroot: <jailBase>/firecracker/<id>/root,
17+// so shared files are hard-linked into it and named "/name"; per-job files are
18+// created there and handed to the jailed uid. Without the jailer (development
19+// and tests), Firecracker sees the real filesystem: shared files are used in
20+// place and per-job files live in <stateDir>/vm/<id>.
21+type layout struct {
22+ jailed bool
23+ root string // host path of the VM's working directory
24+ dir string // the directory removed whole at cleanup
25+ uid int
26+}
27+
28+// newLayout: execName is the base name of the Firecracker binary, which the
29+// jailer uses as the first directory below its chroot base.
30+func newLayout(stateDir, execName, id string, jailed bool, uid int) layout {
31+ if jailed {
32+ dir := filepath.Join(stateDir, "jail", execName, id)
33+ return layout{jailed: true, root: filepath.Join(dir, "root"), dir: dir, uid: uid}
34+ }
35+ dir := filepath.Join(stateDir, "vm", id)
36+ return layout{root: dir, dir: dir, uid: uid}
37+}
38+
39+// jailBase is the --chroot-base-dir given to the jailer.
40+func jailBase(stateDir string) string { return filepath.Join(stateDir, "jail") }
41+
42+// host returns the host path of a per-job file called name.
43+func (l layout) host(name string) string { return filepath.Join(l.root, name) }
44+
45+// fc returns the path Firecracker must be given for a per-job file.
46+func (l layout) fc(name string) string {
47+ if l.jailed {
48+ return "/" + name
49+ }
50+ return l.host(name)
51+}
52+
53+// share makes the read-only shared file src available to Firecracker and
54+// returns its path for Firecracker. In a jail it is hard-linked, which requires
55+// the same filesystem as the state directory, and src must be readable by the
56+// jailed uid (the link shares the inode: it is never chowned).
57+func (l layout) share(src, name string) (string, error) {
58+ if !l.jailed {
59+ return src, nil
60+ }
61+ dst := l.host(name)
62+ if err := os.Link(src, dst); err != nil {
63+ if errors.Is(err, syscall.EXDEV) {
64+ return "", fmt.Errorf("cannot link %s into the jail: it must be on the same filesystem as the state directory (hard link across filesystems)", src)
65+ }
66+ return "", fmt.Errorf("link %s into the jail: %w", src, err)
67+ }
68+ return "/" + name, nil
69+}
70+
71+// own hands a per-job file in the jail to the jailed uid; no-op without jailer.
72+func (l layout) own(name string) error {
73+ if !l.jailed {
74+ return nil
75+ }
76+ return os.Chown(l.host(name), l.uid, l.uid)
77+}
78+
79+// checkReadable verifies that other users can read the shared file, which the
80+// jailed uid needs. Called once per file at driver start for jailed mode.
81+func checkWorldReadable(path string) error {
82+ st, err := os.Stat(path)
83+ if err != nil {
84+ return err
85+ }
86+ if st.Mode().Perm()&0o004 == 0 {
87+ return fmt.Errorf("%s is not world-readable (mode %v): the jailed uid could not open it", path, st.Mode().Perm())
88+ }
89+ return nil
90+}
91+
92+// copyFile copies src to dst (mode 0600) and fsyncs it.
93+func copyFile(src, dst string) error {
94+ in, err := os.Open(src)
95+ if err != nil {
96+ return err
97+ }
98+ defer func() { _ = in.Close() }()
99+ out, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
100+ if err != nil {
101+ return err
102+ }
103+ if _, err := io.Copy(out, in); err != nil {
104+ _ = out.Close()
105+ return err
106+ }
107+ if err := out.Sync(); err != nil {
108+ _ = out.Close()
109+ return err
110+ }
111+ return out.Close()
112+}
@@ -0,0 +1,112 @@
1+// SPDX-License-Identifier: MIT
2+
3+package vm
4+
5+import (
6+ "errors"
7+ "fmt"
8+ "io"
9+ "os"
10+ "path/filepath"
11+ "syscall"
12+)
13+
14+// layout is where the files of one VM live and how Firecracker names them.
15+//
16+// With the jailer, Firecracker sees only its chroot: <jailBase>/firecracker/<id>/root,
17+// so shared files are hard-linked into it and named "/name"; per-job files are
18+// created there and handed to the jailed uid. Without the jailer (development
19+// and tests), Firecracker sees the real filesystem: shared files are used in
20+// place and per-job files live in <stateDir>/vm/<id>.
21+type layout struct {
22+ jailed bool
23+ root string // host path of the VM's working directory
24+ dir string // the directory removed whole at cleanup
25+ uid int
26+}
27+
28+// newLayout: execName is the base name of the Firecracker binary, which the
29+// jailer uses as the first directory below its chroot base.
30+func newLayout(stateDir, execName, id string, jailed bool, uid int) layout {
31+ if jailed {
32+ dir := filepath.Join(stateDir, "jail", execName, id)
33+ return layout{jailed: true, root: filepath.Join(dir, "root"), dir: dir, uid: uid}
34+ }
35+ dir := filepath.Join(stateDir, "vm", id)
36+ return layout{root: dir, dir: dir, uid: uid}
37+}
38+
39+// jailBase is the --chroot-base-dir given to the jailer.
40+func jailBase(stateDir string) string { return filepath.Join(stateDir, "jail") }
41+
42+// host returns the host path of a per-job file called name.
43+func (l layout) host(name string) string { return filepath.Join(l.root, name) }
44+
45+// fc returns the path Firecracker must be given for a per-job file.
46+func (l layout) fc(name string) string {
47+ if l.jailed {
48+ return "/" + name
49+ }
50+ return l.host(name)
51+}
52+
53+// share makes the read-only shared file src available to Firecracker and
54+// returns its path for Firecracker. In a jail it is hard-linked, which requires
55+// the same filesystem as the state directory, and src must be readable by the
56+// jailed uid (the link shares the inode: it is never chowned).
57+func (l layout) share(src, name string) (string, error) {
58+ if !l.jailed {
59+ return src, nil
60+ }
61+ dst := l.host(name)
62+ if err := os.Link(src, dst); err != nil {
63+ if errors.Is(err, syscall.EXDEV) {
64+ return "", fmt.Errorf("cannot link %s into the jail: it must be on the same filesystem as the state directory (hard link across filesystems)", src)
65+ }
66+ return "", fmt.Errorf("link %s into the jail: %w", src, err)
67+ }
68+ return "/" + name, nil
69+}
70+
71+// own hands a per-job file in the jail to the jailed uid; no-op without jailer.
72+func (l layout) own(name string) error {
73+ if !l.jailed {
74+ return nil
75+ }
76+ return os.Chown(l.host(name), l.uid, l.uid)
77+}
78+
79+// checkReadable verifies that other users can read the shared file, which the
80+// jailed uid needs. Called once per file at driver start for jailed mode.
81+func checkWorldReadable(path string) error {
82+ st, err := os.Stat(path)
83+ if err != nil {
84+ return err
85+ }
86+ if st.Mode().Perm()&0o004 == 0 {
87+ return fmt.Errorf("%s is not world-readable (mode %v): the jailed uid could not open it", path, st.Mode().Perm())
88+ }
89+ return nil
90+}
91+
92+// copyFile copies src to dst (mode 0600) and fsyncs it.
93+func copyFile(src, dst string) error {
94+ in, err := os.Open(src)
95+ if err != nil {
96+ return err
97+ }
98+ defer func() { _ = in.Close() }()
99+ out, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
100+ if err != nil {
101+ return err
102+ }
103+ if _, err := io.Copy(out, in); err != nil {
104+ _ = out.Close()
105+ return err
106+ }
107+ if err := out.Sync(); err != nil {
108+ _ = out.Close()
109+ return err
110+ }
111+ return out.Close()
112+}
A internal/sandbox/vm/proc.go +94 -0
@@ -0,0 +1,94 @@
1+// SPDX-License-Identifier: MIT
2+
3+package vm
4+
5+import (
6+ "bytes"
7+ "os"
8+ "path/filepath"
9+ "strconv"
10+ "strings"
11+ "sync"
12+ "syscall"
13+ "time"
14+
15+ "bnema.dev/ermite-runner/internal/sandbox"
16+)
17+
18+// firecrackerProcs scans a /proc tree for Firecracker processes started by this
19+// runner: argv[0] is "firecracker" and "--id" is followed by one of our ids
20+// (see newID). Processes of other users of Firecracker are never matched. It
21+// finds orphans even when the state directory was lost.
22+func firecrackerProcs(procDir string) map[int]string {
23+ out := map[int]string{}
24+ ents, err := os.ReadDir(procDir)
25+ if err != nil {
26+ return out
27+ }
28+ for _, e := range ents {
29+ pid, err := strconv.Atoi(e.Name())
30+ if err != nil || pid <= 1 {
31+ continue
32+ }
33+ b, err := os.ReadFile(filepath.Join(procDir, e.Name(), "cmdline"))
34+ if err != nil || len(b) == 0 {
35+ continue
36+ }
37+ args := strings.Split(strings.TrimRight(string(b), "\x00"), "\x00")
38+ if filepath.Base(args[0]) != "firecracker" {
39+ continue
40+ }
41+ for i := 1; i+1 < len(args); i++ {
42+ if args[i] == "--id" && sandbox.ValidID(args[i+1]) {
43+ out[pid] = args[i+1]
44+ }
45+ }
46+ }
47+ return out
48+}
49+
50+// ring keeps the last N bytes written to it: the VMM's stdout/stderr, which
51+// carries the guest serial console, for diagnostics when a boot fails.
52+type ring struct {
53+ mu sync.Mutex
54+ max int
55+ buf []byte
56+}
57+
58+func newRing(max int) *ring { return &ring{max: max} }
59+
60+func (r *ring) Write(p []byte) (int, error) {
61+ r.mu.Lock()
62+ defer r.mu.Unlock()
63+ r.buf = append(r.buf, p...)
64+ if len(r.buf) > r.max {
65+ r.buf = append([]byte(nil), r.buf[len(r.buf)-r.max:]...)
66+ }
67+ return len(p), nil
68+}
69+
70+// String returns the retained tail, trimmed.
71+func (r *ring) String() string {
72+ r.mu.Lock()
73+ defer r.mu.Unlock()
74+ return string(bytes.TrimSpace(r.buf))
75+}
76+
77+// killAndWait sends SIGKILL to pid and waits until it is gone (zombies of
78+// children of this process are reaped by their own Wait).
79+func killAndWait(pid int, timeout time.Duration) {
80+ _ = syscall.Kill(pid, syscall.SIGKILL)
81+ deadline := time.Now().Add(timeout)
82+ for time.Now().Before(deadline) {
83+ if err := syscall.Kill(pid, 0); err != nil {
84+ return
85+ }
86+ // A zombie child still answers kill(0); check its state.
87+ if b, err := os.ReadFile("/proc/" + strconv.Itoa(pid) + "/stat"); err == nil {
88+ if i := bytes.LastIndexByte(b, ')'); i >= 0 && i+2 < len(b) && b[i+2] == 'Z' {
89+ return
90+ }
91+ }
92+ time.Sleep(20 * time.Millisecond)
93+ }
94+}
@@ -0,0 +1,94 @@
1+// SPDX-License-Identifier: MIT
2+
3+package vm
4+
5+import (
6+ "bytes"
7+ "os"
8+ "path/filepath"
9+ "strconv"
10+ "strings"
11+ "sync"
12+ "syscall"
13+ "time"
14+
15+ "bnema.dev/ermite-runner/internal/sandbox"
16+)
17+
18+// firecrackerProcs scans a /proc tree for Firecracker processes started by this
19+// runner: argv[0] is "firecracker" and "--id" is followed by one of our ids
20+// (see newID). Processes of other users of Firecracker are never matched. It
21+// finds orphans even when the state directory was lost.
22+func firecrackerProcs(procDir string) map[int]string {
23+ out := map[int]string{}
24+ ents, err := os.ReadDir(procDir)
25+ if err != nil {
26+ return out
27+ }
28+ for _, e := range ents {
29+ pid, err := strconv.Atoi(e.Name())
30+ if err != nil || pid <= 1 {
31+ continue
32+ }
33+ b, err := os.ReadFile(filepath.Join(procDir, e.Name(), "cmdline"))
34+ if err != nil || len(b) == 0 {
35+ continue
36+ }
37+ args := strings.Split(strings.TrimRight(string(b), "\x00"), "\x00")
38+ if filepath.Base(args[0]) != "firecracker" {
39+ continue
40+ }
41+ for i := 1; i+1 < len(args); i++ {
42+ if args[i] == "--id" && sandbox.ValidID(args[i+1]) {
43+ out[pid] = args[i+1]
44+ }
45+ }
46+ }
47+ return out
48+}
49+
50+// ring keeps the last N bytes written to it: the VMM's stdout/stderr, which
51+// carries the guest serial console, for diagnostics when a boot fails.
52+type ring struct {
53+ mu sync.Mutex
54+ max int
55+ buf []byte
56+}
57+
58+func newRing(max int) *ring { return &ring{max: max} }
59+
60+func (r *ring) Write(p []byte) (int, error) {
61+ r.mu.Lock()
62+ defer r.mu.Unlock()
63+ r.buf = append(r.buf, p...)
64+ if len(r.buf) > r.max {
65+ r.buf = append([]byte(nil), r.buf[len(r.buf)-r.max:]...)
66+ }
67+ return len(p), nil
68+}
69+
70+// String returns the retained tail, trimmed.
71+func (r *ring) String() string {
72+ r.mu.Lock()
73+ defer r.mu.Unlock()
74+ return string(bytes.TrimSpace(r.buf))
75+}
76+
77+// killAndWait sends SIGKILL to pid and waits until it is gone (zombies of
78+// children of this process are reaped by their own Wait).
79+func killAndWait(pid int, timeout time.Duration) {
80+ _ = syscall.Kill(pid, syscall.SIGKILL)
81+ deadline := time.Now().Add(timeout)
82+ for time.Now().Before(deadline) {
83+ if err := syscall.Kill(pid, 0); err != nil {
84+ return
85+ }
86+ // A zombie child still answers kill(0); check its state.
87+ if b, err := os.ReadFile("/proc/" + strconv.Itoa(pid) + "/stat"); err == nil {
88+ if i := bytes.LastIndexByte(b, ')'); i >= 0 && i+2 < len(b) && b[i+2] == 'Z' {
89+ return
90+ }
91+ }
92+ time.Sleep(20 * time.Millisecond)
93+ }
94+}
A internal/sandbox/vm/reap.go +98 -0
@@ -0,0 +1,98 @@
1+// SPDX-License-Identifier: MIT
2+
3+package vm
4+
5+import (
6+ "context"
7+ "errors"
8+ "fmt"
9+ "os"
10+ "path/filepath"
11+ "time"
12+
13+ "github.com/bnema/zerowrap"
14+
15+ "bnema.dev/ermite-runner/internal/netpolicy"
16+ "bnema.dev/ermite-runner/internal/sandbox"
17+)
18+
19+// Reap implements sandbox.Driver. It removes what a previous run of the runner
20+// left: Firecracker processes (by state record and by scanning /proc for our
21+// "--id erXXXXXXXX"), TAP devices named ertap<N>, the nftables table, and VM
22+// directories. It relies on the state-directory lock taken in New: no other
23+// runner instance can own what it removes.
24+func (d *Driver) Reap(ctx context.Context) error {
25+ log := zerowrap.FromCtx(ctx)
26+ var errs []error
27+
28+ recs, err := d.store.list()
29+ if err != nil {
30+ errs = append(errs, err)
31+ }
32+ procs := firecrackerProcs(d.opt.ProcDir)
33+ killed := map[int]bool{}
34+ killProc := func(pid int, id string) {
35+ if killed[pid] {
36+ return
37+ }
38+ killed[pid] = true
39+ log.Warn().Int("pid", pid).Str("vm", id).Msg("reaping leftover firecracker process")
40+ killAndWait(pid, 5*time.Second)
41+ }
42+ // 1. Processes: by record, but only if the pid is still one of ours (pids
43+ // are reused), then any other orphan found in /proc.
44+ for _, r := range recs {
45+ if r.PID > 0 && procs[r.PID] == r.ID {
46+ killProc(r.PID, r.ID)
47+ }
48+ }
49+ for pid, id := range procs {
50+ killProc(pid, id)
51+ }
52+
53+ // 2. Network: the nft table first (it names interfaces), then every TAP.
54+ if err := d.net.Reset(ctx); err != nil {
55+ errs = append(errs, fmt.Errorf("reset nftables table %s: %w", netpolicy.Table, err))
56+ }
57+ taps, err := leftoverTaps(ctx, d.opt.Run)
58+ if err != nil {
59+ errs = append(errs, fmt.Errorf("list interfaces: %w", err))
60+ }
61+ for _, t := range taps {
62+ log.Warn().Str("tap", t).Msg("removing leftover TAP device")
63+ if err := deleteTap(ctx, d.opt.Run, t); err != nil {
64+ errs = append(errs, err)
65+ }
66+ }
67+
68+ // 3. Files: directories of recorded VMs, then any directory with one of our
69+ // ids that has no record (crash before the record was written is
70+ // impossible, but a deleted state dir is not).
71+ // The jailer's chroot base has one directory per exec-file name.
72+ parents := []string{filepath.Join(d.cfg.StateDir, "vm")}
73+ jails, _ := os.ReadDir(filepath.Join(d.cfg.StateDir, "jail"))
74+ for _, j := range jails {
75+ if j.IsDir() {
76+ parents = append(parents, filepath.Join(d.cfg.StateDir, "jail", j.Name()))
77+ }
78+ }
79+ for _, parent := range parents {
80+ ents, _ := os.ReadDir(parent)
81+ for _, e := range ents {
82+ if sandbox.ValidID(e.Name()) {
83+ log.Warn().Str("dir", filepath.Join(parent, e.Name())).Msg("removing leftover VM directory")
84+ if err := removeAll(filepath.Join(parent, e.Name())); err != nil {
85+ errs = append(errs, err)
86+ }
87+ }
88+ }
89+ }
90+ if len(errs) == 0 {
91+ for _, r := range recs {
92+ if err := d.store.remove(r.ID); err != nil {
93+ errs = append(errs, err)
94+ }
95+ }
96+ }
97+ return errors.Join(errs...)
98+}
@@ -0,0 +1,98 @@
1+// SPDX-License-Identifier: MIT
2+
3+package vm
4+
5+import (
6+ "context"
7+ "errors"
8+ "fmt"
9+ "os"
10+ "path/filepath"
11+ "time"
12+
13+ "github.com/bnema/zerowrap"
14+
15+ "bnema.dev/ermite-runner/internal/netpolicy"
16+ "bnema.dev/ermite-runner/internal/sandbox"
17+)
18+
19+// Reap implements sandbox.Driver. It removes what a previous run of the runner
20+// left: Firecracker processes (by state record and by scanning /proc for our
21+// "--id erXXXXXXXX"), TAP devices named ertap<N>, the nftables table, and VM
22+// directories. It relies on the state-directory lock taken in New: no other
23+// runner instance can own what it removes.
24+func (d *Driver) Reap(ctx context.Context) error {
25+ log := zerowrap.FromCtx(ctx)
26+ var errs []error
27+
28+ recs, err := d.store.list()
29+ if err != nil {
30+ errs = append(errs, err)
31+ }
32+ procs := firecrackerProcs(d.opt.ProcDir)
33+ killed := map[int]bool{}
34+ killProc := func(pid int, id string) {
35+ if killed[pid] {
36+ return
37+ }
38+ killed[pid] = true
39+ log.Warn().Int("pid", pid).Str("vm", id).Msg("reaping leftover firecracker process")
40+ killAndWait(pid, 5*time.Second)
41+ }
42+ // 1. Processes: by record, but only if the pid is still one of ours (pids
43+ // are reused), then any other orphan found in /proc.
44+ for _, r := range recs {
45+ if r.PID > 0 && procs[r.PID] == r.ID {
46+ killProc(r.PID, r.ID)
47+ }
48+ }
49+ for pid, id := range procs {
50+ killProc(pid, id)
51+ }
52+
53+ // 2. Network: the nft table first (it names interfaces), then every TAP.
54+ if err := d.net.Reset(ctx); err != nil {
55+ errs = append(errs, fmt.Errorf("reset nftables table %s: %w", netpolicy.Table, err))
56+ }
57+ taps, err := leftoverTaps(ctx, d.opt.Run)
58+ if err != nil {
59+ errs = append(errs, fmt.Errorf("list interfaces: %w", err))
60+ }
61+ for _, t := range taps {
62+ log.Warn().Str("tap", t).Msg("removing leftover TAP device")
63+ if err := deleteTap(ctx, d.opt.Run, t); err != nil {
64+ errs = append(errs, err)
65+ }
66+ }
67+
68+ // 3. Files: directories of recorded VMs, then any directory with one of our
69+ // ids that has no record (crash before the record was written is
70+ // impossible, but a deleted state dir is not).
71+ // The jailer's chroot base has one directory per exec-file name.
72+ parents := []string{filepath.Join(d.cfg.StateDir, "vm")}
73+ jails, _ := os.ReadDir(filepath.Join(d.cfg.StateDir, "jail"))
74+ for _, j := range jails {
75+ if j.IsDir() {
76+ parents = append(parents, filepath.Join(d.cfg.StateDir, "jail", j.Name()))
77+ }
78+ }
79+ for _, parent := range parents {
80+ ents, _ := os.ReadDir(parent)
81+ for _, e := range ents {
82+ if sandbox.ValidID(e.Name()) {
83+ log.Warn().Str("dir", filepath.Join(parent, e.Name())).Msg("removing leftover VM directory")
84+ if err := removeAll(filepath.Join(parent, e.Name())); err != nil {
85+ errs = append(errs, err)
86+ }
87+ }
88+ }
89+ }
90+ if len(errs) == 0 {
91+ for _, r := range recs {
92+ if err := d.store.remove(r.ID); err != nil {
93+ errs = append(errs, err)
94+ }
95+ }
96+ }
97+ return errors.Join(errs...)
98+}
A internal/sandbox/vm/state.go +82 -0
@@ -0,0 +1,82 @@
1+// SPDX-License-Identifier: MIT
2+
3+package vm
4+
5+import (
6+ "encoding/json"
7+ "errors"
8+ "os"
9+ "path/filepath"
10+ "strings"
11+)
12+
13+// record is the on-disk trace of one VM. It is written before the first
14+// resource is created and rewritten as resources appear, so that Reap can
15+// remove them after a crash of the runner. The record is deleted last.
16+type record struct {
17+ ID string `json:"id"`
18+ Slot int `json:"slot"`
19+ Job string `json:"job,omitempty"`
20+ PID int `json:"pid,omitempty"`
21+ Tap string `json:"tap,omitempty"`
22+ Gate string `json:"gateway,omitempty"`
23+ Dir string `json:"dir,omitempty"` // the jail directory of this VM (removed whole)
24+ Files []string `json:"files,omitempty"`
25+}
26+
27+type store struct{ dir string }
28+
29+func (s store) path(id string) string { return filepath.Join(s.dir, id+".json") }
30+
31+func (s store) save(r record) error {
32+ if err := os.MkdirAll(s.dir, 0o700); err != nil {
33+ return err
34+ }
35+ b, err := json.Marshal(r)
36+ if err != nil {
37+ return err
38+ }
39+ tmp := s.path(r.ID) + ".tmp"
40+ if err := os.WriteFile(tmp, b, 0o600); err != nil {
41+ return err
42+ }
43+ return os.Rename(tmp, s.path(r.ID))
44+}
45+
46+func (s store) remove(id string) error {
47+ if err := os.Remove(s.path(id)); err != nil && !errors.Is(err, os.ErrNotExist) {
48+ return err
49+ }
50+ return nil
51+}
52+
53+// list returns every readable record; unreadable or corrupt files are returned
54+// as ID-only records so that Reap still deletes them.
55+func (s store) list() ([]record, error) {
56+ ents, err := os.ReadDir(s.dir)
57+ if errors.Is(err, os.ErrNotExist) {
58+ return nil, nil
59+ }
60+ if err != nil {
61+ return nil, err
62+ }
63+ var out []record
64+ for _, e := range ents {
65+ name := e.Name()
66+ if e.IsDir() || !strings.HasSuffix(name, ".json") {
67+ // A *.tmp left by a crash mid-write.
68+ if strings.HasSuffix(name, ".tmp") {
69+ _ = os.Remove(filepath.Join(s.dir, name))
70+ }
71+ continue
72+ }
73+ id := strings.TrimSuffix(name, ".json")
74+ var r record
75+ if b, err := os.ReadFile(filepath.Join(s.dir, name)); err == nil && json.Unmarshal(b, &r) == nil && r.ID == id {
76+ out = append(out, r)
77+ continue
78+ }
79+ out = append(out, record{ID: id, Slot: -1})
80+ }
81+ return out, nil
82+}
@@ -0,0 +1,82 @@
1+// SPDX-License-Identifier: MIT
2+
3+package vm
4+
5+import (
6+ "encoding/json"
7+ "errors"
8+ "os"
9+ "path/filepath"
10+ "strings"
11+)
12+
13+// record is the on-disk trace of one VM. It is written before the first
14+// resource is created and rewritten as resources appear, so that Reap can
15+// remove them after a crash of the runner. The record is deleted last.
16+type record struct {
17+ ID string `json:"id"`
18+ Slot int `json:"slot"`
19+ Job string `json:"job,omitempty"`
20+ PID int `json:"pid,omitempty"`
21+ Tap string `json:"tap,omitempty"`
22+ Gate string `json:"gateway,omitempty"`
23+ Dir string `json:"dir,omitempty"` // the jail directory of this VM (removed whole)
24+ Files []string `json:"files,omitempty"`
25+}
26+
27+type store struct{ dir string }
28+
29+func (s store) path(id string) string { return filepath.Join(s.dir, id+".json") }
30+
31+func (s store) save(r record) error {
32+ if err := os.MkdirAll(s.dir, 0o700); err != nil {
33+ return err
34+ }
35+ b, err := json.Marshal(r)
36+ if err != nil {
37+ return err
38+ }
39+ tmp := s.path(r.ID) + ".tmp"
40+ if err := os.WriteFile(tmp, b, 0o600); err != nil {
41+ return err
42+ }
43+ return os.Rename(tmp, s.path(r.ID))
44+}
45+
46+func (s store) remove(id string) error {
47+ if err := os.Remove(s.path(id)); err != nil && !errors.Is(err, os.ErrNotExist) {
48+ return err
49+ }
50+ return nil
51+}
52+
53+// list returns every readable record; unreadable or corrupt files are returned
54+// as ID-only records so that Reap still deletes them.
55+func (s store) list() ([]record, error) {
56+ ents, err := os.ReadDir(s.dir)
57+ if errors.Is(err, os.ErrNotExist) {
58+ return nil, nil
59+ }
60+ if err != nil {
61+ return nil, err
62+ }
63+ var out []record
64+ for _, e := range ents {
65+ name := e.Name()
66+ if e.IsDir() || !strings.HasSuffix(name, ".json") {
67+ // A *.tmp left by a crash mid-write.
68+ if strings.HasSuffix(name, ".tmp") {
69+ _ = os.Remove(filepath.Join(s.dir, name))
70+ }
71+ continue
72+ }
73+ id := strings.TrimSuffix(name, ".json")
74+ var r record
75+ if b, err := os.ReadFile(filepath.Join(s.dir, name)); err == nil && json.Unmarshal(b, &r) == nil && r.ID == id {
76+ out = append(out, r)
77+ continue
78+ }
79+ out = append(out, record{ID: id, Slot: -1})
80+ }
81+ return out, nil
82+}
A internal/sandbox/vm/tap.go +76 -0
@@ -0,0 +1,76 @@
1+// SPDX-License-Identifier: MIT
2+
3+package vm
4+
5+import (
6+ "context"
7+ "fmt"
8+ "net/netip"
9+ "strconv"
10+ "strings"
11+
12+ "bnema.dev/ermite-runner/internal/cmdrun"
13+)
14+
15+// tapPrefix must match the "ertap*" wildcard of the nftables policy.
16+const tapPrefix = "ertap"
17+
18+func tapName(slot int) string { return tapPrefix + strconv.Itoa(slot) }
19+
20+// createTap creates a persistent TAP owned by uid (the user Firecracker runs
21+// as, so it may open it), gives the host side its /30 address and brings it up.
22+// It needs CAP_NET_ADMIN. On error the TAP is removed again.
23+func createTap(ctx context.Context, run cmdrun.Runner, name string, uid int, gateway netip.Addr) error {
24+ u := strconv.Itoa(uid)
25+ if _, err := run.Run(ctx, nil, "ip", "tuntap", "add", "dev", name, "mode", "tap", "user", u, "group", u); err != nil {
26+ return fmt.Errorf("create tap %s: %w", name, err)
27+ }
28+ for _, args := range [][]string{
29+ {"addr", "add", gateway.String() + "/30", "dev", name},
30+ {"link", "set", "dev", name, "up"},
31+ } {
32+ if _, err := run.Run(ctx, nil, "ip", args...); err != nil {
33+ _ = deleteTap(context.WithoutCancel(ctx), run, name)
34+ return fmt.Errorf("configure tap %s: %w", name, err)
35+ }
36+ }
37+ return nil
38+}
39+
40+// deleteTap removes a TAP; a missing device is not an error.
41+func deleteTap(ctx context.Context, run cmdrun.Runner, name string) error {
42+ out, err := run.Run(ctx, nil, "ip", "link", "del", "dev", name)
43+ if err != nil && (strings.Contains(string(out), "Cannot find device") || strings.Contains(err.Error(), "Cannot find device")) {
44+ return nil
45+ }
46+ return err
47+}
48+
49+// leftoverTaps lists interfaces named like ours ("ertap<N>").
50+func leftoverTaps(ctx context.Context, run cmdrun.Runner) ([]string, error) {
51+ out, err := run.Run(ctx, nil, "ip", "-o", "link", "show")
52+ if err != nil {
53+ return nil, err
54+ }
55+ return parseTapNames(string(out)), nil
56+}
57+
58+// parseTapNames extracts "ertap<digits>" interface names from `ip -o link show`
59+// ("3: ertap0: <...>" or "3: ertap0@if2: <...>").
60+func parseTapNames(s string) []string {
61+ var out []string
62+ for _, line := range strings.Split(s, "\n") {
63+ _, rest, ok := strings.Cut(line, ": ")
64+ if !ok {
65+ continue
66+ }
67+ name, _, _ := strings.Cut(rest, ":")
68+ name, _, _ = strings.Cut(name, "@")
69+ if n, ok := strings.CutPrefix(name, tapPrefix); ok && n != "" {
70+ if _, err := strconv.Atoi(n); err == nil {
71+ out = append(out, name)
72+ }
73+ }
74+ }
75+ return out
76+}
@@ -0,0 +1,76 @@
1+// SPDX-License-Identifier: MIT
2+
3+package vm
4+
5+import (
6+ "context"
7+ "fmt"
8+ "net/netip"
9+ "strconv"
10+ "strings"
11+
12+ "bnema.dev/ermite-runner/internal/cmdrun"
13+)
14+
15+// tapPrefix must match the "ertap*" wildcard of the nftables policy.
16+const tapPrefix = "ertap"
17+
18+func tapName(slot int) string { return tapPrefix + strconv.Itoa(slot) }
19+
20+// createTap creates a persistent TAP owned by uid (the user Firecracker runs
21+// as, so it may open it), gives the host side its /30 address and brings it up.
22+// It needs CAP_NET_ADMIN. On error the TAP is removed again.
23+func createTap(ctx context.Context, run cmdrun.Runner, name string, uid int, gateway netip.Addr) error {
24+ u := strconv.Itoa(uid)
25+ if _, err := run.Run(ctx, nil, "ip", "tuntap", "add", "dev", name, "mode", "tap", "user", u, "group", u); err != nil {
26+ return fmt.Errorf("create tap %s: %w", name, err)
27+ }
28+ for _, args := range [][]string{
29+ {"addr", "add", gateway.String() + "/30", "dev", name},
30+ {"link", "set", "dev", name, "up"},
31+ } {
32+ if _, err := run.Run(ctx, nil, "ip", args...); err != nil {
33+ _ = deleteTap(context.WithoutCancel(ctx), run, name)
34+ return fmt.Errorf("configure tap %s: %w", name, err)
35+ }
36+ }
37+ return nil
38+}
39+
40+// deleteTap removes a TAP; a missing device is not an error.
41+func deleteTap(ctx context.Context, run cmdrun.Runner, name string) error {
42+ out, err := run.Run(ctx, nil, "ip", "link", "del", "dev", name)
43+ if err != nil && (strings.Contains(string(out), "Cannot find device") || strings.Contains(err.Error(), "Cannot find device")) {
44+ return nil
45+ }
46+ return err
47+}
48+
49+// leftoverTaps lists interfaces named like ours ("ertap<N>").
50+func leftoverTaps(ctx context.Context, run cmdrun.Runner) ([]string, error) {
51+ out, err := run.Run(ctx, nil, "ip", "-o", "link", "show")
52+ if err != nil {
53+ return nil, err
54+ }
55+ return parseTapNames(string(out)), nil
56+}
57+
58+// parseTapNames extracts "ertap<digits>" interface names from `ip -o link show`
59+// ("3: ertap0: <...>" or "3: ertap0@if2: <...>").
60+func parseTapNames(s string) []string {
61+ var out []string
62+ for _, line := range strings.Split(s, "\n") {
63+ _, rest, ok := strings.Cut(line, ": ")
64+ if !ok {
65+ continue
66+ }
67+ name, _, _ := strings.Cut(rest, ":")
68+ name, _, _ = strings.Cut(name, "@")
69+ if n, ok := strings.CutPrefix(name, tapPrefix); ok && n != "" {
70+ if _, err := strconv.Atoi(n); err == nil {
71+ out = append(out, name)
72+ }
73+ }
74+ }
75+ return out
76+}
A internal/sandbox/vm/vm_test.go +684 -0
@@ -0,0 +1,684 @@
1+// SPDX-License-Identifier: MIT
2+
3+package vm
4+
5+import (
6+ "context"
7+ "encoding/json"
8+ "errors"
9+ "fmt"
10+ "io"
11+ "net"
12+ "net/http"
13+ "net/netip"
14+ "os"
15+ "os/exec"
16+ "path/filepath"
17+ "strings"
18+ "sync"
19+ "syscall"
20+ "testing"
21+ "time"
22+
23+ "bnema.dev/ermite-runner/internal/cmdrun"
24+ "bnema.dev/ermite-runner/internal/execstream"
25+ "bnema.dev/ermite-runner/internal/execstream/execpb"
26+ "bnema.dev/ermite-runner/internal/sandbox"
27+ "bnema.dev/ermite-runner/internal/sandbox/sandboxtest"
28+)
29+
30+// The tests run a fake Firecracker: this test binary, re-executed through a
31+// shell script named "firecracker" with FC_HELPER set. It serves the HTTP API on
32+// the unix socket, records every request, and on InstanceStart plays the guest:
33+// it connects to the vsock socket and speaks the executor protocol.
34+func TestMain(m *testing.M) {
35+ if os.Getenv("FC_HELPER") != "" {
36+ fakeFirecracker()
37+ return
38+ }
39+ os.Exit(m.Run())
40+}
41+
42+func fakeFirecracker() {
43+ var sock, id string
44+ args := os.Args[1:]
45+ for i := 0; i+1 < len(args); i++ {
46+ switch args[i] {
47+ case "--api-sock":
48+ sock = args[i+1]
49+ case "--id":
50+ id = args[i+1]
51+ }
52+ }
53+ _ = id
54+ mode := os.Getenv("FC_MODE")
55+ if mode == "die" {
56+ fmt.Fprintln(os.Stderr, "fake firecracker: refusing to start")
57+ os.Exit(3)
58+ }
59+ logf, _ := os.OpenFile(os.Getenv("FC_LOG"), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
60+ var vsock string
61+ mux := http.NewServeMux()
62+ mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
63+ body, _ := io.ReadAll(r.Body)
64+ if r.Method == http.MethodPut {
65+ _, _ = fmt.Fprintf(logf, "%s %s %s\n", r.Method, r.URL.Path, body)
66+ }
67+ if r.URL.Path == "/vsock" {
68+ var v vsockDevice
69+ _ = json.Unmarshal(body, &v)
70+ vsock = v.UDSPath
71+ }
72+ w.WriteHeader(http.StatusNoContent)
73+ if r.URL.Path == "/actions" && mode != "hang" {
74+ go func() {
75+ c, err := net.Dial("unix", fmt.Sprintf("%s_%d", vsock, sandbox.VsockPort))
76+ if err != nil {
77+ _, _ = fmt.Fprintln(logf, "vsock dial:", err)
78+ return
79+ }
80+ defer func() { _ = c.Close() }()
81+ g := execstream.NewConn(c)
82+ _ = g.Send(&execpb.Frame{Body: &execpb.Frame_Hello{Hello: &execpb.Hello{Sandbox: "vm"}}})
83+ if _, err := g.Recv(); err != nil {
84+ return
85+ }
86+ _ = g.Send(&execpb.Frame{Body: &execpb.Frame_Output{Output: &execpb.Output{Data: []byte("hello from fake guest\n")}}})
87+ _ = g.Send(&execpb.Frame{Body: &execpb.Frame_Exit{Exit: &execpb.Exit{Code: 0}}})
88+ // Like a real VM, stay alive until killed.
89+ select {}
90+ }()
91+ }
92+ })
93+ ln, err := net.Listen("unix", sock)
94+ if err != nil {
95+ fmt.Fprintln(os.Stderr, err)
96+ os.Exit(2)
97+ }
98+ _ = http.Serve(ln, mux)
99+}
100+
101+type env struct {
102+ cfg sandbox.Config
103+ run *cmdrun.Fake
104+ plumb *sandboxtest.Plumbing
105+ log string
106+ taps map[string]bool
107+ mu sync.Mutex
108+}
109+
110+// newEnv prepares a state dir, fake binaries, kernel and images.
111+func newEnv(t *testing.T, mode string) *env {
112+ t.Helper()
113+ bash, err := exec.LookPath("bash")
114+ if err != nil {
115+ t.Skip("bash needed for the fake firecracker")
116+ }
117+ dir := t.TempDir()
118+ e := &env{taps: map[string]bool{}, plumb: sandboxtest.NewPlumbing(), log: filepath.Join(dir, "fc.log")}
119+ self, _ := os.Executable()
120+ fc := filepath.Join(dir, "firecracker")
121+ script := fmt.Sprintf("#!%s\nexport FC_HELPER=1 FC_MODE=%q FC_LOG=%q\nexec -a firecracker %q \"$@\"\n", bash, mode, e.log, self)
122+ if err := os.WriteFile(fc, []byte(script), 0o755); err != nil {
123+ t.Fatal(err)
124+ }
125+ img := filepath.Join(dir, "image")
126+ _ = os.MkdirAll(img, 0o755)
127+ for _, f := range []string{"vmlinux", "base.ext4"} {
128+ _ = os.WriteFile(filepath.Join(img, f), []byte("x"), 0o644)
129+ }
130+ exe := filepath.Join(dir, "runner")
131+ sandboxtest.WriteStaticELF(t, exe)
132+ e.cfg = sandbox.Config{
133+ Isolation: sandbox.VM, StateDir: filepath.Join(dir, "state"), Capacity: 2, Image: img,
134+ JobCPUs: 1, JobMemoryMiB: 512, JobDiskMiB: 1024, UIDBase: 900000, KVMPath: "/dev/null",
135+ Network: mustPrefix("10.200.0.0/16"), Firecracker: fc, JailerOff: true, Executable: exe, DNSPort: 0,
136+ }
137+ e.run = &cmdrun.Fake{Handle: func(c cmdrun.Call) ([]byte, error) {
138+ e.mu.Lock()
139+ defer e.mu.Unlock()
140+ switch {
141+ case c.Name == "ip" && len(c.Args) > 1 && c.Args[0] == "tuntap":
142+ e.taps[c.Args[3]] = true
143+ case c.Name == "ip" && len(c.Args) > 2 && c.Args[0] == "link" && c.Args[1] == "del":
144+ if !e.taps[c.Args[3]] {
145+ return []byte("Cannot find device"), errors.New("exit status 1")
146+ }
147+ delete(e.taps, c.Args[3])
148+ case c.Name == "ip" && len(c.Args) > 1 && c.Args[0] == "-o":
149+ var b strings.Builder
150+ i := 1
151+ for t := range e.taps {
152+ fmt.Fprintf(&b, "%d: %s: <BROADCAST> mtu 1500\n", i, t)
153+ i++
154+ }
155+ return []byte(b.String()), nil
156+ }
157+ return nil, nil
158+ }}
159+ return e
160+}
161+
162+func (e *env) tapCount() int { e.mu.Lock(); defer e.mu.Unlock(); return len(e.taps) }
163+
164+func (e *env) driver(t *testing.T) *Driver {
165+ t.Helper()
166+ d, err := New(context.Background(), e.cfg, Options{Run: e.run, Net: e.plumb, BootTimeout: 5 * time.Second})
167+ if err != nil {
168+ t.Fatal(err)
169+ }
170+ t.Cleanup(func() { _ = d.Close() })
171+ return d
172+}
173+
174+// leftovers lists everything that must not survive a job.
175+func (e *env) leftovers(t *testing.T) []string {
176+ t.Helper()
177+ var out []string
178+ _ = filepath.WalkDir(e.cfg.StateDir, func(p string, d os.DirEntry, err error) error {
179+ rel, _ := filepath.Rel(e.cfg.StateDir, p)
180+ switch {
181+ case err != nil || rel == ".":
182+ case strings.HasPrefix(rel, "vm/") || strings.HasPrefix(rel, "vms/") || strings.HasPrefix(rel, "jail/"):
183+ out = append(out, rel)
184+ }
185+ return nil
186+ })
187+ if n := e.tapCount(); n != 0 {
188+ out = append(out, fmt.Sprintf("%d TAP devices", n))
189+ }
190+ if n := e.plumb.Live(); n != 0 {
191+ out = append(out, fmt.Sprintf("%d attached guests", n))
192+ }
193+ return out
194+}
195+
196+func runStub(t *testing.T, conn sandbox.Conn) (string, int) {
197+ t.Helper()
198+ var out strings.Builder
199+ _, exit, err := execstream.Drive(context.Background(), conn, &execpb.Run{}, func(o *execpb.Output) { out.Write(o.Data) })
200+ if err != nil {
201+ t.Fatal(err)
202+ }
203+ return out.String(), int(exit.Code)
204+}
205+
206+func TestStartRunStop(t *testing.T) {
207+ e := newEnv(t, "")
208+ d := e.driver(t)
209+ sb, err := d.New()
210+ if err != nil {
211+ t.Fatal(err)
212+ }
213+ conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "job-1"})
214+ if err != nil {
215+ t.Fatal(err)
216+ }
217+ out, code := runStub(t, conn)
218+ if out != "hello from fake guest\n" || code != 0 {
219+ t.Fatalf("out=%q code=%d", out, code)
220+ }
221+ // While running: record, TAP, attachment, process.
222+ if e.tapCount() != 1 || e.plumb.Live() != 1 {
223+ t.Fatalf("tap=%d attached=%d", e.tapCount(), e.plumb.Live())
224+ }
225+ if recs, _ := d.store.list(); len(recs) != 1 || recs[0].Job != "job-1" || recs[0].PID == 0 || recs[0].Tap != "ertap0" {
226+ t.Fatalf("records: %+v", recs)
227+ }
228+ pid := sb.(*vmSandbox).rec.PID
229+
230+ if err := sb.Stop(context.Background()); err != nil {
231+ t.Fatal(err)
232+ }
233+ if left := e.leftovers(t); len(left) != 0 {
234+ t.Fatalf("leftovers after Stop: %v", left)
235+ }
236+ if err := syscall.Kill(pid, 0); err == nil {
237+ t.Fatalf("firecracker pid %d still alive", pid)
238+ }
239+ if err := sb.Stop(context.Background()); err != nil {
240+ t.Fatalf("Stop must be idempotent: %v", err)
241+ }
242+ if d.slots.InUse() != 0 {
243+ t.Fatal("slot not released")
244+ }
245+}
246+
247+// The API calls must follow the order Firecracker needs and carry the
248+// plan's disk layout: exec disk (boot root, RO), base (RO), scratch (RW).
249+func TestAPICallsAndDiskLayout(t *testing.T) {
250+ e := newEnv(t, "")
251+ d := e.driver(t)
252+ sb, _ := d.New()
253+ conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j", CPUs: 3, MemoryMiB: 768, DiskMiB: 2048})
254+ if err != nil {
255+ t.Fatal(err)
256+ }
257+ runStub(t, conn)
258+ defer func() { _ = sb.Stop(context.Background()) }()
259+
260+ b, _ := os.ReadFile(e.log)
261+ lines := strings.Split(strings.TrimSpace(string(b)), "\n")
262+ var paths []string
263+ for _, l := range lines {
264+ f := strings.Fields(l)
265+ if len(f) >= 2 && f[0] == "PUT" {
266+ paths = append(paths, f[1])
267+ }
268+ }
269+ want := []string{"/machine-config", "/boot-source", "/drives/rootfs", "/drives/base", "/drives/scratch", "/network-interfaces/eth0", "/vsock", "/entropy", "/actions"}
270+ if strings.Join(paths, ",") != strings.Join(want, ",") {
271+ t.Fatalf("call order:\n got %v\nwant %v", paths, want)
272+ }
273+ log := string(b)
274+ for _, frag := range []string{
275+ `"vcpu_count":3`, `"mem_size_mib":768`, `"smt":false`,
276+ `"drive_id":"rootfs"`, `"is_root_device":true,"is_read_only":true`,
277+ `"drive_id":"base"`, `"is_root_device":false,"is_read_only":true`,
278+ `"drive_id":"scratch"`, `"is_root_device":false,"is_read_only":false`,
279+ `"host_dev_name":"ertap0"`, `"guest_cid":3`, `"action_type":"InstanceStart"`,
280+ "ipv6.disable=1", "ermite.connect=vsock:1024", "ermite.gw=10.200.0.1", "ip=10.200.0.2::10.200.0.1:255.255.255.252",
281+ } {
282+ if !strings.Contains(log, frag) {
283+ t.Errorf("API log lacks %q", frag)
284+ }
285+ }
286+ // The scratch disk was created sparse at the requested size, and formatted.
287+ var mkfs []string
288+ for _, l := range e.run.Lines() {
289+ if strings.HasPrefix(l, "mkfs.ext4") {
290+ mkfs = append(mkfs, l)
291+ }
292+ }
293+ if len(mkfs) == 0 {
294+ t.Fatalf("no mkfs call: %v", e.run.Lines())
295+ }
296+}
297+
298+func TestJobsDoNotShareSlotsOrTaps(t *testing.T) {
299+ e := newEnv(t, "")
300+ d := e.driver(t)
301+ var sbs []sandbox.Sandbox
302+ for i := range 2 {
303+ sb, err := d.New()
304+ if err != nil {
305+ t.Fatal(err)
306+ }
307+ conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: fmt.Sprint(i)})
308+ if err != nil {
309+ t.Fatal(err)
310+ }
311+ runStub(t, conn)
312+ sbs = append(sbs, sb)
313+ }
314+ if _, err := d.New(); !errors.Is(err, sandbox.ErrCapacity) {
315+ t.Fatalf("capacity not enforced: %v", err)
316+ }
317+ if e.tapCount() != 2 {
318+ t.Fatalf("taps: %v", e.taps)
319+ }
320+ for _, sb := range sbs {
321+ _ = sb.Stop(context.Background())
322+ }
323+ if left := e.leftovers(t); len(left) != 0 {
324+ t.Fatalf("leftovers: %v", left)
325+ }
326+}
327+
328+func TestStartFailureCleansEverything(t *testing.T) {
329+ e := newEnv(t, "die")
330+ d := e.driver(t)
331+ sb, _ := d.New()
332+ _, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
333+ if err == nil || !strings.Contains(err.Error(), "refusing to start") {
334+ t.Fatalf("want the VMM console in the error, got %v", err)
335+ }
336+ if left := e.leftovers(t); len(left) != 0 {
337+ t.Fatalf("leftovers after failed Start: %v", left)
338+ }
339+ if d.slots.InUse() != 0 {
340+ t.Fatal("slot leaked by a failed Start")
341+ }
342+}
343+
344+func TestBootTimeoutCleansEverything(t *testing.T) {
345+ e := newEnv(t, "hang")
346+ d, err := New(context.Background(), e.cfg, Options{Run: e.run, Net: e.plumb, BootTimeout: 300 * time.Millisecond})
347+ if err != nil {
348+ t.Fatal(err)
349+ }
350+ defer func() { _ = d.Close() }()
351+ sb, _ := d.New()
352+ _, err = sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
353+ if err == nil || !strings.Contains(err.Error(), "did not connect") {
354+ t.Fatalf("got %v", err)
355+ }
356+ if left := e.leftovers(t); len(left) != 0 {
357+ t.Fatalf("leftovers: %v", left)
358+ }
359+}
360+
361+func TestStartCancelledByContext(t *testing.T) {
362+ e := newEnv(t, "hang")
363+ d := e.driver(t)
364+ sb, _ := d.New()
365+ ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
366+ defer cancel()
367+ if _, err := sb.Start(ctx, sandbox.JobSpec{JobID: "j"}); !errors.Is(err, context.DeadlineExceeded) {
368+ t.Fatalf("got %v", err)
369+ }
370+ if left := e.leftovers(t); len(left) != 0 {
371+ t.Fatalf("leftovers: %v", left)
372+ }
373+}
374+
375+func TestNetworkAttachFailureCleans(t *testing.T) {
376+ e := newEnv(t, "")
377+ d := e.driver(t)
378+ e.plumb.FailNext = errors.New("nft broke")
379+ sb, _ := d.New()
380+ if _, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"}); err == nil || !strings.Contains(err.Error(), "nft broke") {
381+ t.Fatalf("got %v", err)
382+ }
383+ if left := e.leftovers(t); len(left) != 0 {
384+ t.Fatalf("leftovers: %v", left)
385+ }
386+}
387+
388+func TestStopWithoutStartReleasesSlot(t *testing.T) {
389+ e := newEnv(t, "")
390+ d := e.driver(t)
391+ sb, _ := d.New()
392+ if err := sb.Stop(context.Background()); err != nil {
393+ t.Fatal(err)
394+ }
395+ if d.slots.InUse() != 0 {
396+ t.Fatal("slot leaked")
397+ }
398+ if _, err := sb.Start(context.Background(), sandbox.JobSpec{}); err == nil {
399+ t.Fatal("Start after Stop accepted")
400+ }
401+}
402+
403+// Stop must finish the cleanup even when the caller's context is already done
404+// (a cancelled job).
405+func TestStopRunsWithCancelledContext(t *testing.T) {
406+ e := newEnv(t, "")
407+ d := e.driver(t)
408+ sb, _ := d.New()
409+ conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
410+ if err != nil {
411+ t.Fatal(err)
412+ }
413+ runStub(t, conn)
414+ ctx, cancel := context.WithCancel(context.Background())
415+ cancel()
416+ if err := sb.Stop(ctx); err != nil {
417+ t.Fatal(err)
418+ }
419+ if left := e.leftovers(t); len(left) != 0 {
420+ t.Fatalf("leftovers: %v", left)
421+ }
422+}
423+
424+// A killed runner leaves a running Firecracker, a TAP, nft rules and files. The
425+// next start must reap all of it (AC4: nothing survives, incl. killed jobs).
426+func TestReapAfterRunnerCrash(t *testing.T) {
427+ e := newEnv(t, "")
428+ d1 := e.driver(t)
429+ sb, _ := d1.New()
430+ conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
431+ if err != nil {
432+ t.Fatal(err)
433+ }
434+ runStub(t, conn)
435+ pid := sb.(*vmSandbox).rec.PID
436+ if err := syscall.Kill(pid, 0); err != nil {
437+ t.Fatal("fake VM not running")
438+ }
439+ // The runner dies: its lock goes away, nothing else is cleaned.
440+ _ = d1.lock.Close()
441+ d1.lock = nil
442+ _ = conn.Close()
443+ if e.tapCount() != 1 {
444+ t.Fatal("setup: no TAP")
445+ }
446+
447+ d2 := e.driver(t)
448+ if err := d2.Reap(context.Background()); err != nil {
449+ t.Fatal(err)
450+ }
451+ if left := e.leftovers(t); len(left) != 0 {
452+ t.Fatalf("leftovers after Reap: %v", left)
453+ }
454+ deadline := time.Now().Add(3 * time.Second)
455+ for syscall.Kill(pid, 0) == nil && time.Now().Before(deadline) {
456+ // The fake is a child of this test process: it is a zombie until the
457+ // original driver's Wait goroutine collects it.
458+ if b, err := os.ReadFile(fmt.Sprintf("/proc/%d/stat", pid)); err == nil && strings.Contains(string(b), ") Z") {
459+ break
460+ }
461+ time.Sleep(20 * time.Millisecond)
462+ }
463+ if b, _ := os.ReadFile(fmt.Sprintf("/proc/%d/stat", pid)); len(b) > 0 && !strings.Contains(string(b), ") Z") {
464+ t.Fatalf("firecracker %d survived Reap: %s", pid, b)
465+ }
466+ if e.plumb.Resets == 0 {
467+ t.Fatal("the nftables table was not reset")
468+ }
469+}
470+
471+func TestReapRemovesCorruptRecordAndTempFiles(t *testing.T) {
472+ e := newEnv(t, "")
473+ d := e.driver(t)
474+ _ = os.MkdirAll(d.store.dir, 0o700)
475+ _ = os.WriteFile(filepath.Join(d.store.dir, "er00112233.json"), []byte("{garbage"), 0o600)
476+ _ = os.WriteFile(filepath.Join(d.store.dir, "er44556677.json.tmp"), []byte("{"), 0o600)
477+ _ = os.MkdirAll(filepath.Join(e.cfg.StateDir, "vm", "er00112233"), 0o700)
478+ _ = os.MkdirAll(filepath.Join(e.cfg.StateDir, "vm", "not-ours"), 0o700) // must be left alone
479+ if err := d.Reap(context.Background()); err != nil {
480+ t.Fatal(err)
481+ }
482+ if _, err := os.Stat(filepath.Join(e.cfg.StateDir, "vm", "not-ours")); err != nil {
483+ t.Fatal("Reap removed a directory that is not ours")
484+ }
485+ _ = os.Remove(filepath.Join(e.cfg.StateDir, "vm", "not-ours"))
486+ if left := e.leftovers(t); len(left) != 0 {
487+ t.Fatalf("leftovers: %v", left)
488+ }
489+}
490+
491+func TestReapLeavesForeignTapsAlone(t *testing.T) {
492+ e := newEnv(t, "")
493+ e.taps["tap0"] = true
494+ e.taps["ertapx"] = true // not ertap<digits>
495+ e.taps["ertap7"] = true
496+ d := e.driver(t)
497+ if err := d.Reap(context.Background()); err != nil {
498+ t.Fatal(err)
499+ }
500+ if !e.taps["tap0"] || !e.taps["ertapx"] || e.taps["ertap7"] {
501+ t.Fatalf("taps after reap: %v", e.taps)
502+ }
503+}
504+
505+func TestSecondRunnerOnSameStateDirRefused(t *testing.T) {
506+ e := newEnv(t, "")
507+ _ = e.driver(t)
508+ _, err := New(context.Background(), e.cfg, Options{Run: e.run, Net: e.plumb})
509+ if err == nil || !strings.Contains(err.Error(), "another ermite-runner") {
510+ t.Fatalf("got %v", err)
511+ }
512+}
513+
514+func TestNewRejectsDynamicExecutable(t *testing.T) {
515+ e := newEnv(t, "")
516+ e.cfg.Executable = "/bin/sh"
517+ if _, err := New(context.Background(), e.cfg, Options{Run: e.run, Net: e.plumb}); err == nil {
518+ t.Skip("/bin/sh is static here")
519+ }
520+}
521+
522+func TestExecDiskIsKeyedByExecutableHash(t *testing.T) {
523+ e := newEnv(t, "")
524+ d1 := e.driver(t)
525+ first := d1.execDisk
526+ _ = d1.Close()
527+ // Same executable: reused, no second mkfs of an exec disk.
528+ n := len(e.run.Calls)
529+ d2 := e.driver(t)
530+ if d2.execDisk != first {
531+ t.Fatal("exec disk rebuilt for the same executable")
532+ }
533+ for _, c := range e.run.Calls[n:] {
534+ if c.Name == "mkfs.ext4" {
535+ t.Fatal("mkfs run again for an unchanged executable")
536+ }
537+ }
538+ _ = d2.Close()
539+ // A runner upgrade: new disk, old one removed.
540+ if err := os.WriteFile(e.cfg.Executable, append(readFile(t, e.cfg.Executable), 0), 0o755); err != nil {
541+ t.Fatal(err)
542+ }
543+ d3 := e.driver(t)
544+ if d3.execDisk == first {
545+ t.Fatal("exec disk not rebuilt after the executable changed")
546+ }
547+ if _, err := os.Stat(first); err == nil {
548+ t.Fatal("stale exec disk kept")
549+ }
550+}
551+
552+func readFile(t *testing.T, p string) []byte {
553+ t.Helper()
554+ b, err := os.ReadFile(p)
555+ if err != nil {
556+ t.Fatal(err)
557+ }
558+ return b
559+}
560+
561+func TestJailedLayout(t *testing.T) {
562+ l := newLayout("/var/lib/er", "firecracker", "er00112233", true, 900003)
563+ if l.root != "/var/lib/er/jail/firecracker/er00112233/root" || l.dir != "/var/lib/er/jail/firecracker/er00112233" {
564+ t.Fatalf("%+v", l)
565+ }
566+ if l.fc("scratch.ext4") != "/scratch.ext4" || l.host("scratch.ext4") != l.root+"/scratch.ext4" {
567+ t.Fatal("jailed paths")
568+ }
569+ if jailBase("/var/lib/er") != "/var/lib/er/jail" {
570+ t.Fatal("jail base")
571+ }
572+ if l2 := newLayout("/var/lib/er", "firecracker-v1.17", "er00112233", true, 1); l2.dir != "/var/lib/er/jail/firecracker-v1.17/er00112233" {
573+ t.Fatalf("the jailer names its chroot after the exec file: %s", l2.dir)
574+ }
575+ u := newLayout("/var/lib/er", "firecracker", "er00112233", false, 1000)
576+ if u.fc("api.sock") != "/var/lib/er/vm/er00112233/api.sock" {
577+ t.Fatalf("unjailed path %s", u.fc("api.sock"))
578+ }
579+ if p, _ := u.share("/img/base.ext4", "base.ext4"); p != "/img/base.ext4" {
580+ t.Fatal("unjailed share must use the file in place")
581+ }
582+}
583+
584+func TestJailedShareHardLinks(t *testing.T) {
585+ dir := t.TempDir()
586+ src := filepath.Join(dir, "base.ext4")
587+ _ = os.WriteFile(src, []byte("data"), 0o644)
588+ l := newLayout(dir, "firecracker", "er00112233", true, os.Getuid())
589+ if err := os.MkdirAll(l.root, 0o700); err != nil {
590+ t.Fatal(err)
591+ }
592+ p, err := l.share(src, "base.ext4")
593+ if err != nil || p != "/base.ext4" {
594+ t.Fatalf("%q %v", p, err)
595+ }
596+ a, _ := os.Stat(src)
597+ b, _ := os.Stat(l.host("base.ext4"))
598+ if !os.SameFile(a, b) {
599+ t.Fatal("shared file must be a hard link, never a copy")
600+ }
601+ if err := checkWorldReadable(src); err != nil {
602+ t.Fatal(err)
603+ }
604+ _ = os.Chmod(src, 0o600)
605+ if err := checkWorldReadable(src); err == nil {
606+ t.Fatal("unreadable-by-others file accepted for the jail")
607+ }
608+}
609+
610+func TestBootArgs(t *testing.T) {
611+ a := bootArgs(mustAddr("10.200.0.6"), mustAddr("10.200.0.5"), 1791818522)
612+ for _, want := range []string{
613+ "ipv6.disable=1", "root=/dev/vda ro", "init=/ermite-runner", "reboot=k", "panic=1",
614+ "ip=10.200.0.6::10.200.0.5:255.255.255.252:ermite-job:eth0:off:10.200.0.5",
615+ "ermite.sandbox=vm", "ermite.connect=vsock:1024", "ermite.gw=10.200.0.5", "ermite.time=1791818522",
616+ } {
617+ if !strings.Contains(a, want) {
618+ t.Errorf("boot args lack %q: %s", want, a)
619+ }
620+ }
621+ if !strings.HasSuffix(a, "-- init") {
622+ t.Errorf("init subcommand missing: %s", a)
623+ }
624+ if macFor(0) != "06:00:ac:10:00:00" || macFor(257) != "06:00:ac:10:01:01" {
625+ t.Fatal(macFor(257))
626+ }
627+}
628+
629+func TestParseTapNames(t *testing.T) {
630+ in := "1: lo: <LOOPBACK> mtu 65536\n2: eth0: <UP> mtu 1500\n5: ertap0: <BROADCAST> mtu 1500\n9: ertap12@NONE: <X>\n10: ertapx: <X>\n11: tap0: <X>\n12: ertap: <X>\n"
631+ got := strings.Join(parseTapNames(in), ",")
632+ if got != "ertap0,ertap12" {
633+ t.Fatalf("got %q", got)
634+ }
635+}
636+
637+func TestFirecrackerProcsScan(t *testing.T) {
638+ proc := t.TempDir()
639+ mk := func(pid, cmdline string) {
640+ _ = os.MkdirAll(filepath.Join(proc, pid), 0o755)
641+ _ = os.WriteFile(filepath.Join(proc, pid, "cmdline"), []byte(cmdline), 0o644)
642+ }
643+ mk("100", "firecracker\x00--id\x00er00112233\x00--api-sock\x00/x\x00")
644+ mk("101", "/usr/bin/firecracker\x00--api-sock\x00/x\x00--id\x00er44556677\x00")
645+ mk("102", "firecracker\x00--id\x00other-tool-vm\x00") // someone else's VM
646+ mk("103", "bash\x00--id\x00er00112233\x00") // not firecracker
647+ mk("1", "firecracker\x00--id\x00er99999999\x00") // pid 1 is never touched
648+ mk("abc", "")
649+ got := firecrackerProcs(proc)
650+ if len(got) != 2 || got[100] != "er00112233" || got[101] != "er44556677" {
651+ t.Fatalf("%v", got)
652+ }
653+}
654+
655+func TestRingKeepsTail(t *testing.T) {
656+ r := newRing(8)
657+ _, _ = r.Write([]byte("0123456789"))
658+ _, _ = r.Write([]byte("ab"))
659+ if r.String() != "23456789ab"[2:] {
660+ t.Fatalf("%q", r.String())
661+ }
662+}
663+
664+func TestAPIErrorsSurfaceFirecrackerFault(t *testing.T) {
665+ sock := filepath.Join(t.TempDir(), "s")
666+ ln, err := net.Listen("unix", sock)
667+ if err != nil {
668+ t.Fatal(err)
669+ }
670+ defer func() { _ = ln.Close() }()
671+ go func() {
672+ _ = http.Serve(ln, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
673+ w.WriteHeader(400)
674+ _, _ = w.Write([]byte(`{"fault_message":"Invalid request"}`))
675+ }))
676+ }()
677+ err = newAPIClient(sock).put(context.Background(), "/boot-source", bootSource{})
678+ if err == nil || !strings.Contains(err.Error(), "Invalid request") || !strings.Contains(err.Error(), "/boot-source") {
679+ t.Fatalf("got %v", err)
680+ }
681+}
682+
683+func mustPrefix(s string) netip.Prefix { return netip.MustParsePrefix(s) }
684+func mustAddr(s string) netip.Addr { return netip.MustParseAddr(s) }
@@ -0,0 +1,684 @@
1+// SPDX-License-Identifier: MIT
2+
3+package vm
4+
5+import (
6+ "context"
7+ "encoding/json"
8+ "errors"
9+ "fmt"
10+ "io"
11+ "net"
12+ "net/http"
13+ "net/netip"
14+ "os"
15+ "os/exec"
16+ "path/filepath"
17+ "strings"
18+ "sync"
19+ "syscall"
20+ "testing"
21+ "time"
22+
23+ "bnema.dev/ermite-runner/internal/cmdrun"
24+ "bnema.dev/ermite-runner/internal/execstream"
25+ "bnema.dev/ermite-runner/internal/execstream/execpb"
26+ "bnema.dev/ermite-runner/internal/sandbox"
27+ "bnema.dev/ermite-runner/internal/sandbox/sandboxtest"
28+)
29+
30+// The tests run a fake Firecracker: this test binary, re-executed through a
31+// shell script named "firecracker" with FC_HELPER set. It serves the HTTP API on
32+// the unix socket, records every request, and on InstanceStart plays the guest:
33+// it connects to the vsock socket and speaks the executor protocol.
34+func TestMain(m *testing.M) {
35+ if os.Getenv("FC_HELPER") != "" {
36+ fakeFirecracker()
37+ return
38+ }
39+ os.Exit(m.Run())
40+}
41+
42+func fakeFirecracker() {
43+ var sock, id string
44+ args := os.Args[1:]
45+ for i := 0; i+1 < len(args); i++ {
46+ switch args[i] {
47+ case "--api-sock":
48+ sock = args[i+1]
49+ case "--id":
50+ id = args[i+1]
51+ }
52+ }
53+ _ = id
54+ mode := os.Getenv("FC_MODE")
55+ if mode == "die" {
56+ fmt.Fprintln(os.Stderr, "fake firecracker: refusing to start")
57+ os.Exit(3)
58+ }
59+ logf, _ := os.OpenFile(os.Getenv("FC_LOG"), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
60+ var vsock string
61+ mux := http.NewServeMux()
62+ mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
63+ body, _ := io.ReadAll(r.Body)
64+ if r.Method == http.MethodPut {
65+ _, _ = fmt.Fprintf(logf, "%s %s %s\n", r.Method, r.URL.Path, body)
66+ }
67+ if r.URL.Path == "/vsock" {
68+ var v vsockDevice
69+ _ = json.Unmarshal(body, &v)
70+ vsock = v.UDSPath
71+ }
72+ w.WriteHeader(http.StatusNoContent)
73+ if r.URL.Path == "/actions" && mode != "hang" {
74+ go func() {
75+ c, err := net.Dial("unix", fmt.Sprintf("%s_%d", vsock, sandbox.VsockPort))
76+ if err != nil {
77+ _, _ = fmt.Fprintln(logf, "vsock dial:", err)
78+ return
79+ }
80+ defer func() { _ = c.Close() }()
81+ g := execstream.NewConn(c)
82+ _ = g.Send(&execpb.Frame{Body: &execpb.Frame_Hello{Hello: &execpb.Hello{Sandbox: "vm"}}})
83+ if _, err := g.Recv(); err != nil {
84+ return
85+ }
86+ _ = g.Send(&execpb.Frame{Body: &execpb.Frame_Output{Output: &execpb.Output{Data: []byte("hello from fake guest\n")}}})
87+ _ = g.Send(&execpb.Frame{Body: &execpb.Frame_Exit{Exit: &execpb.Exit{Code: 0}}})
88+ // Like a real VM, stay alive until killed.
89+ select {}
90+ }()
91+ }
92+ })
93+ ln, err := net.Listen("unix", sock)
94+ if err != nil {
95+ fmt.Fprintln(os.Stderr, err)
96+ os.Exit(2)
97+ }
98+ _ = http.Serve(ln, mux)
99+}
100+
101+type env struct {
102+ cfg sandbox.Config
103+ run *cmdrun.Fake
104+ plumb *sandboxtest.Plumbing
105+ log string
106+ taps map[string]bool
107+ mu sync.Mutex
108+}
109+
110+// newEnv prepares a state dir, fake binaries, kernel and images.
111+func newEnv(t *testing.T, mode string) *env {
112+ t.Helper()
113+ bash, err := exec.LookPath("bash")
114+ if err != nil {
115+ t.Skip("bash needed for the fake firecracker")
116+ }
117+ dir := t.TempDir()
118+ e := &env{taps: map[string]bool{}, plumb: sandboxtest.NewPlumbing(), log: filepath.Join(dir, "fc.log")}
119+ self, _ := os.Executable()
120+ fc := filepath.Join(dir, "firecracker")
121+ script := fmt.Sprintf("#!%s\nexport FC_HELPER=1 FC_MODE=%q FC_LOG=%q\nexec -a firecracker %q \"$@\"\n", bash, mode, e.log, self)
122+ if err := os.WriteFile(fc, []byte(script), 0o755); err != nil {
123+ t.Fatal(err)
124+ }
125+ img := filepath.Join(dir, "image")
126+ _ = os.MkdirAll(img, 0o755)
127+ for _, f := range []string{"vmlinux", "base.ext4"} {
128+ _ = os.WriteFile(filepath.Join(img, f), []byte("x"), 0o644)
129+ }
130+ exe := filepath.Join(dir, "runner")
131+ sandboxtest.WriteStaticELF(t, exe)
132+ e.cfg = sandbox.Config{
133+ Isolation: sandbox.VM, StateDir: filepath.Join(dir, "state"), Capacity: 2, Image: img,
134+ JobCPUs: 1, JobMemoryMiB: 512, JobDiskMiB: 1024, UIDBase: 900000, KVMPath: "/dev/null",
135+ Network: mustPrefix("10.200.0.0/16"), Firecracker: fc, JailerOff: true, Executable: exe, DNSPort: 0,
136+ }
137+ e.run = &cmdrun.Fake{Handle: func(c cmdrun.Call) ([]byte, error) {
138+ e.mu.Lock()
139+ defer e.mu.Unlock()
140+ switch {
141+ case c.Name == "ip" && len(c.Args) > 1 && c.Args[0] == "tuntap":
142+ e.taps[c.Args[3]] = true
143+ case c.Name == "ip" && len(c.Args) > 2 && c.Args[0] == "link" && c.Args[1] == "del":
144+ if !e.taps[c.Args[3]] {
145+ return []byte("Cannot find device"), errors.New("exit status 1")
146+ }
147+ delete(e.taps, c.Args[3])
148+ case c.Name == "ip" && len(c.Args) > 1 && c.Args[0] == "-o":
149+ var b strings.Builder
150+ i := 1
151+ for t := range e.taps {
152+ fmt.Fprintf(&b, "%d: %s: <BROADCAST> mtu 1500\n", i, t)
153+ i++
154+ }
155+ return []byte(b.String()), nil
156+ }
157+ return nil, nil
158+ }}
159+ return e
160+}
161+
162+func (e *env) tapCount() int { e.mu.Lock(); defer e.mu.Unlock(); return len(e.taps) }
163+
164+func (e *env) driver(t *testing.T) *Driver {
165+ t.Helper()
166+ d, err := New(context.Background(), e.cfg, Options{Run: e.run, Net: e.plumb, BootTimeout: 5 * time.Second})
167+ if err != nil {
168+ t.Fatal(err)
169+ }
170+ t.Cleanup(func() { _ = d.Close() })
171+ return d
172+}
173+
174+// leftovers lists everything that must not survive a job.
175+func (e *env) leftovers(t *testing.T) []string {
176+ t.Helper()
177+ var out []string
178+ _ = filepath.WalkDir(e.cfg.StateDir, func(p string, d os.DirEntry, err error) error {
179+ rel, _ := filepath.Rel(e.cfg.StateDir, p)
180+ switch {
181+ case err != nil || rel == ".":
182+ case strings.HasPrefix(rel, "vm/") || strings.HasPrefix(rel, "vms/") || strings.HasPrefix(rel, "jail/"):
183+ out = append(out, rel)
184+ }
185+ return nil
186+ })
187+ if n := e.tapCount(); n != 0 {
188+ out = append(out, fmt.Sprintf("%d TAP devices", n))
189+ }
190+ if n := e.plumb.Live(); n != 0 {
191+ out = append(out, fmt.Sprintf("%d attached guests", n))
192+ }
193+ return out
194+}
195+
196+func runStub(t *testing.T, conn sandbox.Conn) (string, int) {
197+ t.Helper()
198+ var out strings.Builder
199+ _, exit, err := execstream.Drive(context.Background(), conn, &execpb.Run{}, func(o *execpb.Output) { out.Write(o.Data) })
200+ if err != nil {
201+ t.Fatal(err)
202+ }
203+ return out.String(), int(exit.Code)
204+}
205+
206+func TestStartRunStop(t *testing.T) {
207+ e := newEnv(t, "")
208+ d := e.driver(t)
209+ sb, err := d.New()
210+ if err != nil {
211+ t.Fatal(err)
212+ }
213+ conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "job-1"})
214+ if err != nil {
215+ t.Fatal(err)
216+ }
217+ out, code := runStub(t, conn)
218+ if out != "hello from fake guest\n" || code != 0 {
219+ t.Fatalf("out=%q code=%d", out, code)
220+ }
221+ // While running: record, TAP, attachment, process.
222+ if e.tapCount() != 1 || e.plumb.Live() != 1 {
223+ t.Fatalf("tap=%d attached=%d", e.tapCount(), e.plumb.Live())
224+ }
225+ if recs, _ := d.store.list(); len(recs) != 1 || recs[0].Job != "job-1" || recs[0].PID == 0 || recs[0].Tap != "ertap0" {
226+ t.Fatalf("records: %+v", recs)
227+ }
228+ pid := sb.(*vmSandbox).rec.PID
229+
230+ if err := sb.Stop(context.Background()); err != nil {
231+ t.Fatal(err)
232+ }
233+ if left := e.leftovers(t); len(left) != 0 {
234+ t.Fatalf("leftovers after Stop: %v", left)
235+ }
236+ if err := syscall.Kill(pid, 0); err == nil {
237+ t.Fatalf("firecracker pid %d still alive", pid)
238+ }
239+ if err := sb.Stop(context.Background()); err != nil {
240+ t.Fatalf("Stop must be idempotent: %v", err)
241+ }
242+ if d.slots.InUse() != 0 {
243+ t.Fatal("slot not released")
244+ }
245+}
246+
247+// The API calls must follow the order Firecracker needs and carry the
248+// plan's disk layout: exec disk (boot root, RO), base (RO), scratch (RW).
249+func TestAPICallsAndDiskLayout(t *testing.T) {
250+ e := newEnv(t, "")
251+ d := e.driver(t)
252+ sb, _ := d.New()
253+ conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j", CPUs: 3, MemoryMiB: 768, DiskMiB: 2048})
254+ if err != nil {
255+ t.Fatal(err)
256+ }
257+ runStub(t, conn)
258+ defer func() { _ = sb.Stop(context.Background()) }()
259+
260+ b, _ := os.ReadFile(e.log)
261+ lines := strings.Split(strings.TrimSpace(string(b)), "\n")
262+ var paths []string
263+ for _, l := range lines {
264+ f := strings.Fields(l)
265+ if len(f) >= 2 && f[0] == "PUT" {
266+ paths = append(paths, f[1])
267+ }
268+ }
269+ want := []string{"/machine-config", "/boot-source", "/drives/rootfs", "/drives/base", "/drives/scratch", "/network-interfaces/eth0", "/vsock", "/entropy", "/actions"}
270+ if strings.Join(paths, ",") != strings.Join(want, ",") {
271+ t.Fatalf("call order:\n got %v\nwant %v", paths, want)
272+ }
273+ log := string(b)
274+ for _, frag := range []string{
275+ `"vcpu_count":3`, `"mem_size_mib":768`, `"smt":false`,
276+ `"drive_id":"rootfs"`, `"is_root_device":true,"is_read_only":true`,
277+ `"drive_id":"base"`, `"is_root_device":false,"is_read_only":true`,
278+ `"drive_id":"scratch"`, `"is_root_device":false,"is_read_only":false`,
279+ `"host_dev_name":"ertap0"`, `"guest_cid":3`, `"action_type":"InstanceStart"`,
280+ "ipv6.disable=1", "ermite.connect=vsock:1024", "ermite.gw=10.200.0.1", "ip=10.200.0.2::10.200.0.1:255.255.255.252",
281+ } {
282+ if !strings.Contains(log, frag) {
283+ t.Errorf("API log lacks %q", frag)
284+ }
285+ }
286+ // The scratch disk was created sparse at the requested size, and formatted.
287+ var mkfs []string
288+ for _, l := range e.run.Lines() {
289+ if strings.HasPrefix(l, "mkfs.ext4") {
290+ mkfs = append(mkfs, l)
291+ }
292+ }
293+ if len(mkfs) == 0 {
294+ t.Fatalf("no mkfs call: %v", e.run.Lines())
295+ }
296+}
297+
298+func TestJobsDoNotShareSlotsOrTaps(t *testing.T) {
299+ e := newEnv(t, "")
300+ d := e.driver(t)
301+ var sbs []sandbox.Sandbox
302+ for i := range 2 {
303+ sb, err := d.New()
304+ if err != nil {
305+ t.Fatal(err)
306+ }
307+ conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: fmt.Sprint(i)})
308+ if err != nil {
309+ t.Fatal(err)
310+ }
311+ runStub(t, conn)
312+ sbs = append(sbs, sb)
313+ }
314+ if _, err := d.New(); !errors.Is(err, sandbox.ErrCapacity) {
315+ t.Fatalf("capacity not enforced: %v", err)
316+ }
317+ if e.tapCount() != 2 {
318+ t.Fatalf("taps: %v", e.taps)
319+ }
320+ for _, sb := range sbs {
321+ _ = sb.Stop(context.Background())
322+ }
323+ if left := e.leftovers(t); len(left) != 0 {
324+ t.Fatalf("leftovers: %v", left)
325+ }
326+}
327+
328+func TestStartFailureCleansEverything(t *testing.T) {
329+ e := newEnv(t, "die")
330+ d := e.driver(t)
331+ sb, _ := d.New()
332+ _, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
333+ if err == nil || !strings.Contains(err.Error(), "refusing to start") {
334+ t.Fatalf("want the VMM console in the error, got %v", err)
335+ }
336+ if left := e.leftovers(t); len(left) != 0 {
337+ t.Fatalf("leftovers after failed Start: %v", left)
338+ }
339+ if d.slots.InUse() != 0 {
340+ t.Fatal("slot leaked by a failed Start")
341+ }
342+}
343+
344+func TestBootTimeoutCleansEverything(t *testing.T) {
345+ e := newEnv(t, "hang")
346+ d, err := New(context.Background(), e.cfg, Options{Run: e.run, Net: e.plumb, BootTimeout: 300 * time.Millisecond})
347+ if err != nil {
348+ t.Fatal(err)
349+ }
350+ defer func() { _ = d.Close() }()
351+ sb, _ := d.New()
352+ _, err = sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
353+ if err == nil || !strings.Contains(err.Error(), "did not connect") {
354+ t.Fatalf("got %v", err)
355+ }
356+ if left := e.leftovers(t); len(left) != 0 {
357+ t.Fatalf("leftovers: %v", left)
358+ }
359+}
360+
361+func TestStartCancelledByContext(t *testing.T) {
362+ e := newEnv(t, "hang")
363+ d := e.driver(t)
364+ sb, _ := d.New()
365+ ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
366+ defer cancel()
367+ if _, err := sb.Start(ctx, sandbox.JobSpec{JobID: "j"}); !errors.Is(err, context.DeadlineExceeded) {
368+ t.Fatalf("got %v", err)
369+ }
370+ if left := e.leftovers(t); len(left) != 0 {
371+ t.Fatalf("leftovers: %v", left)
372+ }
373+}
374+
375+func TestNetworkAttachFailureCleans(t *testing.T) {
376+ e := newEnv(t, "")
377+ d := e.driver(t)
378+ e.plumb.FailNext = errors.New("nft broke")
379+ sb, _ := d.New()
380+ if _, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"}); err == nil || !strings.Contains(err.Error(), "nft broke") {
381+ t.Fatalf("got %v", err)
382+ }
383+ if left := e.leftovers(t); len(left) != 0 {
384+ t.Fatalf("leftovers: %v", left)
385+ }
386+}
387+
388+func TestStopWithoutStartReleasesSlot(t *testing.T) {
389+ e := newEnv(t, "")
390+ d := e.driver(t)
391+ sb, _ := d.New()
392+ if err := sb.Stop(context.Background()); err != nil {
393+ t.Fatal(err)
394+ }
395+ if d.slots.InUse() != 0 {
396+ t.Fatal("slot leaked")
397+ }
398+ if _, err := sb.Start(context.Background(), sandbox.JobSpec{}); err == nil {
399+ t.Fatal("Start after Stop accepted")
400+ }
401+}
402+
403+// Stop must finish the cleanup even when the caller's context is already done
404+// (a cancelled job).
405+func TestStopRunsWithCancelledContext(t *testing.T) {
406+ e := newEnv(t, "")
407+ d := e.driver(t)
408+ sb, _ := d.New()
409+ conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
410+ if err != nil {
411+ t.Fatal(err)
412+ }
413+ runStub(t, conn)
414+ ctx, cancel := context.WithCancel(context.Background())
415+ cancel()
416+ if err := sb.Stop(ctx); err != nil {
417+ t.Fatal(err)
418+ }
419+ if left := e.leftovers(t); len(left) != 0 {
420+ t.Fatalf("leftovers: %v", left)
421+ }
422+}
423+
424+// A killed runner leaves a running Firecracker, a TAP, nft rules and files. The
425+// next start must reap all of it (AC4: nothing survives, incl. killed jobs).
426+func TestReapAfterRunnerCrash(t *testing.T) {
427+ e := newEnv(t, "")
428+ d1 := e.driver(t)
429+ sb, _ := d1.New()
430+ conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
431+ if err != nil {
432+ t.Fatal(err)
433+ }
434+ runStub(t, conn)
435+ pid := sb.(*vmSandbox).rec.PID
436+ if err := syscall.Kill(pid, 0); err != nil {
437+ t.Fatal("fake VM not running")
438+ }
439+ // The runner dies: its lock goes away, nothing else is cleaned.
440+ _ = d1.lock.Close()
441+ d1.lock = nil
442+ _ = conn.Close()
443+ if e.tapCount() != 1 {
444+ t.Fatal("setup: no TAP")
445+ }
446+
447+ d2 := e.driver(t)
448+ if err := d2.Reap(context.Background()); err != nil {
449+ t.Fatal(err)
450+ }
451+ if left := e.leftovers(t); len(left) != 0 {
452+ t.Fatalf("leftovers after Reap: %v", left)
453+ }
454+ deadline := time.Now().Add(3 * time.Second)
455+ for syscall.Kill(pid, 0) == nil && time.Now().Before(deadline) {
456+ // The fake is a child of this test process: it is a zombie until the
457+ // original driver's Wait goroutine collects it.
458+ if b, err := os.ReadFile(fmt.Sprintf("/proc/%d/stat", pid)); err == nil && strings.Contains(string(b), ") Z") {
459+ break
460+ }
461+ time.Sleep(20 * time.Millisecond)
462+ }
463+ if b, _ := os.ReadFile(fmt.Sprintf("/proc/%d/stat", pid)); len(b) > 0 && !strings.Contains(string(b), ") Z") {
464+ t.Fatalf("firecracker %d survived Reap: %s", pid, b)
465+ }
466+ if e.plumb.Resets == 0 {
467+ t.Fatal("the nftables table was not reset")
468+ }
469+}
470+
471+func TestReapRemovesCorruptRecordAndTempFiles(t *testing.T) {
472+ e := newEnv(t, "")
473+ d := e.driver(t)
474+ _ = os.MkdirAll(d.store.dir, 0o700)
475+ _ = os.WriteFile(filepath.Join(d.store.dir, "er00112233.json"), []byte("{garbage"), 0o600)
476+ _ = os.WriteFile(filepath.Join(d.store.dir, "er44556677.json.tmp"), []byte("{"), 0o600)
477+ _ = os.MkdirAll(filepath.Join(e.cfg.StateDir, "vm", "er00112233"), 0o700)
478+ _ = os.MkdirAll(filepath.Join(e.cfg.StateDir, "vm", "not-ours"), 0o700) // must be left alone
479+ if err := d.Reap(context.Background()); err != nil {
480+ t.Fatal(err)
481+ }
482+ if _, err := os.Stat(filepath.Join(e.cfg.StateDir, "vm", "not-ours")); err != nil {
483+ t.Fatal("Reap removed a directory that is not ours")
484+ }
485+ _ = os.Remove(filepath.Join(e.cfg.StateDir, "vm", "not-ours"))
486+ if left := e.leftovers(t); len(left) != 0 {
487+ t.Fatalf("leftovers: %v", left)
488+ }
489+}
490+
491+func TestReapLeavesForeignTapsAlone(t *testing.T) {
492+ e := newEnv(t, "")
493+ e.taps["tap0"] = true
494+ e.taps["ertapx"] = true // not ertap<digits>
495+ e.taps["ertap7"] = true
496+ d := e.driver(t)
497+ if err := d.Reap(context.Background()); err != nil {
498+ t.Fatal(err)
499+ }
500+ if !e.taps["tap0"] || !e.taps["ertapx"] || e.taps["ertap7"] {
501+ t.Fatalf("taps after reap: %v", e.taps)
502+ }
503+}
504+
505+func TestSecondRunnerOnSameStateDirRefused(t *testing.T) {
506+ e := newEnv(t, "")
507+ _ = e.driver(t)
508+ _, err := New(context.Background(), e.cfg, Options{Run: e.run, Net: e.plumb})
509+ if err == nil || !strings.Contains(err.Error(), "another ermite-runner") {
510+ t.Fatalf("got %v", err)
511+ }
512+}
513+
514+func TestNewRejectsDynamicExecutable(t *testing.T) {
515+ e := newEnv(t, "")
516+ e.cfg.Executable = "/bin/sh"
517+ if _, err := New(context.Background(), e.cfg, Options{Run: e.run, Net: e.plumb}); err == nil {
518+ t.Skip("/bin/sh is static here")
519+ }
520+}
521+
522+func TestExecDiskIsKeyedByExecutableHash(t *testing.T) {
523+ e := newEnv(t, "")
524+ d1 := e.driver(t)
525+ first := d1.execDisk
526+ _ = d1.Close()
527+ // Same executable: reused, no second mkfs of an exec disk.
528+ n := len(e.run.Calls)
529+ d2 := e.driver(t)
530+ if d2.execDisk != first {
531+ t.Fatal("exec disk rebuilt for the same executable")
532+ }
533+ for _, c := range e.run.Calls[n:] {
534+ if c.Name == "mkfs.ext4" {
535+ t.Fatal("mkfs run again for an unchanged executable")
536+ }
537+ }
538+ _ = d2.Close()
539+ // A runner upgrade: new disk, old one removed.
540+ if err := os.WriteFile(e.cfg.Executable, append(readFile(t, e.cfg.Executable), 0), 0o755); err != nil {
541+ t.Fatal(err)
542+ }
543+ d3 := e.driver(t)
544+ if d3.execDisk == first {
545+ t.Fatal("exec disk not rebuilt after the executable changed")
546+ }
547+ if _, err := os.Stat(first); err == nil {
548+ t.Fatal("stale exec disk kept")
549+ }
550+}
551+
552+func readFile(t *testing.T, p string) []byte {
553+ t.Helper()
554+ b, err := os.ReadFile(p)
555+ if err != nil {
556+ t.Fatal(err)
557+ }
558+ return b
559+}
560+
561+func TestJailedLayout(t *testing.T) {
562+ l := newLayout("/var/lib/er", "firecracker", "er00112233", true, 900003)
563+ if l.root != "/var/lib/er/jail/firecracker/er00112233/root" || l.dir != "/var/lib/er/jail/firecracker/er00112233" {
564+ t.Fatalf("%+v", l)
565+ }
566+ if l.fc("scratch.ext4") != "/scratch.ext4" || l.host("scratch.ext4") != l.root+"/scratch.ext4" {
567+ t.Fatal("jailed paths")
568+ }
569+ if jailBase("/var/lib/er") != "/var/lib/er/jail" {
570+ t.Fatal("jail base")
571+ }
572+ if l2 := newLayout("/var/lib/er", "firecracker-v1.17", "er00112233", true, 1); l2.dir != "/var/lib/er/jail/firecracker-v1.17/er00112233" {
573+ t.Fatalf("the jailer names its chroot after the exec file: %s", l2.dir)
574+ }
575+ u := newLayout("/var/lib/er", "firecracker", "er00112233", false, 1000)
576+ if u.fc("api.sock") != "/var/lib/er/vm/er00112233/api.sock" {
577+ t.Fatalf("unjailed path %s", u.fc("api.sock"))
578+ }
579+ if p, _ := u.share("/img/base.ext4", "base.ext4"); p != "/img/base.ext4" {
580+ t.Fatal("unjailed share must use the file in place")
581+ }
582+}
583+
584+func TestJailedShareHardLinks(t *testing.T) {
585+ dir := t.TempDir()
586+ src := filepath.Join(dir, "base.ext4")
587+ _ = os.WriteFile(src, []byte("data"), 0o644)
588+ l := newLayout(dir, "firecracker", "er00112233", true, os.Getuid())
589+ if err := os.MkdirAll(l.root, 0o700); err != nil {
590+ t.Fatal(err)
591+ }
592+ p, err := l.share(src, "base.ext4")
593+ if err != nil || p != "/base.ext4" {
594+ t.Fatalf("%q %v", p, err)
595+ }
596+ a, _ := os.Stat(src)
597+ b, _ := os.Stat(l.host("base.ext4"))
598+ if !os.SameFile(a, b) {
599+ t.Fatal("shared file must be a hard link, never a copy")
600+ }
601+ if err := checkWorldReadable(src); err != nil {
602+ t.Fatal(err)
603+ }
604+ _ = os.Chmod(src, 0o600)
605+ if err := checkWorldReadable(src); err == nil {
606+ t.Fatal("unreadable-by-others file accepted for the jail")
607+ }
608+}
609+
610+func TestBootArgs(t *testing.T) {
611+ a := bootArgs(mustAddr("10.200.0.6"), mustAddr("10.200.0.5"), 1791818522)
612+ for _, want := range []string{
613+ "ipv6.disable=1", "root=/dev/vda ro", "init=/ermite-runner", "reboot=k", "panic=1",
614+ "ip=10.200.0.6::10.200.0.5:255.255.255.252:ermite-job:eth0:off:10.200.0.5",
615+ "ermite.sandbox=vm", "ermite.connect=vsock:1024", "ermite.gw=10.200.0.5", "ermite.time=1791818522",
616+ } {
617+ if !strings.Contains(a, want) {
618+ t.Errorf("boot args lack %q: %s", want, a)
619+ }
620+ }
621+ if !strings.HasSuffix(a, "-- init") {
622+ t.Errorf("init subcommand missing: %s", a)
623+ }
624+ if macFor(0) != "06:00:ac:10:00:00" || macFor(257) != "06:00:ac:10:01:01" {
625+ t.Fatal(macFor(257))
626+ }
627+}
628+
629+func TestParseTapNames(t *testing.T) {
630+ in := "1: lo: <LOOPBACK> mtu 65536\n2: eth0: <UP> mtu 1500\n5: ertap0: <BROADCAST> mtu 1500\n9: ertap12@NONE: <X>\n10: ertapx: <X>\n11: tap0: <X>\n12: ertap: <X>\n"
631+ got := strings.Join(parseTapNames(in), ",")
632+ if got != "ertap0,ertap12" {
633+ t.Fatalf("got %q", got)
634+ }
635+}
636+
637+func TestFirecrackerProcsScan(t *testing.T) {
638+ proc := t.TempDir()
639+ mk := func(pid, cmdline string) {
640+ _ = os.MkdirAll(filepath.Join(proc, pid), 0o755)
641+ _ = os.WriteFile(filepath.Join(proc, pid, "cmdline"), []byte(cmdline), 0o644)
642+ }
643+ mk("100", "firecracker\x00--id\x00er00112233\x00--api-sock\x00/x\x00")
644+ mk("101", "/usr/bin/firecracker\x00--api-sock\x00/x\x00--id\x00er44556677\x00")
645+ mk("102", "firecracker\x00--id\x00other-tool-vm\x00") // someone else's VM
646+ mk("103", "bash\x00--id\x00er00112233\x00") // not firecracker
647+ mk("1", "firecracker\x00--id\x00er99999999\x00") // pid 1 is never touched
648+ mk("abc", "")
649+ got := firecrackerProcs(proc)
650+ if len(got) != 2 || got[100] != "er00112233" || got[101] != "er44556677" {
651+ t.Fatalf("%v", got)
652+ }
653+}
654+
655+func TestRingKeepsTail(t *testing.T) {
656+ r := newRing(8)
657+ _, _ = r.Write([]byte("0123456789"))
658+ _, _ = r.Write([]byte("ab"))
659+ if r.String() != "23456789ab"[2:] {
660+ t.Fatalf("%q", r.String())
661+ }
662+}
663+
664+func TestAPIErrorsSurfaceFirecrackerFault(t *testing.T) {
665+ sock := filepath.Join(t.TempDir(), "s")
666+ ln, err := net.Listen("unix", sock)
667+ if err != nil {
668+ t.Fatal(err)
669+ }
670+ defer func() { _ = ln.Close() }()
671+ go func() {
672+ _ = http.Serve(ln, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
673+ w.WriteHeader(400)
674+ _, _ = w.Write([]byte(`{"fault_message":"Invalid request"}`))
675+ }))
676+ }()
677+ err = newAPIClient(sock).put(context.Background(), "/boot-source", bootSource{})
678+ if err == nil || !strings.Contains(err.Error(), "Invalid request") || !strings.Contains(err.Error(), "/boot-source") {
679+ t.Fatalf("got %v", err)
680+ }
681+}
682+
683+func mustPrefix(s string) netip.Prefix { return netip.MustParsePrefix(s) }
684+func mustAddr(s string) netip.Addr { return netip.MustParseAddr(s) }

Keyboard shortcuts

Single-key shortcuts are on. ctrl k and esc work whatever this says.

ctrl k :
go to a section, branch, tag, file or repository
?
this list
j k
move the row cursor down / up
enter
open the row under the cursor; on Files, a directory opens or closes in place
↑ ↓ ← → home end
in the file tree (focus on a row): previous / next row, close / open a directory or go to its parent / first entry, first / last row. They never open a file
l h
on Files: open a directory or a file / close a directory or go up to its parent, then to the parent page. Elsewhere: into the side panel (on a merge request, its files) / back to the content; there j k move between its controls
g g G
first / last row
/
filter the list; on Files, find a file anywhere in the repository (esc clears, enter opens the best match)
b
switch branch or tag
g s g f g c g b g m g i g r g a
tabs: summary, files, commits, branches & tags, merge requests, issues, releases, actions (when available)
g h
home
w
wide mode: use the whole width of the screen, or not (remembered)
y
copy a permalink pinned to the commit
d u x
diff: split / unified / collapse or expand all
e E
diff: the whole file in view / every file, with the lines the hunks hide
n
merge request: next note in the diff
esc
close a dialog, leave a field

Branches 4

Tags 11

↑↓ moveenter openesc close