fix(sandbox/vm): jailer cgroup v2 limits and PID namespace, rate limiters, keep network while the VMM may run, abortable boot, reap by --id, Lchown/fchown

commit: 2f75a567ab084fc5eee50d32b4091e58f7b74a50
author: brice · 10d ago · 2026-10-12 16:10 UTC
parent: 1905b39
signature: GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email

7 files · +803 -172

M internal/sandbox/vm/api.go +50 -7


@@ -32,16 +32,59 @@ type bootSource struct {
32	32		}
33	33
34	34		type drive struct {
35		-	DriveID string `json:"drive_id"`
36		-	PathOnHost string `json:"path_on_host"`
37		-	IsRootDevice bool `json:"is_root_device"`
38		-	IsReadOnly bool `json:"is_read_only"`
	35	+	DriveID string `json:"drive_id"`
	36	+	PathOnHost string `json:"path_on_host"`
	37	+	IsRootDevice bool `json:"is_root_device"`
	38	+	IsReadOnly bool `json:"is_read_only"`
	39	+	RateLimiter *rateLimiter `json:"rate_limiter,omitempty"`
39	40		}
40	41
41	42		type networkInterface struct {
42		-	IfaceID string `json:"iface_id"`
43		-	HostDevName string `json:"host_dev_name"`
44		-	GuestMAC string `json:"guest_mac"`
	43	+	IfaceID string `json:"iface_id"`
	44	+	HostDevName string `json:"host_dev_name"`
	45	+	GuestMAC string `json:"guest_mac"`
	46	+	RxRateLimiter *rateLimiter `json:"rx_rate_limiter,omitempty"`
	47	+	TxRateLimiter *rateLimiter `json:"tx_rate_limiter,omitempty"`
	48	+	}
	49	+
	50	+	// tokenBucket is Firecracker's rate limiter bucket: Size tokens (bytes or
	51	+	// operations) are refilled every RefillTime milliseconds.
	52	+	type tokenBucket struct {
	53	+	Size int64 `json:"size"`
	54	+	OneTimeBurst int64 `json:"one_time_burst,omitempty"`
	55	+	RefillTime int64 `json:"refill_time"`
	56	+	}
	57	+
	58	+	type rateLimiter struct {
	59	+	Bandwidth *tokenBucket `json:"bandwidth,omitempty"`
	60	+	Ops *tokenBucket `json:"ops,omitempty"`
	61	+	}
	62	+
	63	+	// diskLimiter limits a disk to bandwidthMiB MiB/s and iops operations/s (zero
	64	+	// disables a limit; both zero means no limiter). A one-second burst is allowed
	65	+	// at the start so that short bursts (a git checkout) are not slowed down.
	66	+	func diskLimiter(bandwidthMiB, iops int) *rateLimiter {
	67	+	var rl rateLimiter
	68	+	if bandwidthMiB > 0 {
	69	+	b := int64(bandwidthMiB) << 20
	70	+	rl.Bandwidth = &tokenBucket{Size: b, OneTimeBurst: b, RefillTime: 1000}
	71	+	}
	72	+	if iops > 0 {
	73	+	rl.Ops = &tokenBucket{Size: int64(iops), OneTimeBurst: int64(iops), RefillTime: 1000}
	74	+	}
	75	+	if rl.Bandwidth == nil && rl.Ops == nil {
	76	+	return nil
	77	+	}
	78	+	return &rl
	79	+	}
	80	+
	81	+	// netLimiter limits one direction of a network interface to mbit Mbit/s.
	82	+	func netLimiter(mbit int) *rateLimiter {
	83	+	if mbit <= 0 {
	84	+	return nil
	85	+	}
	86	+	b := int64(mbit) * 1_000_000 / 8
	87	+	return &rateLimiter{Bandwidth: &tokenBucket{Size: b, OneTimeBurst: b, RefillTime: 1000}}
45	88		}
46	89
47	90		type vsockDevice struct {


@@ -32,16 +32,59 @@ type bootSource struct {
32		}	32		}
33			33
34		type drive struct {	34		type drive struct {
35	-	DriveID string `json:"drive_id"`	35	+	DriveID string `json:"drive_id"`
36	-	PathOnHost string `json:"path_on_host"`	36	+	PathOnHost string `json:"path_on_host"`
37	-	IsRootDevice bool `json:"is_root_device"`	37	+	IsRootDevice bool `json:"is_root_device"`
38	-	IsReadOnly bool `json:"is_read_only"`	38	+	IsReadOnly bool `json:"is_read_only"`
			39	+	RateLimiter *rateLimiter `json:"rate_limiter,omitempty"`
39		}	40		}
40			41
41		type networkInterface struct {	42		type networkInterface struct {
42	-	IfaceID string `json:"iface_id"`	43	+	IfaceID string `json:"iface_id"`
43	-	HostDevName string `json:"host_dev_name"`	44	+	HostDevName string `json:"host_dev_name"`
44	-	GuestMAC string `json:"guest_mac"`	45	+	GuestMAC string `json:"guest_mac"`
			46	+	RxRateLimiter *rateLimiter `json:"rx_rate_limiter,omitempty"`
			47	+	TxRateLimiter *rateLimiter `json:"tx_rate_limiter,omitempty"`
			48	+	}
			49	+
			50	+	// tokenBucket is Firecracker's rate limiter bucket: Size tokens (bytes or
			51	+	// operations) are refilled every RefillTime milliseconds.
			52	+	type tokenBucket struct {
			53	+	Size int64 `json:"size"`
			54	+	OneTimeBurst int64 `json:"one_time_burst,omitempty"`
			55	+	RefillTime int64 `json:"refill_time"`
			56	+	}
			57	+
			58	+	type rateLimiter struct {
			59	+	Bandwidth *tokenBucket `json:"bandwidth,omitempty"`
			60	+	Ops *tokenBucket `json:"ops,omitempty"`
			61	+	}
			62	+
			63	+	// diskLimiter limits a disk to bandwidthMiB MiB/s and iops operations/s (zero
			64	+	// disables a limit; both zero means no limiter). A one-second burst is allowed
			65	+	// at the start so that short bursts (a git checkout) are not slowed down.
			66	+	func diskLimiter(bandwidthMiB, iops int) *rateLimiter {
			67	+	var rl rateLimiter
			68	+	if bandwidthMiB > 0 {
			69	+	b := int64(bandwidthMiB) << 20
			70	+	rl.Bandwidth = &tokenBucket{Size: b, OneTimeBurst: b, RefillTime: 1000}
			71	+	}
			72	+	if iops > 0 {
			73	+	rl.Ops = &tokenBucket{Size: int64(iops), OneTimeBurst: int64(iops), RefillTime: 1000}
			74	+	}
			75	+	if rl.Bandwidth == nil && rl.Ops == nil {
			76	+	return nil
			77	+	}
			78	+	return &rl
			79	+	}
			80	+
			81	+	// netLimiter limits one direction of a network interface to mbit Mbit/s.
			82	+	func netLimiter(mbit int) *rateLimiter {
			83	+	if mbit <= 0 {
			84	+	return nil
			85	+	}
			86	+	b := int64(mbit) * 1_000_000 / 8
			87	+	return &rateLimiter{Bandwidth: &tokenBucket{Size: b, OneTimeBurst: b, RefillTime: 1000}}
45		}	88		}
46			89
47		type vsockDevice struct {	90		type vsockDevice struct {

M internal/sandbox/vm/driver.go +283 -119


@@ -20,12 +20,12 @@ import (
20	20		"os"
21	21		"os/exec"
22	22		"path/filepath"
	23	+	"strconv"
23	24		"sync"
24	25		"syscall"
25	26		"time"
26	27
27	28		"github.com/bnema/zerowrap"
28		-	"golang.org/x/sys/unix"
29	29
30	30		"bnema.dev/ermite-runner/internal/cmdrun"
31	31		"bnema.dev/ermite-runner/internal/netpolicy"

@@ -36,8 +36,11 @@ import (
36	36		// Options are the testable knobs of the driver.
37	37		type Options struct {
38	38		Run cmdrun.Runner
39		-	// ProcDir is the /proc used to find orphan Firecracker processes.
	39	+	// ProcDir is the /proc used to find orphan VMM processes.
40	40		ProcDir string
	41	+	// CgroupRoot is the cgroup v2 mount, where the jailer creates one cgroup per
	42	+	// VM (default /sys/fs/cgroup).
	43	+	CgroupRoot string
41	44		// BootTimeout bounds boot + init + executor connection (default 90 s).
42	45		BootTimeout time.Duration
43	46		// Net overrides the network plumbing (tests).

@@ -45,8 +48,17 @@ type Options struct {
45	48		// FirecrackerArgs are extra arguments for Firecracker (tests use them to
46	49		// select the fake).
47	50		FirecrackerArgs []string
	51	+	// FreeSpace overrides statfs for the free-space warning (tests).
	52	+	FreeSpace func(string) (int64, error)
	53	+	// kill overrides killAndWait (tests simulate a VMM that cannot be killed).
	54	+	kill func(pid int, timeout time.Duration) bool
48	55		}
49	56
	57	+	// vmmOverheadMiB is what the Firecracker process itself may use on top of the
	58	+	// guest memory (device emulation, the API server, the seccomp filter, stacks):
	59	+	// it is added to memory.max so that the guest, not the VMM, is what gets limited.
	60	+	const vmmOverheadMiB = 256
	61	+
50	62		// Driver implements sandbox.Driver for micro-VMs.
51	63		type Driver struct {
52	64		cfg sandbox.Config

@@ -56,14 +68,15 @@ type Driver struct {
56	68		execDisk string
57	69		store store
58	70		net netsetup.Plumbing
59		-	lock *os.File
	71	+	lock *sandbox.HostLock
60	72
61	73		mu sync.Mutex
62	74		slots *sandbox.Slots
63	75		}
64	76
65		-	// New builds the driver: resolves binaries, builds the exec disk, takes the
66		-	// state-directory lock and enables IPv4 forwarding. Run Reap before New sandboxes.
	77	+	// New builds the driver: takes the host-wide lock, resolves binaries, builds the
	78	+	// exec disk and enables IPv4 forwarding (a host-wide sysctl that stays on: see
	79	+	// the README). Run Reap before New sandboxes.
67	80		func New(ctx context.Context, cfg sandbox.Config, opt Options) (*Driver, error) {
68	81		if opt.Run == nil {
69	82		opt.Run = cmdrun.Exec{}

@@ -71,9 +84,15 @@ func New(ctx context.Context, cfg sandbox.Config, opt Options) (*Driver, error)
71	84		if opt.ProcDir == "" {
72	85		opt.ProcDir = "/proc"
73	86		}
	87	+	if opt.CgroupRoot == "" {
	88	+	opt.CgroupRoot = "/sys/fs/cgroup"
	89	+	}
74	90		if opt.BootTimeout == 0 {
75	91		opt.BootTimeout = 90 * time.Second
76	92		}
	93	+	if opt.kill == nil {
	94	+	opt.kill = killAndWait
	95	+	}
77	96		d := &Driver{cfg: cfg, opt: opt, slots: sandbox.NewSlots(cfg.Capacity), store: store{dir: filepath.Join(cfg.StateDir, "vms")}}
78	97
79	98		var err error

@@ -95,7 +114,7 @@ func New(ctx context.Context, cfg sandbox.Config, opt Options) (*Driver, error)
95	114		if err := os.MkdirAll(cfg.StateDir, 0o755); err != nil {
96	115		return nil, fmt.Errorf("state directory: %w", err)
97	116		}
98		-	if err := d.acquireLock(); err != nil {
	117	+	if d.lock, err = sandbox.AcquireHostLock(cfg.LockFile); err != nil {
99	118		return nil, err
100	119		}
101	120		fail := func(err error) (*Driver, error) { _ = d.Close(); return nil, err }

@@ -107,6 +126,7 @@ func New(ctx context.Context, cfg sandbox.Config, opt Options) (*Driver, error)
107	126		}
108	127		}
109	128		}
	129	+	sandbox.WarnIfLowSpace(ctx, cfg.StateDir, cfg.Capacity, cfg.JobDiskMiB, opt.FreeSpace)
110	130		if d.execDisk, err = ensureExecDisk(ctx, opt.Run, exe, cfg.StateDir); err != nil {
111	131		return fail(err)
112	132		}

@@ -138,44 +158,33 @@ func resolveBin(configured, name string) (string, error) {
138	158		return abs, nil
139	159		}
140	160
141		-	func (d *Driver) acquireLock() error {
142		-	f, err := os.OpenFile(filepath.Join(d.cfg.StateDir, "vm.lock"), os.O_CREATE\|os.O_RDWR, 0o600)
143		-	if err != nil {
144		-	return err
145		-	}
146		-	if err := unix.Flock(int(f.Fd()), unix.LOCK_EX\|unix.LOCK_NB); err != nil {
147		-	_ = f.Close()
148		-	return fmt.Errorf("another ermite-runner already uses state directory %s (reaping would destroy its VMs): %w", d.cfg.StateDir, err)
149		-	}
150		-	d.lock = f
151		-	return nil
152		-	}
153		-
154	161		// Mode implements sandbox.Driver.
155	162		func (d *Driver) Mode() sandbox.Mode { return sandbox.VM }
156	163
157		-	// Close releases the lock and the DNS listeners.
	164	+	// Isolation implements sandbox.Driver.
	165	+	func (d *Driver) Isolation() sandbox.Isolation { return sandbox.IsolationVM }
	166	+
	167	+	// Close releases the host lock and the DNS listeners.
158	168		func (d *Driver) Close() error {
159	169		if d.net != nil {
160	170		d.net.Close()
161	171		}
162		-	if d.lock != nil {
163		-	err := d.lock.Close()
164		-	d.lock = nil
165		-	return err
166		-	}
167		-	return nil
	172	+	return d.lock.Release()
168	173		}
169	174
170	175		// New implements sandbox.Driver.
171	176		func (d *Driver) New() (sandbox.Sandbox, error) {
	177	+	id, err := sandbox.NewID()
	178	+	if err != nil {
	179	+	return nil, err
	180	+	}
172	181		d.mu.Lock()
173	182		defer d.mu.Unlock()
174	183		slot, err := d.slots.Acquire()
175	184		if err != nil {
176	185		return nil, err
177	186		}
178		-	return &vmSandbox{d: d, slot: slot}, nil
	187	+	return &vmSandbox{d: d, slot: slot, id: id}, nil
179	188		}
180	189
181	190		func (d *Driver) release(slot int) {

@@ -184,14 +193,26 @@ func (d *Driver) release(slot int) {
184	193		d.mu.Unlock()
185	194		}
186	195
	196	+	func (d *Driver) markLeaked(slot int) {
	197	+	d.mu.Lock()
	198	+	d.slots.MarkLeaked(slot)
	199	+	d.mu.Unlock()
	200	+	}
	201	+
187	202		// vmSandbox is one VM, single use.
188	203		type vmSandbox struct {
189	204		d *Driver
190	205		slot int
	206	+	id string
	207	+	gate sandbox.BootGate
	208	+
	209	+	mu sync.Mutex
	210	+	started bool
	211	+	stopped bool
	212	+	// leakErr is the error of a cleanup that left something behind; later Stop
	213	+	// calls return it again.
	214	+	leakErr error
191	215
192		-	mu sync.Mutex
193		-	started bool
194		-	stopped bool
195	216		rec record
196	217		lay layout
197	218		cmd *exec.Cmd

@@ -204,10 +225,20 @@ type vmSandbox struct {
204	225		attached bool
205	226		}
206	227
	228	+	// ID implements sandbox.Sandbox.
	229	+	func (s *vmSandbox) ID() string { return s.id }
	230	+
207	231		func (s *vmSandbox) uid() int { return s.d.cfg.UIDBase + s.slot }
208	232
209	233		// Start implements sandbox.Sandbox.
210	234		func (s *vmSandbox) Start(ctx context.Context, spec sandbox.JobSpec) (sandbox.Conn, error) {
	235	+	// A Stop from another goroutine aborts the boot through the gate.
	236	+	bootCtx, err := s.gate.Begin(ctx)
	237	+	if err != nil {
	238	+	return nil, fmt.Errorf("vm sandbox: %w", err)
	239	+	}
	240	+	defer s.gate.End()
	241	+
211	242		s.mu.Lock()
212	243		defer s.mu.Unlock()
213	244		if s.started \|\| s.stopped {

@@ -218,18 +249,13 @@ func (s *vmSandbox) Start(ctx context.Context, spec sandbox.JobSpec) (sandbox.Co
218	249		d := s.d
219	250		spec = d.cfg.Resolve(spec)
220	251
221		-	id, err := sandbox.NewID()
222		-	if err != nil {
223		-	s.releaseLocked()
224		-	return nil, err
225		-	}
226	252		jailed := !d.cfg.JailerOff
227	253		runUID := s.uid()
228	254		if !jailed {
229	255		runUID = os.Getuid()
230	256		}
231		-	s.lay = newLayout(d.cfg.StateDir, filepath.Base(d.fcPath), id, jailed, runUID)
232		-	s.rec = record{ID: id, Slot: s.slot, Job: spec.JobID, Dir: s.lay.dir}
	257	+	s.lay = newLayout(d.cfg.StateDir, filepath.Base(d.fcPath), s.id, jailed, runUID)
	258	+	s.rec = record{ID: s.id, Slot: s.slot, Job: spec.JobID, Dir: s.lay.dir}
233	259		// The record exists before the first resource, so any crash from here on is
234	260		// reapable.
235	261		if err := d.store.save(s.rec); err != nil {

@@ -237,7 +263,7 @@ func (s *vmSandbox) Start(ctx context.Context, spec sandbox.JobSpec) (sandbox.Co
237	263		return nil, fmt.Errorf("write state record: %w", err)
238	264		}
239	265
240		-	conn, err := s.boot(ctx, spec, id)
	266	+	conn, err := s.boot(bootCtx, spec)
241	267		if err != nil {
242	268		console := ""
243	269		if s.console != nil {

@@ -249,98 +275,140 @@ func (s *vmSandbox) Start(ctx context.Context, spec sandbox.JobSpec) (sandbox.Co
249	275		}
250	276		return nil, errors.Join(err, cerr)
251	277		}
252		-	log.Info().Str("vm", id).Int("slot", s.slot).Str("job", spec.JobID).Msg("micro-VM running")
	278	+	log.Info().Str("vm", s.id).Int("slot", s.slot).Str("job", spec.JobID).Msg("micro-VM running")
253	279		return conn, nil
254	280		}
255	281
256		-	func (s *vmSandbox) boot(ctx context.Context, spec sandbox.JobSpec, id string) (sandbox.Conn, error) {
257		-	d := s.d
258		-	netw, err := netpolicy.Subnet(d.cfg.Network, s.slot)
	282	+	// boot runs the stages in order; each stage records what it created so that
	283	+	// cleanupLocked can undo exactly that.
	284	+	func (s *vmSandbox) boot(ctx context.Context, spec sandbox.JobSpec) (sandbox.Conn, error) {
	285	+	netw, err := netpolicy.Subnet(s.d.cfg.Network, s.slot)
	286	+	if err != nil {
	287	+	return nil, err
	288	+	}
	289	+	if err := s.bootNetwork(ctx, netw); err != nil {
	290	+	return nil, err
	291	+	}
	292	+	api, err := s.bootVMM(ctx, spec)
	293	+	if err != nil {
	294	+	return nil, err
	295	+	}
	296	+	files, err := s.bootFiles(ctx, spec)
	297	+	if err != nil {
	298	+	return nil, err
	299	+	}
	300	+	accepted, err := s.bootVsock()
259	301		if err != nil {
260	302		return nil, err
261	303		}
	304	+	if err := api.configure(ctx, s.vmConfig(spec, netw, files)); err != nil {
	305	+	return nil, err
	306	+	}
	307	+	return s.waitExecutor(ctx, accepted)
	308	+	}
262	309
263		-	// Network first: the TAP must exist when Firecracker opens it.
	310	+	// bootNetwork creates the TAP (it must exist when Firecracker opens it) and
	311	+	// installs the firewall policy and the DNS listener.
	312	+	func (s *vmSandbox) bootNetwork(ctx context.Context, netw netpolicy.Net) error {
	313	+	d := s.d
264	314		s.tap, s.gateway = tapName(s.slot), netw.Gateway
265	315		s.rec.Tap, s.rec.Gate = s.tap, s.gateway.String()
266	316		if err := d.store.save(s.rec); err != nil {
267		-	return nil, err
	317	+	return err
268	318		}
269	319		if err := createTap(ctx, d.opt.Run, s.tap, s.lay.uid, s.gateway); err != nil {
270		-	return nil, err
	320	+	return err
271	321		}
272	322		s.attached = true // from here Detach is idempotent and always safe
273		-	if err := d.net.Attach(ctx, netpolicy.Guest{Iface: s.tap, Gateway: netw.Gateway, Addr: netw.Guest}); err != nil {
274		-	return nil, err
275		-	}
	323	+	return d.net.Attach(ctx, netpolicy.Guest{Iface: s.tap, Gateway: netw.Gateway, Addr: netw.Guest})
	324	+	}
276	325
277		-	// The VMM process.
	326	+	// bootVMM starts the jailer (or Firecracker) and waits for its API.
	327	+	func (s vmSandbox) bootVMM(ctx context.Context, spec sandbox.JobSpec) (apiClient, error) {
278	328		apiSock := s.lay.host("api.sock")
279	329		if s.lay.jailed {
280	330		apiSock = s.lay.host("run/firecracker.socket")
281	331		} else if err := os.MkdirAll(s.lay.root, 0o700); err != nil {
282	332		return nil, err
283	333		}
284		-	if err := s.startVMM(id, apiSock); err != nil {
	334	+	if err := s.startVMM(spec, apiSock); err != nil {
285	335		return nil, err
286	336		}
287	337		api := newAPIClient(apiSock)
288	338		if err := s.waitAPI(ctx, api, apiSock); err != nil {
289	339		return nil, err
290	340		}
	341	+	return api, nil
	342	+	}
291	343
292		-	// Files: the process only opens them when configured below.
293		-	execDisk, err := s.lay.share(d.execDisk, "exec.ext4")
294		-	if err != nil {
295		-	return nil, err
296		-	}
297		-	baseDisk, err := s.lay.share(d.cfg.BaseImagePath(), "base.ext4")
298		-	if err != nil {
299		-	return nil, err
	344	+	// vmFiles are the paths Firecracker is given.
	345	+	type vmFiles struct{ execDisk, baseDisk, kernel string }
	346	+
	347	+	// bootFiles makes the shared files available and creates the scratch disk.
	348	+	func (s *vmSandbox) bootFiles(ctx context.Context, spec sandbox.JobSpec) (vmFiles, error) {
	349	+	d := s.d
	350	+	var f vmFiles
	351	+	var err error
	352	+	if f.execDisk, err = s.lay.share(d.execDisk, "exec.ext4"); err != nil {
	353	+	return f, err
300	354		}
301		-	kernel, err := s.lay.share(d.cfg.KernelPath(), "kernel")
302		-	if err != nil {
303		-	return nil, err
	355	+	if f.baseDisk, err = s.lay.share(d.cfg.BaseImagePath(), "base.ext4"); err != nil {
	356	+	return f, err
304	357		}
305		-	if err := makeScratch(ctx, d.opt.Run, s.lay.host("scratch.ext4"), spec.DiskMiB); err != nil {
306		-	return nil, err
	358	+	if f.kernel, err = s.lay.share(d.cfg.KernelPath(), "kernel"); err != nil {
	359	+	return f, err
307	360		}
308		-	if err := s.lay.own("scratch.ext4"); err != nil {
309		-	return nil, err
	361	+	owner := -1
	362	+	if s.lay.jailed {
	363	+	owner = s.lay.uid
310	364		}
	365	+	return f, makeScratch(ctx, d.opt.Run, s.lay.host("scratch.ext4"), spec.DiskMiB, owner)
	366	+	}
311	367
312		-	// The host side of the vsock stream: Firecracker connects to
313		-	// <uds_path>_<port> when the guest connects to the host CID.
314		-	udsName := "v.sock"
315		-	ln, err := net.Listen("unix", s.lay.host(udsName)+fmt.Sprintf("_%d", sandbox.VsockPort))
	368	+	// bootVsock listens on the host side of the vsock stream: Firecracker connects
	369	+	// to <uds_path>_<port> when the guest connects to the host CID.
	370	+	func (s *vmSandbox) bootVsock() (<-chan acceptResult, error) {
	371	+	path := s.lay.host(udsName) + fmt.Sprintf("_%d", sandbox.VsockPort)
	372	+	ln, err := net.Listen("unix", path)
316	373		if err != nil {
317	374		return nil, fmt.Errorf("listen vsock socket: %w", err)
318	375		}
319	376		s.listener = ln
320	377		if s.lay.jailed {
321		-	if err := os.Chown(s.lay.host(udsName+fmt.Sprintf("_%d", sandbox.VsockPort)), s.lay.uid, s.lay.uid); err != nil {
	378	+	if err := os.Lchown(path, s.lay.uid, s.lay.uid); err != nil {
322	379		return nil, err
323	380		}
324	381		}
325	382		accepted := make(chan acceptResult, 1)
326	383		go func() { c, err := ln.Accept(); accepted <- acceptResult{c, err} }()
	384	+	return accepted, nil
	385	+	}
	386	+
	387	+	const udsName = "v.sock"
327	388
328		-	err = api.configure(ctx, vmConfig{
	389	+	func (s *vmSandbox) vmConfig(spec sandbox.JobSpec, netw netpolicy.Net, f vmFiles) vmConfig {
	390	+	cfg := s.d.cfg
	391	+	return vmConfig{
329	392		Machine: machineConfig{VcpuCount: spec.CPUs, MemSizeMiB: spec.MemoryMiB},
330		-	Boot: bootSource{KernelImagePath: kernel, BootArgs: bootArgs(netw.Guest, netw.Gateway, time.Now().Unix())},
	393	+	Boot: bootSource{KernelImagePath: f.kernel, BootArgs: bootArgs(netw.Guest, netw.Gateway, time.Now().Unix())},
331	394		Drives: []drive{
332		-	{DriveID: "rootfs", PathOnHost: execDisk, IsRootDevice: true, IsReadOnly: true},
333		-	{DriveID: "base", PathOnHost: baseDisk, IsReadOnly: true},
334		-	{DriveID: "scratch", PathOnHost: s.lay.fc("scratch.ext4")},
	395	+	{DriveID: "rootfs", PathOnHost: f.execDisk, IsRootDevice: true, IsReadOnly: true},
	396	+	{DriveID: "base", PathOnHost: f.baseDisk, IsReadOnly: true},
	397	+	{DriveID: "scratch", PathOnHost: s.lay.fc("scratch.ext4"), RateLimiter: diskLimiter(cfg.DiskBandwidthMiB, cfg.DiskIOPS)},
	398	+	},
	399	+	Net: networkInterface{
	400	+	IfaceID: "eth0", HostDevName: s.tap, GuestMAC: macFor(s.slot),
	401	+	RxRateLimiter: netLimiter(cfg.NetMbit), TxRateLimiter: netLimiter(cfg.NetMbit),
335	402		},
336		-	Net: networkInterface{IfaceID: "eth0", HostDevName: s.tap, GuestMAC: macFor(s.slot)},
337	403		Vsock: vsockDevice{GuestCID: 3, UDSPath: s.lay.fc(udsName)},
338		-	})
339		-	if err != nil {
340		-	return nil, err
341	404		}
	405	+	}
342	406
343		-	timer := time.NewTimer(d.opt.BootTimeout)
	407	+	// waitExecutor waits for the guest's executor to connect. Only ctx (the caller's
	408	+	// context, or Stop through the boot gate) and the boot timeout end the wait: a
	409	+	// boot cannot be aborted any other way.
	410	+	func (s *vmSandbox) waitExecutor(ctx context.Context, accepted <-chan acceptResult) (sandbox.Conn, error) {
	411	+	timer := time.NewTimer(s.d.opt.BootTimeout)
344	412		defer timer.Stop()
345	413		select {
346	414		case r := <-accepted:

@@ -352,9 +420,9 @@ func (s *vmSandbox) boot(ctx context.Context, spec sandbox.JobSpec, id string) (
352	420		case <-s.exited:
353	421		return nil, errors.New("firecracker exited before the executor connected")
354	422		case <-timer.C:
355		-	return nil, fmt.Errorf("executor did not connect within %s", d.opt.BootTimeout)
	423	+	return nil, fmt.Errorf("executor did not connect within %s", s.d.opt.BootTimeout)
356	424		case <-ctx.Done():
357		-	return nil, ctx.Err()
	425	+	return nil, context.Cause(ctx)
358	426		}
359	427		}
360	428

@@ -363,18 +431,38 @@ type acceptResult struct {
363	431		err error
364	432		}
365	433
366		-	func (s *vmSandbox) startVMM(id, apiSock string) error {
	434	+	// jailerArgs builds the jailer command line (after the binary name).
	435	+	//
	436	+	// - cgroup v2 with memory.max (guest memory + VMM overhead) and pids.max (a
	437	+	// VMM needs a handful of threads per vCPU): a runaway or compromised VMM
	438	+	// cannot take the host's memory or fork-bomb it;
	439	+	// - a new PID namespace: the VMM cannot see or signal host processes;
	440	+	// - no-file: the descriptor budget of the VMM.
	441	+	func jailerArgs(jailer, fc, id string, uid int, chrootBase string, spec sandbox.JobSpec, extra []string) []string {
	442	+	memBytes := (int64(spec.MemoryMiB) + vmmOverheadMiB) << 20
	443	+	pids := 32 + 8*spec.CPUs
	444	+	args := []string{
	445	+	"--id", id, "--exec-file", fc,
	446	+	"--uid", strconv.Itoa(uid), "--gid", strconv.Itoa(uid),
	447	+	"--chroot-base-dir", chrootBase,
	448	+	"--cgroup-version", "2",
	449	+	"--cgroup", "memory.max=" + strconv.FormatInt(memBytes, 10),
	450	+	"--cgroup", "pids.max=" + strconv.Itoa(pids),
	451	+	"--new-pid-ns",
	452	+	"--resource-limit", "no-file=4096",
	453	+	"--",
	454	+	}
	455	+	return append(args, extra...)
	456	+	}
	457	+
	458	+	func (s *vmSandbox) startVMM(spec sandbox.JobSpec, apiSock string) error {
367	459		d := s.d
368	460		var cmd *exec.Cmd
369	461		if s.lay.jailed {
370		-	args := []string{"--id", id, "--exec-file", d.fcPath,
371		-	"--uid", fmt.Sprint(s.uid()), "--gid", fmt.Sprint(s.uid()),
372		-	"--chroot-base-dir", jailBase(d.cfg.StateDir),
373		-	"--resource-limit", "no-file=4096",
374		-	"--"}
375		-	cmd = exec.Command(d.jailer, append(args, d.opt.FirecrackerArgs...)...)
	462	+	args := jailerArgs(d.jailer, d.fcPath, s.id, s.uid(), jailBase(d.cfg.StateDir), spec, d.opt.FirecrackerArgs)
	463	+	cmd = exec.Command(d.jailer, args...)
376	464		} else {
377		-	args := append([]string{"--id", id, "--api-sock", apiSock}, d.opt.FirecrackerArgs...)
	465	+	args := append([]string{"--id", s.id, "--api-sock", apiSock}, d.opt.FirecrackerArgs...)
378	466		cmd = exec.Command(d.fcPath, args...)
379	467		}
380	468		s.console = newRing(16 << 10)

@@ -385,11 +473,36 @@ func (s *vmSandbox) startVMM(id, apiSock string) error {
385	473		}
386	474		s.cmd = cmd
387	475		s.exited = make(chan struct{})
388		-	go func() { _ = cmd.Wait(); close(s.exited) }()
	476	+	jailed := s.lay.jailed
	477	+	go func() {
	478	+	_ = cmd.Wait()
	479	+	// With --new-pid-ns the jailer may exit as soon as it has started the
	480	+	// VMM in its namespace: the VMM is then gone only when its own pid is.
	481	+	if jailed {
	482	+	s.awaitVMMGone()
	483	+	}
	484	+	close(s.exited)
	485	+	}()
389	486		s.rec.PID = cmd.Process.Pid
390	487		return s.d.store.save(s.rec)
391	488		}
392	489
	490	+	// awaitVMMGone blocks until no process of this VM's id is left.
	491	+	func (s *vmSandbox) awaitVMMGone() {
	492	+	for {
	493	+	alive := false
	494	+	for pid, id := range vmmProcs(s.d.opt.ProcDir) {
	495	+	if id == s.id && pidAlive(pid) {
	496	+	alive = true
	497	+	}
	498	+	}
	499	+	if !alive {
	500	+	return
	501	+	}
	502	+	time.Sleep(50 * time.Millisecond)
	503	+	}
	504	+	}
	505	+
393	506		func (s vmSandbox) waitAPI(ctx context.Context, api apiClient, sock string) error {
394	507		deadline := time.Now().Add(10 * time.Second)
395	508		for {

@@ -400,7 +513,7 @@ func (s vmSandbox) waitAPI(ctx context.Context, api apiClient, sock string) er
400	513		case <-s.exited:
401	514		return errors.New("firecracker exited before its API was ready")
402	515		case <-ctx.Done():
403		-	return ctx.Err()
	516	+	return context.Cause(ctx)
404	517		case <-time.After(25 * time.Millisecond):
405	518		}
406	519		if time.Now().After(deadline) {

@@ -411,13 +524,14 @@ func (s vmSandbox) waitAPI(ctx context.Context, api apiClient, sock string) er
411	524
412	525		// Stop implements sandbox.Sandbox.
413	526		func (s *vmSandbox) Stop(ctx context.Context) error {
	527	+	// Abort a boot in progress first: Start holds s.mu for its whole duration.
	528	+	s.gate.Abort()
414	529		s.mu.Lock()
415	530		defer s.mu.Unlock()
416	531		if s.stopped {
417		-	return nil
	532	+	return s.leakErr
418	533		}
419	534		if !s.started {
420		-	s.stopped = true
421	535		s.releaseLocked()
422	536		return nil
423	537		}

@@ -429,10 +543,31 @@ func (s *vmSandbox) releaseLocked() {
429	543		s.d.release(s.slot)
430	544		}
431	545
432		-	// cleanupLocked removes everything the VM created, in dependency order, and
433		-	// keeps going after an error. The state record and the slot are released only
434		-	// when everything is gone, so that a half-failed cleanup is retried by Reap
435		-	// rather than forgotten and the slot (hence TAP name) is not reused meanwhile.
	546	+	// vmmPIDs returns every host process of this VM: the launched command and
	547	+	// whatever the /proc scan finds with this id.
	548	+	func (s *vmSandbox) vmmPIDs() []int {
	549	+	set := map[int]bool{}
	550	+	if s.cmd != nil && s.cmd.Process != nil {
	551	+	set[s.cmd.Process.Pid] = true
	552	+	}
	553	+	for pid, id := range vmmProcs(s.d.opt.ProcDir) {
	554	+	if id == s.id {
	555	+	set[pid] = true
	556	+	}
	557	+	}
	558	+	var out []int
	559	+	for pid := range set {
	560	+	out = append(out, pid)
	561	+	}
	562	+	return out
	563	+	}
	564	+
	565	+	// cleanupLocked removes everything the VM created, in dependency order. Steps
	566	+	// that need the VMM to be dead (network device, firewall rules, files, cgroup)
	567	+	// are skipped when it could not be killed: removing the TAP or the policy under
	568	+	// a VM that may still run would break the isolation it relies on, so the
	569	+	// sandbox is reported as leaked (sandbox.ErrLeaked), its slot stays taken, and
	570	+	// Reap finishes the job.
436	571		func (s *vmSandbox) cleanupLocked(ctx context.Context) error {
437	572		log := zerowrap.FromCtx(ctx)
438	573		d := s.d

@@ -445,42 +580,60 @@ func (s *vmSandbox) cleanupLocked(ctx context.Context) error {
445	580		_ = s.listener.Close()
446	581		s.listener = nil
447	582		}
	583	+	dead := true
448	584		if s.cmd != nil && s.cmd.Process != nil {
449		-	killAndWait(s.cmd.Process.Pid, 5*time.Second)
	585	+	_ = syscall.Kill(-s.cmd.Process.Pid, syscall.SIGKILL) // the whole group
	586	+	}
	587	+	for _, pid := range s.vmmPIDs() {
	588	+	if !d.opt.kill(pid, 5*time.Second) {
	589	+	dead = false
	590	+	errs = append(errs, fmt.Errorf("vmm process %d did not exit", pid))
	591	+	}
	592	+	}
	593	+	if dead && s.exited != nil {
450	594		select {
451	595		case <-s.exited:
452	596		case <-time.After(5 * time.Second):
453		-	errs = append(errs, fmt.Errorf("firecracker pid %d did not exit", s.cmd.Process.Pid))
	597	+	dead = false
	598	+	errs = append(errs, errors.New("vmm exit was not observed"))
454	599		}
455	600		}
456		-	if s.attached {
457		-	if err := d.net.Detach(ctx, s.tap, s.gateway); err != nil {
458		-	errs = append(errs, err)
	601	+	if dead {
	602	+	if s.attached {
	603	+	if err := d.net.Detach(ctx, s.tap, s.gateway); err != nil {
	604	+	errs = append(errs, err)
	605	+	}
	606	+	if err := deleteTap(ctx, d.opt.Run, s.tap); err != nil {
	607	+	errs = append(errs, err)
	608	+	} else {
	609	+	s.attached = false
	610	+	}
459	611		}
460		-	if err := deleteTap(ctx, d.opt.Run, s.tap); err != nil {
461		-	errs = append(errs, err)
462		-	} else {
463		-	s.attached = false
	612	+	if s.lay.dir != "" {
	613	+	if err := removeAll(s.lay.dir); err != nil {
	614	+	errs = append(errs, err)
	615	+	}
464	616		}
465		-	}
466		-	if s.lay.dir != "" {
467		-	if err := removeAll(s.lay.dir); err != nil {
468		-	errs = append(errs, err)
	617	+	if s.lay.jailed {
	618	+	if err := removeCgroup(d.opt.CgroupRoot, filepath.Base(d.fcPath), s.id); err != nil {
	619	+	errs = append(errs, err)
	620	+	}
469	621		}
470		-	}
471		-	if len(errs) == 0 {
472		-	if err := d.store.remove(s.rec.ID); err != nil {
473		-	errs = append(errs, err)
	622	+	if len(errs) == 0 {
	623	+	if err := d.store.remove(s.id); err != nil {
	624	+	errs = append(errs, err)
	625	+	}
474	626		}
475	627		}
476	628		if len(errs) == 0 {
477	629		s.releaseLocked()
478	630		return nil
479	631		}
480		-	s.stopped = true // do not retry from here; Reap at the next start will
481		-	err := errors.Join(errs...)
482		-	log.Error().Err(err).Str("vm", s.rec.ID).Msg("micro-VM cleanup incomplete; the next runner start will retry")
483		-	return err
	632	+	s.stopped = true
	633	+	d.markLeaked(s.slot)
	634	+	s.leakErr = fmt.Errorf("%w: vm %s: %w", sandbox.ErrLeaked, s.id, errors.Join(errs...))
	635	+	log.Error().Err(s.leakErr).Str("vm", s.id).Msg("micro-VM cleanup incomplete; call Reap to finish it")
	636	+	return s.leakErr
484	637		}
485	638
486	639		// removeAll is os.RemoveAll that also reports a directory that is still there.

@@ -493,3 +646,14 @@ func removeAll(dir string) error {
493	646		}
494	647		return nil
495	648		}
	649	+
	650	+	// removeCgroup removes the cgroup the jailer created for a VM
	651	+	// (<root>/<parent>/<id>, parent = the exec file name). The jailer never removes
	652	+	// it. A missing cgroup is fine; a non-empty one fails (rmdir semantics).
	653	+	func removeCgroup(root, parent, id string) error {
	654	+	err := os.Remove(filepath.Join(root, parent, id))
	655	+	if err == nil \|\| errors.Is(err, os.ErrNotExist) {
	656	+	return nil
	657	+	}
	658	+	return fmt.Errorf("remove cgroup %s: %w", filepath.Join(root, parent, id), err)
	659	+	}


@@ -20,12 +20,12 @@ import (
20		"os"	20		"os"
21		"os/exec"	21		"os/exec"
22		"path/filepath"	22		"path/filepath"
			23	+	"strconv"
23		"sync"	24		"sync"
24		"syscall"	25		"syscall"
25		"time"	26		"time"
26			27
27		"github.com/bnema/zerowrap"	28		"github.com/bnema/zerowrap"
28	-	"golang.org/x/sys/unix"
29			29
30		"bnema.dev/ermite-runner/internal/cmdrun"	30		"bnema.dev/ermite-runner/internal/cmdrun"
31		"bnema.dev/ermite-runner/internal/netpolicy"	31		"bnema.dev/ermite-runner/internal/netpolicy"

@@ -36,8 +36,11 @@ import (
36		// Options are the testable knobs of the driver.	36		// Options are the testable knobs of the driver.
37		type Options struct {	37		type Options struct {
38		Run cmdrun.Runner	38		Run cmdrun.Runner
39	-	// ProcDir is the /proc used to find orphan Firecracker processes.	39	+	// ProcDir is the /proc used to find orphan VMM processes.
40		ProcDir string	40		ProcDir string
			41	+	// CgroupRoot is the cgroup v2 mount, where the jailer creates one cgroup per
			42	+	// VM (default /sys/fs/cgroup).
			43	+	CgroupRoot string
41		// BootTimeout bounds boot + init + executor connection (default 90 s).	44		// BootTimeout bounds boot + init + executor connection (default 90 s).
42		BootTimeout time.Duration	45		BootTimeout time.Duration
43		// Net overrides the network plumbing (tests).	46		// Net overrides the network plumbing (tests).

@@ -45,8 +48,17 @@ type Options struct {
45		// FirecrackerArgs are extra arguments for Firecracker (tests use them to	48		// FirecrackerArgs are extra arguments for Firecracker (tests use them to
46		// select the fake).	49		// select the fake).
47		FirecrackerArgs []string	50		FirecrackerArgs []string
			51	+	// FreeSpace overrides statfs for the free-space warning (tests).
			52	+	FreeSpace func(string) (int64, error)
			53	+	// kill overrides killAndWait (tests simulate a VMM that cannot be killed).
			54	+	kill func(pid int, timeout time.Duration) bool
48		}	55		}
49			56
			57	+	// vmmOverheadMiB is what the Firecracker process itself may use on top of the
			58	+	// guest memory (device emulation, the API server, the seccomp filter, stacks):
			59	+	// it is added to memory.max so that the guest, not the VMM, is what gets limited.
			60	+	const vmmOverheadMiB = 256
			61	+
50		// Driver implements sandbox.Driver for micro-VMs.	62		// Driver implements sandbox.Driver for micro-VMs.
51		type Driver struct {	63		type Driver struct {
52		cfg sandbox.Config	64		cfg sandbox.Config

@@ -56,14 +68,15 @@ type Driver struct {
56		execDisk string	68		execDisk string
57		store store	69		store store
58		net netsetup.Plumbing	70		net netsetup.Plumbing
59	-	lock *os.File	71	+	lock *sandbox.HostLock
60			72
61		mu sync.Mutex	73		mu sync.Mutex
62		slots *sandbox.Slots	74		slots *sandbox.Slots
63		}	75		}
64			76
65	-	// New builds the driver: resolves binaries, builds the exec disk, takes the	77	+	// New builds the driver: takes the host-wide lock, resolves binaries, builds the
66	-	// state-directory lock and enables IPv4 forwarding. Run Reap before New sandboxes.	78	+	// exec disk and enables IPv4 forwarding (a host-wide sysctl that stays on: see
			79	+	// the README). Run Reap before New sandboxes.
67		func New(ctx context.Context, cfg sandbox.Config, opt Options) (*Driver, error) {	80		func New(ctx context.Context, cfg sandbox.Config, opt Options) (*Driver, error) {
68		if opt.Run == nil {	81		if opt.Run == nil {
69		opt.Run = cmdrun.Exec{}	82		opt.Run = cmdrun.Exec{}

@@ -71,9 +84,15 @@ func New(ctx context.Context, cfg sandbox.Config, opt Options) (*Driver, error)
71		if opt.ProcDir == "" {	84		if opt.ProcDir == "" {
72		opt.ProcDir = "/proc"	85		opt.ProcDir = "/proc"
73		}	86		}
			87	+	if opt.CgroupRoot == "" {
			88	+	opt.CgroupRoot = "/sys/fs/cgroup"
			89	+	}
74		if opt.BootTimeout == 0 {	90		if opt.BootTimeout == 0 {
75		opt.BootTimeout = 90 * time.Second	91		opt.BootTimeout = 90 * time.Second
76		}	92		}
			93	+	if opt.kill == nil {
			94	+	opt.kill = killAndWait
			95	+	}
77		d := &Driver{cfg: cfg, opt: opt, slots: sandbox.NewSlots(cfg.Capacity), store: store{dir: filepath.Join(cfg.StateDir, "vms")}}	96		d := &Driver{cfg: cfg, opt: opt, slots: sandbox.NewSlots(cfg.Capacity), store: store{dir: filepath.Join(cfg.StateDir, "vms")}}
78			97
79		var err error	98		var err error

@@ -95,7 +114,7 @@ func New(ctx context.Context, cfg sandbox.Config, opt Options) (*Driver, error)
95		if err := os.MkdirAll(cfg.StateDir, 0o755); err != nil {	114		if err := os.MkdirAll(cfg.StateDir, 0o755); err != nil {
96		return nil, fmt.Errorf("state directory: %w", err)	115		return nil, fmt.Errorf("state directory: %w", err)
97		}	116		}
98	-	if err := d.acquireLock(); err != nil {	117	+	if d.lock, err = sandbox.AcquireHostLock(cfg.LockFile); err != nil {
99		return nil, err	118		return nil, err
100		}	119		}
101		fail := func(err error) (*Driver, error) { _ = d.Close(); return nil, err }	120		fail := func(err error) (*Driver, error) { _ = d.Close(); return nil, err }

@@ -107,6 +126,7 @@ func New(ctx context.Context, cfg sandbox.Config, opt Options) (*Driver, error)
107		}	126		}
108		}	127		}
109		}	128		}
			129	+	sandbox.WarnIfLowSpace(ctx, cfg.StateDir, cfg.Capacity, cfg.JobDiskMiB, opt.FreeSpace)
110		if d.execDisk, err = ensureExecDisk(ctx, opt.Run, exe, cfg.StateDir); err != nil {	130		if d.execDisk, err = ensureExecDisk(ctx, opt.Run, exe, cfg.StateDir); err != nil {
111		return fail(err)	131		return fail(err)
112		}	132		}

@@ -138,44 +158,33 @@ func resolveBin(configured, name string) (string, error) {
138		return abs, nil	158		return abs, nil
139		}	159		}
140			160
141	-	func (d *Driver) acquireLock() error {
142	-	f, err := os.OpenFile(filepath.Join(d.cfg.StateDir, "vm.lock"), os.O_CREATE\|os.O_RDWR, 0o600)
143	-	if err != nil {
144	-	return err
145	-	}
146	-	if err := unix.Flock(int(f.Fd()), unix.LOCK_EX\|unix.LOCK_NB); err != nil {
147	-	_ = f.Close()
148	-	return fmt.Errorf("another ermite-runner already uses state directory %s (reaping would destroy its VMs): %w", d.cfg.StateDir, err)
149	-	}
150	-	d.lock = f
151	-	return nil
152	-	}
153	-
154		// Mode implements sandbox.Driver.	161		// Mode implements sandbox.Driver.
155		func (d *Driver) Mode() sandbox.Mode { return sandbox.VM }	162		func (d *Driver) Mode() sandbox.Mode { return sandbox.VM }
156			163
157	-	// Close releases the lock and the DNS listeners.	164	+	// Isolation implements sandbox.Driver.
			165	+	func (d *Driver) Isolation() sandbox.Isolation { return sandbox.IsolationVM }
			166	+
			167	+	// Close releases the host lock and the DNS listeners.
158		func (d *Driver) Close() error {	168		func (d *Driver) Close() error {
159		if d.net != nil {	169		if d.net != nil {
160		d.net.Close()	170		d.net.Close()
161		}	171		}
162	-	if d.lock != nil {	172	+	return d.lock.Release()
163	-	err := d.lock.Close()
164	-	d.lock = nil
165	-	return err
166	-	}
167	-	return nil
168		}	173		}
169			174
170		// New implements sandbox.Driver.	175		// New implements sandbox.Driver.
171		func (d *Driver) New() (sandbox.Sandbox, error) {	176		func (d *Driver) New() (sandbox.Sandbox, error) {
			177	+	id, err := sandbox.NewID()
			178	+	if err != nil {
			179	+	return nil, err
			180	+	}
172		d.mu.Lock()	181		d.mu.Lock()
173		defer d.mu.Unlock()	182		defer d.mu.Unlock()
174		slot, err := d.slots.Acquire()	183		slot, err := d.slots.Acquire()
175		if err != nil {	184		if err != nil {
176		return nil, err	185		return nil, err
177		}	186		}
178	-	return &vmSandbox{d: d, slot: slot}, nil	187	+	return &vmSandbox{d: d, slot: slot, id: id}, nil
179		}	188		}
180			189
181		func (d *Driver) release(slot int) {	190		func (d *Driver) release(slot int) {

@@ -184,14 +193,26 @@ func (d *Driver) release(slot int) {
184		d.mu.Unlock()	193		d.mu.Unlock()
185		}	194		}
186			195
			196	+	func (d *Driver) markLeaked(slot int) {
			197	+	d.mu.Lock()
			198	+	d.slots.MarkLeaked(slot)
			199	+	d.mu.Unlock()
			200	+	}
			201	+
187		// vmSandbox is one VM, single use.	202		// vmSandbox is one VM, single use.
188		type vmSandbox struct {	203		type vmSandbox struct {
189		d *Driver	204		d *Driver
190		slot int	205		slot int
			206	+	id string
			207	+	gate sandbox.BootGate
			208	+
			209	+	mu sync.Mutex
			210	+	started bool
			211	+	stopped bool
			212	+	// leakErr is the error of a cleanup that left something behind; later Stop
			213	+	// calls return it again.
			214	+	leakErr error
191			215
192	-	mu sync.Mutex
193	-	started bool
194	-	stopped bool
195		rec record	216		rec record
196		lay layout	217		lay layout
197		cmd *exec.Cmd	218		cmd *exec.Cmd

@@ -204,10 +225,20 @@ type vmSandbox struct {
204		attached bool	225		attached bool
205		}	226		}
206			227
			228	+	// ID implements sandbox.Sandbox.
			229	+	func (s *vmSandbox) ID() string { return s.id }
			230	+
207		func (s *vmSandbox) uid() int { return s.d.cfg.UIDBase + s.slot }	231		func (s *vmSandbox) uid() int { return s.d.cfg.UIDBase + s.slot }
208			232
209		// Start implements sandbox.Sandbox.	233		// Start implements sandbox.Sandbox.
210		func (s *vmSandbox) Start(ctx context.Context, spec sandbox.JobSpec) (sandbox.Conn, error) {	234		func (s *vmSandbox) Start(ctx context.Context, spec sandbox.JobSpec) (sandbox.Conn, error) {
			235	+	// A Stop from another goroutine aborts the boot through the gate.
			236	+	bootCtx, err := s.gate.Begin(ctx)
			237	+	if err != nil {
			238	+	return nil, fmt.Errorf("vm sandbox: %w", err)
			239	+	}
			240	+	defer s.gate.End()
			241	+
211		s.mu.Lock()	242		s.mu.Lock()
212		defer s.mu.Unlock()	243		defer s.mu.Unlock()
213		if s.started \|\| s.stopped {	244		if s.started \|\| s.stopped {

@@ -218,18 +249,13 @@ func (s *vmSandbox) Start(ctx context.Context, spec sandbox.JobSpec) (sandbox.Co
218		d := s.d	249		d := s.d
219		spec = d.cfg.Resolve(spec)	250		spec = d.cfg.Resolve(spec)
220			251
221	-	id, err := sandbox.NewID()
222	-	if err != nil {
223	-	s.releaseLocked()
224	-	return nil, err
225	-	}
226		jailed := !d.cfg.JailerOff	252		jailed := !d.cfg.JailerOff
227		runUID := s.uid()	253		runUID := s.uid()
228		if !jailed {	254		if !jailed {
229		runUID = os.Getuid()	255		runUID = os.Getuid()
230		}	256		}
231	-	s.lay = newLayout(d.cfg.StateDir, filepath.Base(d.fcPath), id, jailed, runUID)	257	+	s.lay = newLayout(d.cfg.StateDir, filepath.Base(d.fcPath), s.id, jailed, runUID)
232	-	s.rec = record{ID: id, Slot: s.slot, Job: spec.JobID, Dir: s.lay.dir}	258	+	s.rec = record{ID: s.id, Slot: s.slot, Job: spec.JobID, Dir: s.lay.dir}
233		// The record exists before the first resource, so any crash from here on is	259		// The record exists before the first resource, so any crash from here on is
234		// reapable.	260		// reapable.
235		if err := d.store.save(s.rec); err != nil {	261		if err := d.store.save(s.rec); err != nil {

@@ -237,7 +263,7 @@ func (s *vmSandbox) Start(ctx context.Context, spec sandbox.JobSpec) (sandbox.Co
237		return nil, fmt.Errorf("write state record: %w", err)	263		return nil, fmt.Errorf("write state record: %w", err)
238		}	264		}
239			265
240	-	conn, err := s.boot(ctx, spec, id)	266	+	conn, err := s.boot(bootCtx, spec)
241		if err != nil {	267		if err != nil {
242		console := ""	268		console := ""
243		if s.console != nil {	269		if s.console != nil {

@@ -249,98 +275,140 @@ func (s *vmSandbox) Start(ctx context.Context, spec sandbox.JobSpec) (sandbox.Co
249		}	275		}
250		return nil, errors.Join(err, cerr)	276		return nil, errors.Join(err, cerr)
251		}	277		}
252	-	log.Info().Str("vm", id).Int("slot", s.slot).Str("job", spec.JobID).Msg("micro-VM running")	278	+	log.Info().Str("vm", s.id).Int("slot", s.slot).Str("job", spec.JobID).Msg("micro-VM running")
253		return conn, nil	279		return conn, nil
254		}	280		}
255			281
256	-	func (s *vmSandbox) boot(ctx context.Context, spec sandbox.JobSpec, id string) (sandbox.Conn, error) {	282	+	// boot runs the stages in order; each stage records what it created so that
257	-	d := s.d	283	+	// cleanupLocked can undo exactly that.
258	-	netw, err := netpolicy.Subnet(d.cfg.Network, s.slot)	284	+	func (s *vmSandbox) boot(ctx context.Context, spec sandbox.JobSpec) (sandbox.Conn, error) {
			285	+	netw, err := netpolicy.Subnet(s.d.cfg.Network, s.slot)
			286	+	if err != nil {
			287	+	return nil, err
			288	+	}
			289	+	if err := s.bootNetwork(ctx, netw); err != nil {
			290	+	return nil, err
			291	+	}
			292	+	api, err := s.bootVMM(ctx, spec)
			293	+	if err != nil {
			294	+	return nil, err
			295	+	}
			296	+	files, err := s.bootFiles(ctx, spec)
			297	+	if err != nil {
			298	+	return nil, err
			299	+	}
			300	+	accepted, err := s.bootVsock()
259		if err != nil {	301		if err != nil {
260		return nil, err	302		return nil, err
261		}	303		}
			304	+	if err := api.configure(ctx, s.vmConfig(spec, netw, files)); err != nil {
			305	+	return nil, err
			306	+	}
			307	+	return s.waitExecutor(ctx, accepted)
			308	+	}
262			309
263	-	// Network first: the TAP must exist when Firecracker opens it.	310	+	// bootNetwork creates the TAP (it must exist when Firecracker opens it) and
			311	+	// installs the firewall policy and the DNS listener.
			312	+	func (s *vmSandbox) bootNetwork(ctx context.Context, netw netpolicy.Net) error {
			313	+	d := s.d
264		s.tap, s.gateway = tapName(s.slot), netw.Gateway	314		s.tap, s.gateway = tapName(s.slot), netw.Gateway
265		s.rec.Tap, s.rec.Gate = s.tap, s.gateway.String()	315		s.rec.Tap, s.rec.Gate = s.tap, s.gateway.String()
266		if err := d.store.save(s.rec); err != nil {	316		if err := d.store.save(s.rec); err != nil {
267	-	return nil, err	317	+	return err
268		}	318		}
269		if err := createTap(ctx, d.opt.Run, s.tap, s.lay.uid, s.gateway); err != nil {	319		if err := createTap(ctx, d.opt.Run, s.tap, s.lay.uid, s.gateway); err != nil {
270	-	return nil, err	320	+	return err
271		}	321		}
272		s.attached = true // from here Detach is idempotent and always safe	322		s.attached = true // from here Detach is idempotent and always safe
273	-	if err := d.net.Attach(ctx, netpolicy.Guest{Iface: s.tap, Gateway: netw.Gateway, Addr: netw.Guest}); err != nil {	323	+	return d.net.Attach(ctx, netpolicy.Guest{Iface: s.tap, Gateway: netw.Gateway, Addr: netw.Guest})
274	-	return nil, err	324	+	}
275	-	}
276			325
277	-	// The VMM process.	326	+	// bootVMM starts the jailer (or Firecracker) and waits for its API.
			327	+	func (s vmSandbox) bootVMM(ctx context.Context, spec sandbox.JobSpec) (apiClient, error) {
278		apiSock := s.lay.host("api.sock")	328		apiSock := s.lay.host("api.sock")
279		if s.lay.jailed {	329		if s.lay.jailed {
280		apiSock = s.lay.host("run/firecracker.socket")	330		apiSock = s.lay.host("run/firecracker.socket")
281		} else if err := os.MkdirAll(s.lay.root, 0o700); err != nil {	331		} else if err := os.MkdirAll(s.lay.root, 0o700); err != nil {
282		return nil, err	332		return nil, err
283		}	333		}
284	-	if err := s.startVMM(id, apiSock); err != nil {	334	+	if err := s.startVMM(spec, apiSock); err != nil {
285		return nil, err	335		return nil, err
286		}	336		}
287		api := newAPIClient(apiSock)	337		api := newAPIClient(apiSock)
288		if err := s.waitAPI(ctx, api, apiSock); err != nil {	338		if err := s.waitAPI(ctx, api, apiSock); err != nil {
289		return nil, err	339		return nil, err
290		}	340		}
			341	+	return api, nil
			342	+	}
291			343
292	-	// Files: the process only opens them when configured below.	344	+	// vmFiles are the paths Firecracker is given.
293	-	execDisk, err := s.lay.share(d.execDisk, "exec.ext4")	345	+	type vmFiles struct{ execDisk, baseDisk, kernel string }
294	-	if err != nil {	346	+
295	-	return nil, err	347	+	// bootFiles makes the shared files available and creates the scratch disk.
296	-	}	348	+	func (s *vmSandbox) bootFiles(ctx context.Context, spec sandbox.JobSpec) (vmFiles, error) {
297	-	baseDisk, err := s.lay.share(d.cfg.BaseImagePath(), "base.ext4")	349	+	d := s.d
298	-	if err != nil {	350	+	var f vmFiles
299	-	return nil, err	351	+	var err error
			352	+	if f.execDisk, err = s.lay.share(d.execDisk, "exec.ext4"); err != nil {
			353	+	return f, err
300		}	354		}
301	-	kernel, err := s.lay.share(d.cfg.KernelPath(), "kernel")	355	+	if f.baseDisk, err = s.lay.share(d.cfg.BaseImagePath(), "base.ext4"); err != nil {
302	-	if err != nil {	356	+	return f, err
303	-	return nil, err
304		}	357		}
305	-	if err := makeScratch(ctx, d.opt.Run, s.lay.host("scratch.ext4"), spec.DiskMiB); err != nil {	358	+	if f.kernel, err = s.lay.share(d.cfg.KernelPath(), "kernel"); err != nil {
306	-	return nil, err	359	+	return f, err
307		}	360		}
308	-	if err := s.lay.own("scratch.ext4"); err != nil {	361	+	owner := -1
309	-	return nil, err	362	+	if s.lay.jailed {
			363	+	owner = s.lay.uid
310		}	364		}
			365	+	return f, makeScratch(ctx, d.opt.Run, s.lay.host("scratch.ext4"), spec.DiskMiB, owner)
			366	+	}
311			367
312	-	// The host side of the vsock stream: Firecracker connects to	368	+	// bootVsock listens on the host side of the vsock stream: Firecracker connects
313	-	// <uds_path>_<port> when the guest connects to the host CID.	369	+	// to <uds_path>_<port> when the guest connects to the host CID.
314	-	udsName := "v.sock"	370	+	func (s *vmSandbox) bootVsock() (<-chan acceptResult, error) {
315	-	ln, err := net.Listen("unix", s.lay.host(udsName)+fmt.Sprintf("_%d", sandbox.VsockPort))	371	+	path := s.lay.host(udsName) + fmt.Sprintf("_%d", sandbox.VsockPort)
			372	+	ln, err := net.Listen("unix", path)
316		if err != nil {	373		if err != nil {
317		return nil, fmt.Errorf("listen vsock socket: %w", err)	374		return nil, fmt.Errorf("listen vsock socket: %w", err)
318		}	375		}
319		s.listener = ln	376		s.listener = ln
320		if s.lay.jailed {	377		if s.lay.jailed {
321	-	if err := os.Chown(s.lay.host(udsName+fmt.Sprintf("_%d", sandbox.VsockPort)), s.lay.uid, s.lay.uid); err != nil {	378	+	if err := os.Lchown(path, s.lay.uid, s.lay.uid); err != nil {
322		return nil, err	379		return nil, err
323		}	380		}
324		}	381		}
325		accepted := make(chan acceptResult, 1)	382		accepted := make(chan acceptResult, 1)
326		go func() { c, err := ln.Accept(); accepted <- acceptResult{c, err} }()	383		go func() { c, err := ln.Accept(); accepted <- acceptResult{c, err} }()
			384	+	return accepted, nil
			385	+	}
			386	+
			387	+	const udsName = "v.sock"
327			388
328	-	err = api.configure(ctx, vmConfig{	389	+	func (s *vmSandbox) vmConfig(spec sandbox.JobSpec, netw netpolicy.Net, f vmFiles) vmConfig {
			390	+	cfg := s.d.cfg
			391	+	return vmConfig{
329		Machine: machineConfig{VcpuCount: spec.CPUs, MemSizeMiB: spec.MemoryMiB},	392		Machine: machineConfig{VcpuCount: spec.CPUs, MemSizeMiB: spec.MemoryMiB},
330	-	Boot: bootSource{KernelImagePath: kernel, BootArgs: bootArgs(netw.Guest, netw.Gateway, time.Now().Unix())},	393	+	Boot: bootSource{KernelImagePath: f.kernel, BootArgs: bootArgs(netw.Guest, netw.Gateway, time.Now().Unix())},
331		Drives: []drive{	394		Drives: []drive{
332	-	{DriveID: "rootfs", PathOnHost: execDisk, IsRootDevice: true, IsReadOnly: true},	395	+	{DriveID: "rootfs", PathOnHost: f.execDisk, IsRootDevice: true, IsReadOnly: true},
333	-	{DriveID: "base", PathOnHost: baseDisk, IsReadOnly: true},	396	+	{DriveID: "base", PathOnHost: f.baseDisk, IsReadOnly: true},
334	-	{DriveID: "scratch", PathOnHost: s.lay.fc("scratch.ext4")},	397	+	{DriveID: "scratch", PathOnHost: s.lay.fc("scratch.ext4"), RateLimiter: diskLimiter(cfg.DiskBandwidthMiB, cfg.DiskIOPS)},
			398	+	},
			399	+	Net: networkInterface{
			400	+	IfaceID: "eth0", HostDevName: s.tap, GuestMAC: macFor(s.slot),
			401	+	RxRateLimiter: netLimiter(cfg.NetMbit), TxRateLimiter: netLimiter(cfg.NetMbit),
335		},	402		},
336	-	Net: networkInterface{IfaceID: "eth0", HostDevName: s.tap, GuestMAC: macFor(s.slot)},
337		Vsock: vsockDevice{GuestCID: 3, UDSPath: s.lay.fc(udsName)},	403		Vsock: vsockDevice{GuestCID: 3, UDSPath: s.lay.fc(udsName)},
338	-	})
339	-	if err != nil {
340	-	return nil, err
341		}	404		}
			405	+	}
342			406
343	-	timer := time.NewTimer(d.opt.BootTimeout)	407	+	// waitExecutor waits for the guest's executor to connect. Only ctx (the caller's
			408	+	// context, or Stop through the boot gate) and the boot timeout end the wait: a
			409	+	// boot cannot be aborted any other way.
			410	+	func (s *vmSandbox) waitExecutor(ctx context.Context, accepted <-chan acceptResult) (sandbox.Conn, error) {
			411	+	timer := time.NewTimer(s.d.opt.BootTimeout)
344		defer timer.Stop()	412		defer timer.Stop()
345		select {	413		select {
346		case r := <-accepted:	414		case r := <-accepted:

@@ -352,9 +420,9 @@ func (s *vmSandbox) boot(ctx context.Context, spec sandbox.JobSpec, id string) (
352		case <-s.exited:	420		case <-s.exited:
353		return nil, errors.New("firecracker exited before the executor connected")	421		return nil, errors.New("firecracker exited before the executor connected")
354		case <-timer.C:	422		case <-timer.C:
355	-	return nil, fmt.Errorf("executor did not connect within %s", d.opt.BootTimeout)	423	+	return nil, fmt.Errorf("executor did not connect within %s", s.d.opt.BootTimeout)
356		case <-ctx.Done():	424		case <-ctx.Done():
357	-	return nil, ctx.Err()	425	+	return nil, context.Cause(ctx)
358		}	426		}
359		}	427		}
360			428

@@ -363,18 +431,38 @@ type acceptResult struct {
363		err error	431		err error
364		}	432		}
365			433
366	-	func (s *vmSandbox) startVMM(id, apiSock string) error {	434	+	// jailerArgs builds the jailer command line (after the binary name).
			435	+	//
			436	+	// - cgroup v2 with memory.max (guest memory + VMM overhead) and pids.max (a
			437	+	// VMM needs a handful of threads per vCPU): a runaway or compromised VMM
			438	+	// cannot take the host's memory or fork-bomb it;
			439	+	// - a new PID namespace: the VMM cannot see or signal host processes;
			440	+	// - no-file: the descriptor budget of the VMM.
			441	+	func jailerArgs(jailer, fc, id string, uid int, chrootBase string, spec sandbox.JobSpec, extra []string) []string {
			442	+	memBytes := (int64(spec.MemoryMiB) + vmmOverheadMiB) << 20
			443	+	pids := 32 + 8*spec.CPUs
			444	+	args := []string{
			445	+	"--id", id, "--exec-file", fc,
			446	+	"--uid", strconv.Itoa(uid), "--gid", strconv.Itoa(uid),
			447	+	"--chroot-base-dir", chrootBase,
			448	+	"--cgroup-version", "2",
			449	+	"--cgroup", "memory.max=" + strconv.FormatInt(memBytes, 10),
			450	+	"--cgroup", "pids.max=" + strconv.Itoa(pids),
			451	+	"--new-pid-ns",
			452	+	"--resource-limit", "no-file=4096",
			453	+	"--",
			454	+	}
			455	+	return append(args, extra...)
			456	+	}
			457	+
			458	+	func (s *vmSandbox) startVMM(spec sandbox.JobSpec, apiSock string) error {
367		d := s.d	459		d := s.d
368		var cmd *exec.Cmd	460		var cmd *exec.Cmd
369		if s.lay.jailed {	461		if s.lay.jailed {
370	-	args := []string{"--id", id, "--exec-file", d.fcPath,	462	+	args := jailerArgs(d.jailer, d.fcPath, s.id, s.uid(), jailBase(d.cfg.StateDir), spec, d.opt.FirecrackerArgs)
371	-	"--uid", fmt.Sprint(s.uid()), "--gid", fmt.Sprint(s.uid()),	463	+	cmd = exec.Command(d.jailer, args...)
372	-	"--chroot-base-dir", jailBase(d.cfg.StateDir),
373	-	"--resource-limit", "no-file=4096",
374	-	"--"}
375	-	cmd = exec.Command(d.jailer, append(args, d.opt.FirecrackerArgs...)...)
376		} else {	464		} else {
377	-	args := append([]string{"--id", id, "--api-sock", apiSock}, d.opt.FirecrackerArgs...)	465	+	args := append([]string{"--id", s.id, "--api-sock", apiSock}, d.opt.FirecrackerArgs...)
378		cmd = exec.Command(d.fcPath, args...)	466		cmd = exec.Command(d.fcPath, args...)
379		}	467		}
380		s.console = newRing(16 << 10)	468		s.console = newRing(16 << 10)

@@ -385,11 +473,36 @@ func (s *vmSandbox) startVMM(id, apiSock string) error {
385		}	473		}
386		s.cmd = cmd	474		s.cmd = cmd
387		s.exited = make(chan struct{})	475		s.exited = make(chan struct{})
388	-	go func() { _ = cmd.Wait(); close(s.exited) }()	476	+	jailed := s.lay.jailed
			477	+	go func() {
			478	+	_ = cmd.Wait()
			479	+	// With --new-pid-ns the jailer may exit as soon as it has started the
			480	+	// VMM in its namespace: the VMM is then gone only when its own pid is.
			481	+	if jailed {
			482	+	s.awaitVMMGone()
			483	+	}
			484	+	close(s.exited)
			485	+	}()
389		s.rec.PID = cmd.Process.Pid	486		s.rec.PID = cmd.Process.Pid
390		return s.d.store.save(s.rec)	487		return s.d.store.save(s.rec)
391		}	488		}
392			489
			490	+	// awaitVMMGone blocks until no process of this VM's id is left.
			491	+	func (s *vmSandbox) awaitVMMGone() {
			492	+	for {
			493	+	alive := false
			494	+	for pid, id := range vmmProcs(s.d.opt.ProcDir) {
			495	+	if id == s.id && pidAlive(pid) {
			496	+	alive = true
			497	+	}
			498	+	}
			499	+	if !alive {
			500	+	return
			501	+	}
			502	+	time.Sleep(50 * time.Millisecond)
			503	+	}
			504	+	}
			505	+
393		func (s vmSandbox) waitAPI(ctx context.Context, api apiClient, sock string) error {	506		func (s vmSandbox) waitAPI(ctx context.Context, api apiClient, sock string) error {
394		deadline := time.Now().Add(10 * time.Second)	507		deadline := time.Now().Add(10 * time.Second)
395		for {	508		for {

@@ -400,7 +513,7 @@ func (s vmSandbox) waitAPI(ctx context.Context, api apiClient, sock string) er
400		case <-s.exited:	513		case <-s.exited:
401		return errors.New("firecracker exited before its API was ready")	514		return errors.New("firecracker exited before its API was ready")
402		case <-ctx.Done():	515		case <-ctx.Done():
403	-	return ctx.Err()	516	+	return context.Cause(ctx)
404		case <-time.After(25 * time.Millisecond):	517		case <-time.After(25 * time.Millisecond):
405		}	518		}
406		if time.Now().After(deadline) {	519		if time.Now().After(deadline) {

@@ -411,13 +524,14 @@ func (s vmSandbox) waitAPI(ctx context.Context, api apiClient, sock string) er
411			524
412		// Stop implements sandbox.Sandbox.	525		// Stop implements sandbox.Sandbox.
413		func (s *vmSandbox) Stop(ctx context.Context) error {	526		func (s *vmSandbox) Stop(ctx context.Context) error {
			527	+	// Abort a boot in progress first: Start holds s.mu for its whole duration.
			528	+	s.gate.Abort()
414		s.mu.Lock()	529		s.mu.Lock()
415		defer s.mu.Unlock()	530		defer s.mu.Unlock()
416		if s.stopped {	531		if s.stopped {
417	-	return nil	532	+	return s.leakErr
418		}	533		}
419		if !s.started {	534		if !s.started {
420	-	s.stopped = true
421		s.releaseLocked()	535		s.releaseLocked()
422		return nil	536		return nil
423		}	537		}

@@ -429,10 +543,31 @@ func (s *vmSandbox) releaseLocked() {
429		s.d.release(s.slot)	543		s.d.release(s.slot)
430		}	544		}
431			545
432	-	// cleanupLocked removes everything the VM created, in dependency order, and	546	+	// vmmPIDs returns every host process of this VM: the launched command and
433	-	// keeps going after an error. The state record and the slot are released only	547	+	// whatever the /proc scan finds with this id.
434	-	// when everything is gone, so that a half-failed cleanup is retried by Reap	548	+	func (s *vmSandbox) vmmPIDs() []int {
435	-	// rather than forgotten and the slot (hence TAP name) is not reused meanwhile.	549	+	set := map[int]bool{}
			550	+	if s.cmd != nil && s.cmd.Process != nil {
			551	+	set[s.cmd.Process.Pid] = true
			552	+	}
			553	+	for pid, id := range vmmProcs(s.d.opt.ProcDir) {
			554	+	if id == s.id {
			555	+	set[pid] = true
			556	+	}
			557	+	}
			558	+	var out []int
			559	+	for pid := range set {
			560	+	out = append(out, pid)
			561	+	}
			562	+	return out
			563	+	}
			564	+
			565	+	// cleanupLocked removes everything the VM created, in dependency order. Steps
			566	+	// that need the VMM to be dead (network device, firewall rules, files, cgroup)
			567	+	// are skipped when it could not be killed: removing the TAP or the policy under
			568	+	// a VM that may still run would break the isolation it relies on, so the
			569	+	// sandbox is reported as leaked (sandbox.ErrLeaked), its slot stays taken, and
			570	+	// Reap finishes the job.
436		func (s *vmSandbox) cleanupLocked(ctx context.Context) error {	571		func (s *vmSandbox) cleanupLocked(ctx context.Context) error {
437		log := zerowrap.FromCtx(ctx)	572		log := zerowrap.FromCtx(ctx)
438		d := s.d	573		d := s.d

@@ -445,42 +580,60 @@ func (s *vmSandbox) cleanupLocked(ctx context.Context) error {
445		_ = s.listener.Close()	580		_ = s.listener.Close()
446		s.listener = nil	581		s.listener = nil
447		}	582		}
			583	+	dead := true
448		if s.cmd != nil && s.cmd.Process != nil {	584		if s.cmd != nil && s.cmd.Process != nil {
449	-	killAndWait(s.cmd.Process.Pid, 5*time.Second)	585	+	_ = syscall.Kill(-s.cmd.Process.Pid, syscall.SIGKILL) // the whole group
			586	+	}
			587	+	for _, pid := range s.vmmPIDs() {
			588	+	if !d.opt.kill(pid, 5*time.Second) {
			589	+	dead = false
			590	+	errs = append(errs, fmt.Errorf("vmm process %d did not exit", pid))
			591	+	}
			592	+	}
			593	+	if dead && s.exited != nil {
450		select {	594		select {
451		case <-s.exited:	595		case <-s.exited:
452		case <-time.After(5 * time.Second):	596		case <-time.After(5 * time.Second):
453	-	errs = append(errs, fmt.Errorf("firecracker pid %d did not exit", s.cmd.Process.Pid))	597	+	dead = false
			598	+	errs = append(errs, errors.New("vmm exit was not observed"))
454		}	599		}
455		}	600		}
456	-	if s.attached {	601	+	if dead {
457	-	if err := d.net.Detach(ctx, s.tap, s.gateway); err != nil {	602	+	if s.attached {
458	-	errs = append(errs, err)	603	+	if err := d.net.Detach(ctx, s.tap, s.gateway); err != nil {
			604	+	errs = append(errs, err)
			605	+	}
			606	+	if err := deleteTap(ctx, d.opt.Run, s.tap); err != nil {
			607	+	errs = append(errs, err)
			608	+	} else {
			609	+	s.attached = false
			610	+	}
459		}	611		}
460	-	if err := deleteTap(ctx, d.opt.Run, s.tap); err != nil {	612	+	if s.lay.dir != "" {
461	-	errs = append(errs, err)	613	+	if err := removeAll(s.lay.dir); err != nil {
462	-	} else {	614	+	errs = append(errs, err)
463	-	s.attached = false	615	+	}
464		}	616		}
465	-	}	617	+	if s.lay.jailed {
466	-	if s.lay.dir != "" {	618	+	if err := removeCgroup(d.opt.CgroupRoot, filepath.Base(d.fcPath), s.id); err != nil {
467	-	if err := removeAll(s.lay.dir); err != nil {	619	+	errs = append(errs, err)
468	-	errs = append(errs, err)	620	+	}
469		}	621		}
470	-	}	622	+	if len(errs) == 0 {
471	-	if len(errs) == 0 {	623	+	if err := d.store.remove(s.id); err != nil {
472	-	if err := d.store.remove(s.rec.ID); err != nil {	624	+	errs = append(errs, err)
473	-	errs = append(errs, err)	625	+	}
474		}	626		}
475		}	627		}
476		if len(errs) == 0 {	628		if len(errs) == 0 {
477		s.releaseLocked()	629		s.releaseLocked()
478		return nil	630		return nil
479		}	631		}
480	-	s.stopped = true // do not retry from here; Reap at the next start will	632	+	s.stopped = true
481	-	err := errors.Join(errs...)	633	+	d.markLeaked(s.slot)
482	-	log.Error().Err(err).Str("vm", s.rec.ID).Msg("micro-VM cleanup incomplete; the next runner start will retry")	634	+	s.leakErr = fmt.Errorf("%w: vm %s: %w", sandbox.ErrLeaked, s.id, errors.Join(errs...))
483	-	return err	635	+	log.Error().Err(s.leakErr).Str("vm", s.id).Msg("micro-VM cleanup incomplete; call Reap to finish it")
			636	+	return s.leakErr
484		}	637		}
485			638
486		// removeAll is os.RemoveAll that also reports a directory that is still there.	639		// removeAll is os.RemoveAll that also reports a directory that is still there.

@@ -493,3 +646,14 @@ func removeAll(dir string) error {
493		}	646		}
494		return nil	647		return nil
495		}	648		}
			649	+
			650	+	// removeCgroup removes the cgroup the jailer created for a VM
			651	+	// (<root>/<parent>/<id>, parent = the exec file name). The jailer never removes
			652	+	// it. A missing cgroup is fine; a non-empty one fails (rmdir semantics).
			653	+	func removeCgroup(root, parent, id string) error {
			654	+	err := os.Remove(filepath.Join(root, parent, id))
			655	+	if err == nil \|\| errors.Is(err, os.ErrNotExist) {
			656	+	return nil
			657	+	}
			658	+	return fmt.Errorf("remove cgroup %s: %w", filepath.Join(root, parent, id), err)
			659	+	}

M internal/sandbox/vm/images.go +11 -2


@@ -10,6 +10,7 @@ import (
10	10		"io"
11	11		"os"
12	12		"path/filepath"
	13	+	"syscall"
13	14
14	15		"bnema.dev/ermite-runner/internal/cmdrun"
15	16		"bnema.dev/ermite-runner/internal/sandbox"

@@ -83,11 +84,19 @@ func ensureExecDisk(ctx context.Context, run cmdrun.Runner, exe, stateDir string
83	84		}
84	85
85	86		// makeScratch creates the per-job sparse scratch disk of sizeMiB and formats it.
86		-	func makeScratch(ctx context.Context, run cmdrun.Runner, path string, sizeMiB int) error {
87		-	f, err := os.OpenFile(path, os.O_WRONLY\|os.O_CREATE\|os.O_EXCL, 0o600)
	87	+	// The file is created exclusively without following symlinks and handed to
	88	+	// owner (fchown on the open descriptor, never by path) when owner >= 0.
	89	+	func makeScratch(ctx context.Context, run cmdrun.Runner, path string, sizeMiB, owner int) error {
	90	+	f, err := os.OpenFile(path, os.O_WRONLY\|os.O_CREATE\|os.O_EXCL\|syscall.O_NOFOLLOW, 0o600)
88	91		if err != nil {
89	92		return err
90	93		}
	94	+	if owner >= 0 {
	95	+	if err := f.Chown(owner, owner); err != nil {
	96	+	_ = f.Close()
	97	+	return fmt.Errorf("chown scratch disk: %w", err)
	98	+	}
	99	+	}
91	100		if err := f.Truncate(int64(sizeMiB) << 20); err != nil {
92	101		_ = f.Close()
93	102		return err

 	"io"
 	"os"
 	"path/filepath"
+	"syscall"
 	"bnema.dev/ermite-runner/internal/cmdrun"
 	"bnema.dev/ermite-runner/internal/sandbox"
 }
 // makeScratch creates the per-job sparse scratch disk of sizeMiB and formats it.
-func makeScratch(ctx context.Context, run cmdrun.Runner, path string, sizeMiB int) error {
+// The file is created exclusively without following symlinks and handed to
-	f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
+// owner (fchown on the open descriptor, never by path) when owner >= 0.
+func makeScratch(ctx context.Context, run cmdrun.Runner, path string, sizeMiB, owner int) error {
+	f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL|syscall.O_NOFOLLOW, 0o600)
 	if err != nil {
 		return err
 	}
+	if owner >= 0 {
+		if err := f.Chown(owner, owner); err != nil {
+			_ = f.Close()
+			return fmt.Errorf("chown scratch disk: %w", err)
+		}
+	}
 	if err := f.Truncate(int64(sizeMiB) << 20); err != nil {
 		_ = f.Close()
 		return err

M internal/sandbox/vm/layout.go +3 -1


@@ -69,11 +69,13 @@ func (l layout) share(src, name string) (string, error) {
69	69		}
70	70
71	71		// own hands a per-job file in the jail to the jailed uid; no-op without jailer.
	72	+	// It never follows a symlink (Lchown): the jail is writable by the jailed uid
	73	+	// once Firecracker runs, so a link planted there must not redirect the chown.
72	74		func (l layout) own(name string) error {
73	75		if !l.jailed {
74	76		return nil
75	77		}
76		-	return os.Chown(l.host(name), l.uid, l.uid)
	78	+	return os.Lchown(l.host(name), l.uid, l.uid)
77	79		}
78	80
79	81		// checkReadable verifies that other users can read the shared file, which the

 }
 // own hands a per-job file in the jail to the jailed uid; no-op without jailer.
+// It never follows a symlink (Lchown): the jail is writable by the jailed uid
+// once Firecracker runs, so a link planted there must not redirect the chown.
 func (l layout) own(name string) error {
 	if !l.jailed {
 		return nil
 	}
-	return os.Chown(l.host(name), l.uid, l.uid)
+	return os.Lchown(l.host(name), l.uid, l.uid)
 }
 // checkReadable verifies that other users can read the shared file, which the

M internal/sandbox/vm/proc.go +64 -23


@@ -15,11 +15,15 @@ import (
15	15		"bnema.dev/ermite-runner/internal/sandbox"
16	16		)
17	17
18		-	// firecrackerProcs scans a /proc tree for Firecracker processes started by this
19		-	// runner: argv[0] is "firecracker" and "--id" is followed by one of our ids
20		-	// (see newID). Processes of other users of Firecracker are never matched. It
21		-	// finds orphans even when the state directory was lost.
22		-	func firecrackerProcs(procDir string) map[int]string {
	18	+	// vmmProcs scans a /proc tree for processes started by this runner for a VM:
	19	+	// the Firecracker process and the jailer. A process is ours when its command
	20	+	// line has "--id <id>" with an id of the form sandbox.NewID produces and also
	21	+	// "--api-sock" (Firecracker) or "--exec-file" (jailer). The program name is not
	22	+	// looked at: the binary may be renamed or versioned (firecracker-v1.17), and a
	23	+	// jailer in a PID namespace leaves a Firecracker whose argv[0] is whatever the
	24	+	// jailer chose. Processes of other users of Firecracker (other ids) are never
	25	+	// matched. It finds orphans even when the state directory was lost.
	26	+	func vmmProcs(procDir string) map[int]string {
23	27		out := map[int]string{}
24	28		ents, err := os.ReadDir(procDir)
25	29		if err != nil {

@@ -34,17 +38,57 @@ func firecrackerProcs(procDir string) map[int]string {
34	38		if err != nil \|\| len(b) == 0 {
35	39		continue
36	40		}
37		-	args := strings.Split(strings.TrimRight(string(b), "\x00"), "\x00")
38		-	if filepath.Base(args[0]) != "firecracker" {
39		-	continue
	41	+	if id := vmIDFromCmdline(strings.Split(strings.TrimRight(string(b), "\x00"), "\x00")); id != "" {
	42	+	out[pid] = id
40	43		}
41		-	for i := 1; i+1 < len(args); i++ {
42		-	if args[i] == "--id" && sandbox.ValidID(args[i+1]) {
43		-	out[pid] = args[i+1]
	44	+	}
	45	+	return out
	46	+	}
	47	+
	48	+	// vmIDFromCmdline returns the VM id of a command line, or "".
	49	+	func vmIDFromCmdline(args []string) string {
	50	+	var id string
	51	+	var marker bool
	52	+	for i := 1; i < len(args); i++ {
	53	+	switch args[i] {
	54	+	case "--id":
	55	+	if i+1 < len(args) && sandbox.ValidID(args[i+1]) {
	56	+	id = args[i+1]
44	57		}
	58	+	case "--api-sock", "--exec-file":
	59	+	marker = true
45	60		}
46	61		}
47		-	return out
	62	+	if marker {
	63	+	return id
	64	+	}
	65	+	return ""
	66	+	}
	67	+
	68	+	// readPidFile returns the pid the jailer wrote for a VM run in a PID namespace.
	69	+	func readPidFile(path string) int {
	70	+	b, err := os.ReadFile(path)
	71	+	if err != nil {
	72	+	return 0
	73	+	}
	74	+	pid, err := strconv.Atoi(strings.TrimSpace(string(b)))
	75	+	if err != nil \|\| pid <= 1 {
	76	+	return 0
	77	+	}
	78	+	return pid
	79	+	}
	80	+
	81	+	// pidAlive reports whether pid exists and is not a zombie.
	82	+	func pidAlive(pid int) bool {
	83	+	if err := syscall.Kill(pid, 0); err != nil {
	84	+	return false
	85	+	}
	86	+	if b, err := os.ReadFile("/proc/" + strconv.Itoa(pid) + "/stat"); err == nil {
	87	+	if i := bytes.LastIndexByte(b, ')'); i >= 0 && i+2 < len(b) && b[i+2] == 'Z' {
	88	+	return false
	89	+	}
	90	+	}
	91	+	return true
48	92		}
49	93
50	94		// ring keeps the last N bytes written to it: the VMM's stdout/stderr, which

@@ -74,20 +118,17 @@ func (r *ring) String() string {
74	118		return string(bytes.TrimSpace(r.buf))
75	119		}
76	120
77		-	// killAndWait sends SIGKILL to pid and waits until it is gone (zombies of
78		-	// children of this process are reaped by their own Wait).
79		-	func killAndWait(pid int, timeout time.Duration) {
	121	+	// killAndWait sends SIGKILL to pid and waits until it is gone (or a zombie that
	122	+	// its parent will reap). It reports whether the process is gone.
	123	+	func killAndWait(pid int, timeout time.Duration) bool {
80	124		_ = syscall.Kill(pid, syscall.SIGKILL)
81	125		deadline := time.Now().Add(timeout)
82		-	for time.Now().Before(deadline) {
83		-	if err := syscall.Kill(pid, 0); err != nil {
84		-	return
	126	+	for {
	127	+	if !pidAlive(pid) {
	128	+	return true
85	129		}
86		-	// A zombie child still answers kill(0); check its state.
87		-	if b, err := os.ReadFile("/proc/" + strconv.Itoa(pid) + "/stat"); err == nil {
88		-	if i := bytes.LastIndexByte(b, ')'); i >= 0 && i+2 < len(b) && b[i+2] == 'Z' {
89		-	return
90		-	}
	130	+	if time.Now().After(deadline) {
	131	+	return false
91	132		}
92	133		time.Sleep(20 * time.Millisecond)
93	134		}

 	"bnema.dev/ermite-runner/internal/sandbox"
 )
-// firecrackerProcs scans a /proc tree for Firecracker processes started by this
+// vmmProcs scans a /proc tree for processes started by this runner for a VM:
-// runner: argv[0] is "firecracker" and "--id" is followed by one of our ids
+// the Firecracker process and the jailer. A process is ours when its command
-// (see newID). Processes of other users of Firecracker are never matched. It
+// line has "--id <id>" with an id of the form sandbox.NewID produces and also
-// finds orphans even when the state directory was lost.
+// "--api-sock" (Firecracker) or "--exec-file" (jailer). The program name is not
-func firecrackerProcs(procDir string) map[int]string {
+// looked at: the binary may be renamed or versioned (firecracker-v1.17), and a
+// jailer in a PID namespace leaves a Firecracker whose argv[0] is whatever the
+// jailer chose. Processes of other users of Firecracker (other ids) are never
+// matched. It finds orphans even when the state directory was lost.
+func vmmProcs(procDir string) map[int]string {
 	out := map[int]string{}
 	ents, err := os.ReadDir(procDir)
 	if err != nil {
 		if err != nil || len(b) == 0 {
 			continue
 		}
-		args := strings.Split(strings.TrimRight(string(b), "\x00"), "\x00")
+		if id := vmIDFromCmdline(strings.Split(strings.TrimRight(string(b), "\x00"), "\x00")); id != "" {
-		if filepath.Base(args[0]) != "firecracker" {
+			out[pid] = id
-			continue
 		}
-		for i := 1; i+1 < len(args); i++ {
+	}
-			if args[i] == "--id" && sandbox.ValidID(args[i+1]) {
+	return out
-				out[pid] = args[i+1]
+}
+// vmIDFromCmdline returns the VM id of a command line, or "".
+func vmIDFromCmdline(args []string) string {
+	var id string
+	var marker bool
+	for i := 1; i < len(args); i++ {
+		switch args[i] {
+		case "--id":
+			if i+1 < len(args) && sandbox.ValidID(args[i+1]) {
+				id = args[i+1]
 			}
+		case "--api-sock", "--exec-file":
+			marker = true
 		}
 	}
-	return out
+	if marker {
+		return id
+	}
+	return ""
+}
+// readPidFile returns the pid the jailer wrote for a VM run in a PID namespace.
+func readPidFile(path string) int {
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return 0
+	}
+	pid, err := strconv.Atoi(strings.TrimSpace(string(b)))
+	if err != nil || pid <= 1 {
+		return 0
+	}
+	return pid
+}
+// pidAlive reports whether pid exists and is not a zombie.
+func pidAlive(pid int) bool {
+	if err := syscall.Kill(pid, 0); err != nil {
+		return false
+	}
+	if b, err := os.ReadFile("/proc/" + strconv.Itoa(pid) + "/stat"); err == nil {
+		if i := bytes.LastIndexByte(b, ')'); i >= 0 && i+2 < len(b) && b[i+2] == 'Z' {
+			return false
+		}
+	}
+	return true
 }
 // ring keeps the last N bytes written to it: the VMM's stdout/stderr, which
 	return string(bytes.TrimSpace(r.buf))
 }
-// killAndWait sends SIGKILL to pid and waits until it is gone (zombies of
+// killAndWait sends SIGKILL to pid and waits until it is gone (or a zombie that
-// children of this process are reaped by their own Wait).
+// its parent will reap). It reports whether the process is gone.
-func killAndWait(pid int, timeout time.Duration) {
+func killAndWait(pid int, timeout time.Duration) bool {
 	_ = syscall.Kill(pid, syscall.SIGKILL)
 	deadline := time.Now().Add(timeout)
-	for time.Now().Before(deadline) {
+	for {
-		if err := syscall.Kill(pid, 0); err != nil {
+		if !pidAlive(pid) {
-			return
+			return true
 		}
-		// A zombie child still answers kill(0); check its state.
+		if time.Now().After(deadline) {
-		if b, err := os.ReadFile("/proc/" + strconv.Itoa(pid) + "/stat"); err == nil {
+			return false
-			if i := bytes.LastIndexByte(b, ')'); i >= 0 && i+2 < len(b) && b[i+2] == 'Z' {
-				return
-			}
 		}
 		time.Sleep(20 * time.Millisecond)
 	}

M internal/sandbox/vm/reap.go +49 -14


@@ -8,6 +8,8 @@ import (
8	8		"fmt"
9	9		"os"
10	10		"path/filepath"
	11	+	"strconv"
	12	+	"strings"
11	13		"time"
12	14
13	15		"github.com/bnema/zerowrap"

@@ -17,32 +19,43 @@ import (
17	19		)
18	20
19	21		// Reap implements sandbox.Driver. It removes what a previous run of the runner
20		-	// left: Firecracker processes (by state record and by scanning /proc for our
21		-	// "--id erXXXXXXXX"), TAP devices named ertap<N>, the nftables table, and VM
22		-	// directories. It relies on the state-directory lock taken in New: no other
23		-	// runner instance can own what it removes.
	22	+	// left: VMM processes (Firecracker and jailer: matched by "--id <our id>" in
	23	+	// their command line, never by program name), TAP devices named ertap<N>, the
	24	+	// nftables table, VM directories and jailer cgroups. It also finishes the
	25	+	// cleanup of sandboxes that reported ErrLeaked and frees their slots. It relies
	26	+	// on the host-wide lock taken in New: no other runner can own what it removes.
	27	+	// It returns ErrBusy while a sandbox is running.
24	28		func (d *Driver) Reap(ctx context.Context) error {
25	29		log := zerowrap.FromCtx(ctx)
	30	+	d.mu.Lock()
	31	+	busy := d.slots.Busy()
	32	+	d.mu.Unlock()
	33	+	if busy > 0 {
	34	+	return fmt.Errorf("%w (%d running)", sandbox.ErrBusy, busy)
	35	+	}
26	36		var errs []error
27	37
28	38		recs, err := d.store.list()
29	39		if err != nil {
30	40		errs = append(errs, err)
31	41		}
32		-	procs := firecrackerProcs(d.opt.ProcDir)
	42	+	procs := vmmProcs(d.opt.ProcDir)
33	43		killed := map[int]bool{}
34	44		killProc := func(pid int, id string) {
35	45		if killed[pid] {
36	46		return
37	47		}
38	48		killed[pid] = true
39		-	log.Warn().Int("pid", pid).Str("vm", id).Msg("reaping leftover firecracker process")
40		-	killAndWait(pid, 5*time.Second)
	49	+	log.Warn().Int("pid", pid).Str("vm", id).Msg("reaping leftover VMM process")
	50	+	if !d.opt.kill(pid, 5*time.Second) {
	51	+	errs = append(errs, fmt.Errorf("vmm process %d (%s) did not exit", pid, id))
	52	+	}
41	53		}
42		-	// 1. Processes: by record, but only if the pid is still one of ours (pids
43		-	// are reused), then any other orphan found in /proc.
	54	+	// 1. Processes. A recorded pid is only trusted when /proc/<pid>/cmdline still
	55	+	// carries this record's id (pids are reused); then every other process of
	56	+	// ours found in /proc, which also covers a lost state directory.
44	57		for _, r := range recs {
45		-	if r.PID > 0 && procs[r.PID] == r.ID {
	58	+	if r.PID > 0 && cmdlineID(d.opt.ProcDir, r.PID) == r.ID {
46	59		killProc(r.PID, r.ID)
47	60		}
48	61		}

@@ -65,10 +78,8 @@ func (d *Driver) Reap(ctx context.Context) error {
65	78		}
66	79		}
67	80
68		-	// 3. Files: directories of recorded VMs, then any directory with one of our
69		-	// ids that has no record (crash before the record was written is
70		-	// impossible, but a deleted state dir is not).
71		-	// The jailer's chroot base has one directory per exec-file name.
	81	+	// 3. Files and cgroups. The jailer's chroot base has one directory per
	82	+	// exec-file name.
72	83		parents := []string{filepath.Join(d.cfg.StateDir, "vm")}
73	84		jails, _ := os.ReadDir(filepath.Join(d.cfg.StateDir, "jail"))
74	85		for _, j := range jails {

@@ -87,6 +98,15 @@ func (d *Driver) Reap(ctx context.Context) error {
87	98		}
88	99		}
89	100		}
	101	+	cg := filepath.Join(d.opt.CgroupRoot, filepath.Base(d.fcPath))
	102	+	cents, _ := os.ReadDir(cg)
	103	+	for _, e := range cents {
	104	+	if e.IsDir() && sandbox.ValidID(e.Name()) {
	105	+	if err := removeCgroup(d.opt.CgroupRoot, filepath.Base(d.fcPath), e.Name()); err != nil {
	106	+	errs = append(errs, err)
	107	+	}
	108	+	}
	109	+	}
90	110		if len(errs) == 0 {
91	111		for _, r := range recs {
92	112		if err := d.store.remove(r.ID); err != nil {

@@ -94,5 +114,20 @@ func (d *Driver) Reap(ctx context.Context) error {
94	114		}
95	115		}
96	116		}
	117	+	if len(errs) == 0 {
	118	+	d.mu.Lock()
	119	+	d.slots.ReleaseLeaked()
	120	+	d.mu.Unlock()
	121	+	}
97	122		return errors.Join(errs...)
98	123		}
	124	+
	125	+	// cmdlineID returns the VM id in the command line of pid, or "" (not running,
	126	+	// not ours, or recycled for another program).
	127	+	func cmdlineID(procDir string, pid int) string {
	128	+	b, err := os.ReadFile(filepath.Join(procDir, strconv.Itoa(pid), "cmdline"))
	129	+	if err != nil \|\| len(b) == 0 {
	130	+	return ""
	131	+	}
	132	+	return vmIDFromCmdline(strings.Split(strings.TrimRight(string(b), "\x00"), "\x00"))
	133	+	}

 	"fmt"
 	"os"
 	"path/filepath"
+	"strconv"
+	"strings"
 	"time"
 	"github.com/bnema/zerowrap"
 )
 // Reap implements sandbox.Driver. It removes what a previous run of the runner
-// left: Firecracker processes (by state record and by scanning /proc for our
+// left: VMM processes (Firecracker and jailer: matched by "--id <our id>" in
-// "--id erXXXXXXXX"), TAP devices named ertap<N>, the nftables table, and VM
+// their command line, never by program name), TAP devices named ertap<N>, the
-// directories. It relies on the state-directory lock taken in New: no other
+// nftables table, VM directories and jailer cgroups. It also finishes the
-// runner instance can own what it removes.
+// cleanup of sandboxes that reported ErrLeaked and frees their slots. It relies
+// on the host-wide lock taken in New: no other runner can own what it removes.
+// It returns ErrBusy while a sandbox is running.
 func (d *Driver) Reap(ctx context.Context) error {
 	log := zerowrap.FromCtx(ctx)
+	d.mu.Lock()
+	busy := d.slots.Busy()
+	d.mu.Unlock()
+	if busy > 0 {
+		return fmt.Errorf("%w (%d running)", sandbox.ErrBusy, busy)
+	}
 	var errs []error
 	recs, err := d.store.list()
 	if err != nil {
 		errs = append(errs, err)
 	}
-	procs := firecrackerProcs(d.opt.ProcDir)
+	procs := vmmProcs(d.opt.ProcDir)
 	killed := map[int]bool{}
 	killProc := func(pid int, id string) {
 		if killed[pid] {
 			return
 		}
 		killed[pid] = true
-		log.Warn().Int("pid", pid).Str("vm", id).Msg("reaping leftover firecracker process")
+		log.Warn().Int("pid", pid).Str("vm", id).Msg("reaping leftover VMM process")
-		killAndWait(pid, 5*time.Second)
+		if !d.opt.kill(pid, 5*time.Second) {
+			errs = append(errs, fmt.Errorf("vmm process %d (%s) did not exit", pid, id))
+		}
 	}
-	// 1. Processes: by record, but only if the pid is still one of ours (pids
+	// 1. Processes. A recorded pid is only trusted when /proc/<pid>/cmdline still
-	// are reused), then any other orphan found in /proc.
+	// carries this record's id (pids are reused); then every other process of
+	// ours found in /proc, which also covers a lost state directory.
 	for _, r := range recs {
-		if r.PID > 0 && procs[r.PID] == r.ID {
+		if r.PID > 0 && cmdlineID(d.opt.ProcDir, r.PID) == r.ID {
 			killProc(r.PID, r.ID)
 		}
 	}
 		}
 	}
-	// 3. Files: directories of recorded VMs, then any directory with one of our
+	// 3. Files and cgroups. The jailer's chroot base has one directory per
-	// ids that has no record (crash before the record was written is
+	// exec-file name.
-	// impossible, but a deleted state dir is not).
-	// The jailer's chroot base has one directory per exec-file name.
 	parents := []string{filepath.Join(d.cfg.StateDir, "vm")}
 	jails, _ := os.ReadDir(filepath.Join(d.cfg.StateDir, "jail"))
 	for _, j := range jails {
 			}
 		}
 	}
+	cg := filepath.Join(d.opt.CgroupRoot, filepath.Base(d.fcPath))
+	cents, _ := os.ReadDir(cg)
+	for _, e := range cents {
+		if e.IsDir() && sandbox.ValidID(e.Name()) {
+			if err := removeCgroup(d.opt.CgroupRoot, filepath.Base(d.fcPath), e.Name()); err != nil {
+				errs = append(errs, err)
+			}
+		}
+	}
 	if len(errs) == 0 {
 		for _, r := range recs {
 			if err := d.store.remove(r.ID); err != nil {
 			}
 		}
 	}
+	if len(errs) == 0 {
+		d.mu.Lock()
+		d.slots.ReleaseLeaked()
+		d.mu.Unlock()
+	}
 	return errors.Join(errs...)
 }
+// cmdlineID returns the VM id in the command line of pid, or "" (not running,
+// not ours, or recycled for another program).
+func cmdlineID(procDir string, pid int) string {
+	b, err := os.ReadFile(filepath.Join(procDir, strconv.Itoa(pid), "cmdline"))
+	if err != nil || len(b) == 0 {
+		return ""
+	}
+	return vmIDFromCmdline(strings.Split(strings.TrimRight(string(b), "\x00"), "\x00"))
+}

M internal/sandbox/vm/vm_test.go +343 -6


@@ -3,6 +3,7 @@
3	3		package vm
4	4
5	5		import (
	6	+	"bytes"
6	7		"context"
7	8		"encoding/json"
8	9		"errors"

@@ -20,6 +21,8 @@ import (
20	21		"testing"
21	22		"time"
22	23
	24	+	"github.com/bnema/zerowrap"
	25	+
23	26		"bnema.dev/ermite-runner/internal/cmdrun"
24	27		"bnema.dev/ermite-runner/internal/execstream"
25	28		"bnema.dev/ermite-runner/internal/execstream/execpb"

@@ -108,7 +111,10 @@ type env struct {
108	111		}
109	112
110	113		// newEnv prepares a state dir, fake binaries, kernel and images.
111		-	func newEnv(t testing.T, mode string) env {
	114	+	func newEnv(t testing.T, mode string) env { return newEnvNamed(t, mode, "firecracker") }
	115	+
	116	+	// newEnvNamed runs the fake VMM under another program name (argv[0]).
	117	+	func newEnvNamed(t testing.T, mode, argv0 string) env {
112	118		t.Helper()
113	119		bash, err := exec.LookPath("bash")
114	120		if err != nil {

@@ -118,7 +124,7 @@ func newEnv(t testing.T, mode string) env {
118	124		e := &env{taps: map[string]bool{}, plumb: sandboxtest.NewPlumbing(), log: filepath.Join(dir, "fc.log")}
119	125		self, _ := os.Executable()
120	126		fc := filepath.Join(dir, "firecracker")
121		-	script := fmt.Sprintf("#!%s\nexport FC_HELPER=1 FC_MODE=%q FC_LOG=%q\nexec -a firecracker %q \"$@\"\n", bash, mode, e.log, self)
	127	+	script := fmt.Sprintf("#!%s\nexport FC_HELPER=1 FC_MODE=%q FC_LOG=%q\nexec -a %q %q \"$@\"\n", bash, mode, e.log, argv0, self)
122	128		if err := os.WriteFile(fc, []byte(script), 0o755); err != nil {
123	129		t.Fatal(err)
124	130		}

@@ -133,6 +139,7 @@ func newEnv(t testing.T, mode string) env {
133	139		Isolation: sandbox.VM, StateDir: filepath.Join(dir, "state"), Capacity: 2, Image: img,
134	140		JobCPUs: 1, JobMemoryMiB: 512, JobDiskMiB: 1024, UIDBase: 900000, KVMPath: "/dev/null",
135	141		Network: mustPrefix("10.200.0.0/16"), Firecracker: fc, JailerOff: true, Executable: exe, DNSPort: 0,
	142	+	LockFile: filepath.Join(dir, "host.lock"), JobPids: 1000, DiskBandwidthMiB: 400, DiskIOPS: 20000, NetMbit: 1000,
136	143		}
137	144		e.run = &cmdrun.Fake{Handle: func(c cmdrun.Call) ([]byte, error) {
138	145		e.mu.Lock()

@@ -437,8 +444,7 @@ func TestReapAfterRunnerCrash(t *testing.T) {
437	444		t.Fatal("fake VM not running")
438	445		}
439	446		// The runner dies: its lock goes away, nothing else is cleaned.
440		-	_ = d1.lock.Close()
441		-	d1.lock = nil
	447	+	_ = d1.lock.Release()
442	448		_ = conn.Close()
443	449		if e.tapCount() != 1 {
444	450		t.Fatal("setup: no TAP")

@@ -506,7 +512,7 @@ func TestSecondRunnerOnSameStateDirRefused(t *testing.T) {
506	512		e := newEnv(t, "")
507	513		_ = e.driver(t)
508	514		_, err := New(context.Background(), e.cfg, Options{Run: e.run, Net: e.plumb})
509		-	if err == nil \|\| !strings.Contains(err.Error(), "another ermite-runner") {
	515	+	if err == nil \|\| !strings.Contains(err.Error(), "another ermite-runner is running on this host") {
510	516		t.Fatalf("got %v", err)
511	517		}
512	518		}

@@ -646,7 +652,7 @@ func TestFirecrackerProcsScan(t *testing.T) {
646	652		mk("103", "bash\x00--id\x00er00112233\x00") // not firecracker
647	653		mk("1", "firecracker\x00--id\x00er99999999\x00") // pid 1 is never touched
648	654		mk("abc", "")
649		-	got := firecrackerProcs(proc)
	655	+	got := vmmProcs(proc)
650	656		if len(got) != 2 \|\| got[100] != "er00112233" \|\| got[101] != "er44556677" {
651	657		t.Fatalf("%v", got)
652	658		}

@@ -682,3 +688,334 @@ func TestAPIErrorsSurfaceFirecrackerFault(t *testing.T) {
682	688
683	689		func mustPrefix(s string) netip.Prefix { return netip.MustParsePrefix(s) }
684	690		func mustAddr(s string) netip.Addr { return netip.MustParseAddr(s) }
	691	+
	692	+	// C3: processes are matched by "--id <our id>" in their command line, not by the
	693	+	// program name: a renamed or versioned Firecracker binary is still found and
	694	+	// reaped, and a recycled pid is never killed on the strength of a record alone.
	695	+	func TestReapFindsRenamedVMMByID(t *testing.T) {
	696	+	e := newEnvNamed(t, "", "fc-renamed-1.17")
	697	+	d1 := e.driver(t)
	698	+	sb, _ := d1.New()
	699	+	conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
	700	+	if err != nil {
	701	+	t.Fatal(err)
	702	+	}
	703	+	runStub(t, conn)
	704	+	pid := sb.(*vmSandbox).rec.PID
	705	+	if got := cmdlineID("/proc", pid); got != sb.ID() {
	706	+	t.Fatalf("cmdlineID=%q want %q (argv[0] is %q)", got, sb.ID(), "fc-renamed-1.17")
	707	+	}
	708	+	_ = d1.lock.Release()
	709	+	_ = conn.Close()
	710	+	d2 := e.driver(t)
	711	+	if err := d2.Reap(context.Background()); err != nil {
	712	+	t.Fatal(err)
	713	+	}
	714	+	if pidAlive(pid) {
	715	+	t.Fatalf("a VMM with a renamed binary survived Reap (pid %d)", pid)
	716	+	}
	717	+	}
	718	+
	719	+	func TestVMIDFromCmdline(t *testing.T) {
	720	+	for name, tc := range map[string]struct {
	721	+	args []string
	722	+	want string
	723	+	}{
	724	+	"firecracker": {[]string{"firecracker", "--id", "er00112233", "--api-sock", "/x"}, "er00112233"},
	725	+	"jailer": {[]string{"jailer", "--id", "er00112233", "--exec-file", "/fc", "--uid", "1"}, "er00112233"},
	726	+	"renamed": {[]string{"anything", "--api-sock", "/x", "--id", "er44556677"}, "er44556677"},
	727	+	"foreign id": {[]string{"firecracker", "--id", "my-vm", "--api-sock", "/x"}, ""},
	728	+	"no marker": {[]string{"python", "--id", "er00112233"}, ""},
	729	+	"id without value": {[]string{"firecracker", "--api-sock", "/x", "--id"}, ""},
	730	+	"path-like id": {[]string{"firecracker", "--id", "er../../../x", "--api-sock", "/x"}, ""},
	731	+	} {
	732	+	if got := vmIDFromCmdline(tc.args); got != tc.want {
	733	+	t.Errorf("%s: got %q want %q", name, got, tc.want)
	734	+	}
	735	+	}
	736	+	}
	737	+
	738	+	func TestReapDoesNotTrustARecycledPid(t *testing.T) {
	739	+	e := newEnv(t, "")
	740	+	d := e.driver(t)
	741	+	// A record pointing at this very test process: its command line carries no
	742	+	// VM id, so it must not be killed.
	743	+	_ = d.store.save(record{ID: "er00112233", Slot: 0, PID: os.Getpid()})
	744	+	if err := d.Reap(context.Background()); err != nil {
	745	+	t.Fatal(err)
	746	+	}
	747	+	if !pidAlive(os.Getpid()) {
	748	+	t.Fatal("Reap killed a process on the strength of a stale record")
	749	+	}
	750	+	}
	751	+
	752	+	// C5 + C8: when the VMM cannot be killed, the TAP, the firewall policy and the
	753	+	// files are left alone (the VM may still be running), the error wraps ErrLeaked,
	754	+	// the slot stays taken, and Reap (with a working kill) finishes the cleanup and
	755	+	// frees the slot.
	756	+	func TestStopKeepsNetworkWhenTheVMMSurvives(t *testing.T) {
	757	+	e := newEnv(t, "")
	758	+	stubborn := true
	759	+	d, err := New(context.Background(), e.cfg, Options{Run: e.run, Net: e.plumb, BootTimeout: 5 * time.Second,
	760	+	kill: func(pid int, to time.Duration) bool {
	761	+	if stubborn {
	762	+	return false
	763	+	}
	764	+	return killAndWait(pid, to)
	765	+	}})
	766	+	if err != nil {
	767	+	t.Fatal(err)
	768	+	}
	769	+	defer func() { _ = d.Close() }()
	770	+	sb, _ := d.New()
	771	+	conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
	772	+	if err != nil {
	773	+	t.Fatal(err)
	774	+	}
	775	+	runStub(t, conn)
	776	+
	777	+	err = sb.Stop(context.Background())
	778	+	if !errors.Is(err, sandbox.ErrLeaked) {
	779	+	t.Fatalf("Stop must report ErrLeaked, got %v", err)
	780	+	}
	781	+	if e.tapCount() != 1 \|\| e.plumb.Live() != 1 {
	782	+	t.Fatalf("the TAP and the policy must stay while the VMM may run: taps=%d attached=%d", e.tapCount(), e.plumb.Live())
	783	+	}
	784	+	if recs, _ := d.store.list(); len(recs) != 1 {
	785	+	t.Fatalf("the state record must stay for Reap: %v", recs)
	786	+	}
	787	+	if err := sb.Stop(context.Background()); !errors.Is(err, sandbox.ErrLeaked) {
	788	+	t.Fatalf("a second Stop must keep reporting the leak, got %v", err)
	789	+	}
	790	+	if d.slots.InUse() != 1 {
	791	+	t.Fatal("the slot of a leaked sandbox stays taken")
	792	+	}
	793	+	// Other jobs keep running on the free slot; the leaked one is never reused.
	794	+	other, err := d.New()
	795	+	if err != nil {
	796	+	t.Fatal(err)
	797	+	}
	798	+	if other.(vmSandbox).slot == sb.(vmSandbox).slot {
	799	+	t.Fatal("a leaked slot was handed out again")
	800	+	}
	801	+	// Reap refuses while a sandbox is running...
	802	+	if err := d.Reap(context.Background()); !errors.Is(err, sandbox.ErrBusy) {
	803	+	t.Fatalf("Reap with a handed-out sandbox: %v", err)
	804	+	}
	805	+	_ = other.Stop(context.Background())
	806	+	// ...and finishes the job once idle.
	807	+	stubborn = false
	808	+	if err := d.Reap(context.Background()); err != nil {
	809	+	t.Fatal(err)
	810	+	}
	811	+	if left := e.leftovers(t); len(left) != 0 {
	812	+	t.Fatalf("leftovers after Reap: %v", left)
	813	+	}
	814	+	if d.slots.InUse() != 0 {
	815	+	t.Fatal("Reap must free the slot of the leaked sandbox")
	816	+	}
	817	+	}
	818	+
	819	+	// C7: Stop from another goroutine aborts a boot in progress at once, instead of
	820	+	// waiting for the boot timeout, and leaves nothing behind.
	821	+	func TestStopAbortsABootInProgress(t *testing.T) {
	822	+	e := newEnv(t, "hang")
	823	+	d, err := New(context.Background(), e.cfg, Options{Run: e.run, Net: e.plumb, BootTimeout: time.Minute})
	824	+	if err != nil {
	825	+	t.Fatal(err)
	826	+	}
	827	+	defer func() { _ = d.Close() }()
	828	+	sb, _ := d.New()
	829	+	started := make(chan error, 1)
	830	+	go func() {
	831	+	_, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
	832	+	started <- err
	833	+	}()
	834	+	time.Sleep(700 * time.Millisecond) // the fake is now waiting for the executor
	835	+	t0 := time.Now()
	836	+	if err := sb.Stop(context.Background()); err != nil {
	837	+	t.Fatal(err)
	838	+	}
	839	+	select {
	840	+	case err := <-started:
	841	+	if !errors.Is(err, sandbox.ErrAborted) {
	842	+	t.Fatalf("Start must report that Stop aborted it, got %v", err)
	843	+	}
	844	+	case <-time.After(10 * time.Second):
	845	+	t.Fatal("Start was not aborted by Stop")
	846	+	}
	847	+	if time.Since(t0) > 8*time.Second {
	848	+	t.Fatalf("Stop took %v", time.Since(t0))
	849	+	}
	850	+	if left := e.leftovers(t); len(left) != 0 {
	851	+	t.Fatalf("leftovers: %v", left)
	852	+	}
	853	+	// Stop before Start: Start must refuse.
	854	+	sb2, _ := d.New()
	855	+	_ = sb2.Stop(context.Background())
	856	+	if _, err := sb2.Start(context.Background(), sandbox.JobSpec{}); err == nil {
	857	+	t.Fatal("Start after Stop accepted")
	858	+	}
	859	+	}
	860	+
	861	+	// C16: the sandbox exposes its id, which names its files.
	862	+	func TestSandboxIDNamesItsFiles(t *testing.T) {
	863	+	e := newEnv(t, "")
	864	+	d := e.driver(t)
	865	+	sb, _ := d.New()
	866	+	if !sandbox.ValidID(sb.ID()) {
	867	+	t.Fatalf("id %q", sb.ID())
	868	+	}
	869	+	conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
	870	+	if err != nil {
	871	+	t.Fatal(err)
	872	+	}
	873	+	runStub(t, conn)
	874	+	defer func() { _ = sb.Stop(context.Background()) }()
	875	+	if _, err := os.Stat(filepath.Join(e.cfg.StateDir, "vm", sb.ID())); err != nil {
	876	+	t.Fatalf("the VM directory must be named after the sandbox id: %v", err)
	877	+	}
	878	+	if d.Isolation() != sandbox.IsolationVM \|\| d.Mode() != sandbox.VM {
	879	+	t.Fatal("isolation")
	880	+	}
	881	+	}
	882	+
	883	+	// S7: cgroup v2 limits, a new PID namespace, and the jailer's own options.
	884	+	func TestJailerArgs(t *testing.T) {
	885	+	args := jailerArgs("/usr/bin/jailer", "/usr/bin/firecracker", "er00112233", 900003, "/var/lib/er/jail",
	886	+	sandbox.JobSpec{CPUs: 4, MemoryMiB: 2048}, nil)
	887	+	line := " " + strings.Join(args, " ") + " "
	888	+	for _, want := range []string{
	889	+	" --id er00112233 ", " --exec-file /usr/bin/firecracker ", " --uid 900003 ", " --gid 900003 ",
	890	+	" --chroot-base-dir /var/lib/er/jail ", " --cgroup-version 2 ", " --new-pid-ns ",
	891	+	" --cgroup memory.max=" + fmt.Sprint((2048+vmmOverheadMiB)<<20) + " ",
	892	+	" --cgroup pids.max=64 ", " --resource-limit no-file=4096 ", " -- ",
	893	+	} {
	894	+	if !strings.Contains(line, want) {
	895	+	t.Errorf("jailer args lack %q: %s", want, line)
	896	+	}
	897	+	}
	898	+	if args[len(args)-1] != "--" {
	899	+	t.Errorf("the jailer separator must come last when there are no extra args: %v", args)
	900	+	}
	901	+	more := jailerArgs("j", "f", "er00112233", 1, "/b", sandbox.JobSpec{CPUs: 1, MemoryMiB: 512}, []string{"--x"})
	902	+	if more[len(more)-2] != "--" \|\| more[len(more)-1] != "--x" {
	903	+	t.Errorf("extra args follow the separator: %v", more)
	904	+	}
	905	+	}
	906	+
	907	+	// S7: sane rate limiters on the scratch disk and both directions of eth0, off
	908	+	// when configured to zero, and only on the writable disk.
	909	+	func TestRateLimitersInTheVMConfig(t *testing.T) {
	910	+	e := newEnv(t, "")
	911	+	d := e.driver(t)
	912	+	sb, _ := d.New()
	913	+	conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
	914	+	if err != nil {
	915	+	t.Fatal(err)
	916	+	}
	917	+	runStub(t, conn)
	918	+	defer func() { _ = sb.Stop(context.Background()) }()
	919	+	b, _ := os.ReadFile(e.log)
	920	+	log := string(b)
	921	+	for _, want := range []string{
	922	+	`"rate_limiter":{"bandwidth":{"size":419430400,"one_time_burst":419430400,"refill_time":1000},"ops":{"size":20000,"one_time_burst":20000,"refill_time":1000}}`,
	923	+	`"rx_rate_limiter":{"bandwidth":{"size":125000000,"one_time_burst":125000000,"refill_time":1000}}`,
	924	+	`"tx_rate_limiter":{"bandwidth":{"size":125000000,"one_time_burst":125000000,"refill_time":1000}}`,
	925	+	} {
	926	+	if !strings.Contains(log, want) {
	927	+	t.Errorf("API log lacks %s\n%s", want, log)
	928	+	}
	929	+	}
	930	+	if n := strings.Count(log, `"rate_limiter"`); n != 1 {
	931	+	t.Errorf("only the scratch drive is rate limited, got %d rate_limiter fields", n)
	932	+	}
	933	+	if diskLimiter(0, 0) != nil \|\| netLimiter(0) != nil {
	934	+	t.Error("zero must disable the limiters")
	935	+	}
	936	+	if l := diskLimiter(0, 100); l.Bandwidth != nil \|\| l.Ops == nil {
	937	+	t.Errorf("%+v", l)
	938	+	}
	939	+	}
	940	+
	941	+	// S8: the free-space check warns, and only warns.
	942	+	func TestNewWarnsOnLowFreeSpace(t *testing.T) {
	943	+	e := newEnv(t, "")
	944	+	var buf bytes.Buffer
	945	+	ctx := zerowrap.WithCtx(context.Background(), zerowrap.New(zerowrap.Config{Level: "debug", Format: "json", Output: &buf}))
	946	+	d, err := New(ctx, e.cfg, Options{Run: e.run, Net: e.plumb, FreeSpace: func(string) (int64, error) { return 1, nil }})
	947	+	if err != nil {
	948	+	t.Fatalf("low space must not fail New: %v", err)
	949	+	}
	950	+	defer func() { _ = d.Close() }()
	951	+	if !strings.Contains(buf.String(), "less free space") {
	952	+	t.Fatalf("no warning: %s", buf.String())
	953	+	}
	954	+	}
	955	+
	956	+	// S11: the scratch disk is created without following symlinks and chowned
	957	+	// through its descriptor.
	958	+	func TestMakeScratchRefusesSymlinksAndExistingFiles(t *testing.T) {
	959	+	dir := t.TempDir()
	960	+	target := filepath.Join(dir, "victim")
	961	+	_ = os.WriteFile(target, []byte("precious"), 0o600)
	962	+	link := filepath.Join(dir, "scratch.ext4")
	963	+	if err := os.Symlink(target, link); err != nil {
	964	+	t.Fatal(err)
	965	+	}
	966	+	run := &cmdrun.Fake{}
	967	+	if err := makeScratch(context.Background(), run, link, 8, -1); err == nil {
	968	+	t.Fatal("a planted symlink must be refused")
	969	+	}
	970	+	if b, _ := os.ReadFile(target); string(b) != "precious" {
	971	+	t.Fatalf("the symlink target was modified: %q", b)
	972	+	}
	973	+	if len(run.Calls) != 0 {
	974	+	t.Fatalf("mkfs must not run: %v", run.Lines())
	975	+	}
	976	+	ok := filepath.Join(dir, "fresh.ext4")
	977	+	if err := makeScratch(context.Background(), run, ok, 8, os.Getuid()); err != nil {
	978	+	t.Fatal(err)
	979	+	}
	980	+	if st, err := os.Stat(ok); err != nil \|\| st.Size() != 8<<20 \|\| st.Mode().Perm() != 0o600 {
	981	+	t.Fatalf("%v %v", st, err)
	982	+	}
	983	+	if err := makeScratch(context.Background(), run, ok, 8, -1); err == nil {
	984	+	t.Fatal("an existing file must be refused (O_EXCL)")
	985	+	}
	986	+	}
	987	+
	988	+	func TestOwnUsesLchown(t *testing.T) {
	989	+	dir := t.TempDir()
	990	+	l := newLayout(dir, "firecracker", "er00112233", true, os.Getuid())
	991	+	_ = os.MkdirAll(l.root, 0o700)
	992	+	target := filepath.Join(dir, "victim")
	993	+	_ = os.WriteFile(target, nil, 0o600)
	994	+	if err := os.Symlink(target, l.host("link")); err != nil {
	995	+	t.Fatal(err)
	996	+	}
	997	+	if err := l.own("link"); err != nil {
	998	+	t.Fatalf("Lchown on a link must act on the link itself: %v", err)
	999	+	}
	1000	+	if err := l.own("missing"); err == nil {
	1001	+	t.Fatal("a missing file is an error")
	1002	+	}
	1003	+	if u := newLayout(dir, "f", "er00112233", false, 1); u.own("anything") != nil {
	1004	+	t.Fatal("no-op without jailer")
	1005	+	}
	1006	+	}
	1007	+
	1008	+	func TestRemoveCgroup(t *testing.T) {
	1009	+	root := t.TempDir()
	1010	+	_ = os.MkdirAll(filepath.Join(root, "firecracker", "er00112233"), 0o755)
	1011	+	if err := removeCgroup(root, "firecracker", "er00112233"); err != nil {
	1012	+	t.Fatal(err)
	1013	+	}
	1014	+	if err := removeCgroup(root, "firecracker", "er00112233"); err != nil {
	1015	+	t.Fatalf("a missing cgroup is fine: %v", err)
	1016	+	}
	1017	+	_ = os.MkdirAll(filepath.Join(root, "firecracker", "er44556677", "child"), 0o755)
	1018	+	if err := removeCgroup(root, "firecracker", "er44556677"); err == nil {
	1019	+	t.Fatal("a non-empty cgroup cannot be removed and must be reported")
	1020	+	}
	1021	+	}


@@ -3,6 +3,7 @@
3		package vm	3		package vm
4			4
5		import (	5		import (
			6	+	"bytes"
6		"context"	7		"context"
7		"encoding/json"	8		"encoding/json"
8		"errors"	9		"errors"

@@ -20,6 +21,8 @@ import (
20		"testing"	21		"testing"
21		"time"	22		"time"
22			23
			24	+	"github.com/bnema/zerowrap"
			25	+
23		"bnema.dev/ermite-runner/internal/cmdrun"	26		"bnema.dev/ermite-runner/internal/cmdrun"
24		"bnema.dev/ermite-runner/internal/execstream"	27		"bnema.dev/ermite-runner/internal/execstream"
25		"bnema.dev/ermite-runner/internal/execstream/execpb"	28		"bnema.dev/ermite-runner/internal/execstream/execpb"

@@ -108,7 +111,10 @@ type env struct {
108		}	111		}
109			112
110		// newEnv prepares a state dir, fake binaries, kernel and images.	113		// newEnv prepares a state dir, fake binaries, kernel and images.
111	-	func newEnv(t testing.T, mode string) env {	114	+	func newEnv(t testing.T, mode string) env { return newEnvNamed(t, mode, "firecracker") }
			115	+
			116	+	// newEnvNamed runs the fake VMM under another program name (argv[0]).
			117	+	func newEnvNamed(t testing.T, mode, argv0 string) env {
112		t.Helper()	118		t.Helper()
113		bash, err := exec.LookPath("bash")	119		bash, err := exec.LookPath("bash")
114		if err != nil {	120		if err != nil {

@@ -118,7 +124,7 @@ func newEnv(t testing.T, mode string) env {
118		e := &env{taps: map[string]bool{}, plumb: sandboxtest.NewPlumbing(), log: filepath.Join(dir, "fc.log")}	124		e := &env{taps: map[string]bool{}, plumb: sandboxtest.NewPlumbing(), log: filepath.Join(dir, "fc.log")}
119		self, _ := os.Executable()	125		self, _ := os.Executable()
120		fc := filepath.Join(dir, "firecracker")	126		fc := filepath.Join(dir, "firecracker")
121	-	script := fmt.Sprintf("#!%s\nexport FC_HELPER=1 FC_MODE=%q FC_LOG=%q\nexec -a firecracker %q \"$@\"\n", bash, mode, e.log, self)	127	+	script := fmt.Sprintf("#!%s\nexport FC_HELPER=1 FC_MODE=%q FC_LOG=%q\nexec -a %q %q \"$@\"\n", bash, mode, e.log, argv0, self)
122		if err := os.WriteFile(fc, []byte(script), 0o755); err != nil {	128		if err := os.WriteFile(fc, []byte(script), 0o755); err != nil {
123		t.Fatal(err)	129		t.Fatal(err)
124		}	130		}

@@ -133,6 +139,7 @@ func newEnv(t testing.T, mode string) env {
133		Isolation: sandbox.VM, StateDir: filepath.Join(dir, "state"), Capacity: 2, Image: img,	139		Isolation: sandbox.VM, StateDir: filepath.Join(dir, "state"), Capacity: 2, Image: img,
134		JobCPUs: 1, JobMemoryMiB: 512, JobDiskMiB: 1024, UIDBase: 900000, KVMPath: "/dev/null",	140		JobCPUs: 1, JobMemoryMiB: 512, JobDiskMiB: 1024, UIDBase: 900000, KVMPath: "/dev/null",
135		Network: mustPrefix("10.200.0.0/16"), Firecracker: fc, JailerOff: true, Executable: exe, DNSPort: 0,	141		Network: mustPrefix("10.200.0.0/16"), Firecracker: fc, JailerOff: true, Executable: exe, DNSPort: 0,
			142	+	LockFile: filepath.Join(dir, "host.lock"), JobPids: 1000, DiskBandwidthMiB: 400, DiskIOPS: 20000, NetMbit: 1000,
136		}	143		}
137		e.run = &cmdrun.Fake{Handle: func(c cmdrun.Call) ([]byte, error) {	144		e.run = &cmdrun.Fake{Handle: func(c cmdrun.Call) ([]byte, error) {
138		e.mu.Lock()	145		e.mu.Lock()

@@ -437,8 +444,7 @@ func TestReapAfterRunnerCrash(t *testing.T) {
437		t.Fatal("fake VM not running")	444		t.Fatal("fake VM not running")
438		}	445		}
439		// The runner dies: its lock goes away, nothing else is cleaned.	446		// The runner dies: its lock goes away, nothing else is cleaned.
440	-	_ = d1.lock.Close()	447	+	_ = d1.lock.Release()
441	-	d1.lock = nil
442		_ = conn.Close()	448		_ = conn.Close()
443		if e.tapCount() != 1 {	449		if e.tapCount() != 1 {
444		t.Fatal("setup: no TAP")	450		t.Fatal("setup: no TAP")

@@ -506,7 +512,7 @@ func TestSecondRunnerOnSameStateDirRefused(t *testing.T) {
506		e := newEnv(t, "")	512		e := newEnv(t, "")
507		_ = e.driver(t)	513		_ = e.driver(t)
508		_, err := New(context.Background(), e.cfg, Options{Run: e.run, Net: e.plumb})	514		_, err := New(context.Background(), e.cfg, Options{Run: e.run, Net: e.plumb})
509	-	if err == nil \|\| !strings.Contains(err.Error(), "another ermite-runner") {	515	+	if err == nil \|\| !strings.Contains(err.Error(), "another ermite-runner is running on this host") {
510		t.Fatalf("got %v", err)	516		t.Fatalf("got %v", err)
511		}	517		}
512		}	518		}

@@ -646,7 +652,7 @@ func TestFirecrackerProcsScan(t *testing.T) {
646		mk("103", "bash\x00--id\x00er00112233\x00") // not firecracker	652		mk("103", "bash\x00--id\x00er00112233\x00") // not firecracker
647		mk("1", "firecracker\x00--id\x00er99999999\x00") // pid 1 is never touched	653		mk("1", "firecracker\x00--id\x00er99999999\x00") // pid 1 is never touched
648		mk("abc", "")	654		mk("abc", "")
649	-	got := firecrackerProcs(proc)	655	+	got := vmmProcs(proc)
650		if len(got) != 2 \|\| got[100] != "er00112233" \|\| got[101] != "er44556677" {	656		if len(got) != 2 \|\| got[100] != "er00112233" \|\| got[101] != "er44556677" {
651		t.Fatalf("%v", got)	657		t.Fatalf("%v", got)
652		}	658		}

@@ -682,3 +688,334 @@ func TestAPIErrorsSurfaceFirecrackerFault(t *testing.T) {
682			688
683		func mustPrefix(s string) netip.Prefix { return netip.MustParsePrefix(s) }	689		func mustPrefix(s string) netip.Prefix { return netip.MustParsePrefix(s) }
684		func mustAddr(s string) netip.Addr { return netip.MustParseAddr(s) }	690		func mustAddr(s string) netip.Addr { return netip.MustParseAddr(s) }
			691	+
			692	+	// C3: processes are matched by "--id <our id>" in their command line, not by the
			693	+	// program name: a renamed or versioned Firecracker binary is still found and
			694	+	// reaped, and a recycled pid is never killed on the strength of a record alone.
			695	+	func TestReapFindsRenamedVMMByID(t *testing.T) {
			696	+	e := newEnvNamed(t, "", "fc-renamed-1.17")
			697	+	d1 := e.driver(t)
			698	+	sb, _ := d1.New()
			699	+	conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
			700	+	if err != nil {
			701	+	t.Fatal(err)
			702	+	}
			703	+	runStub(t, conn)
			704	+	pid := sb.(*vmSandbox).rec.PID
			705	+	if got := cmdlineID("/proc", pid); got != sb.ID() {
			706	+	t.Fatalf("cmdlineID=%q want %q (argv[0] is %q)", got, sb.ID(), "fc-renamed-1.17")
			707	+	}
			708	+	_ = d1.lock.Release()
			709	+	_ = conn.Close()
			710	+	d2 := e.driver(t)
			711	+	if err := d2.Reap(context.Background()); err != nil {
			712	+	t.Fatal(err)
			713	+	}
			714	+	if pidAlive(pid) {
			715	+	t.Fatalf("a VMM with a renamed binary survived Reap (pid %d)", pid)
			716	+	}
			717	+	}
			718	+
			719	+	func TestVMIDFromCmdline(t *testing.T) {
			720	+	for name, tc := range map[string]struct {
			721	+	args []string
			722	+	want string
			723	+	}{
			724	+	"firecracker": {[]string{"firecracker", "--id", "er00112233", "--api-sock", "/x"}, "er00112233"},
			725	+	"jailer": {[]string{"jailer", "--id", "er00112233", "--exec-file", "/fc", "--uid", "1"}, "er00112233"},
			726	+	"renamed": {[]string{"anything", "--api-sock", "/x", "--id", "er44556677"}, "er44556677"},
			727	+	"foreign id": {[]string{"firecracker", "--id", "my-vm", "--api-sock", "/x"}, ""},
			728	+	"no marker": {[]string{"python", "--id", "er00112233"}, ""},
			729	+	"id without value": {[]string{"firecracker", "--api-sock", "/x", "--id"}, ""},
			730	+	"path-like id": {[]string{"firecracker", "--id", "er../../../x", "--api-sock", "/x"}, ""},
			731	+	} {
			732	+	if got := vmIDFromCmdline(tc.args); got != tc.want {
			733	+	t.Errorf("%s: got %q want %q", name, got, tc.want)
			734	+	}
			735	+	}
			736	+	}
			737	+
			738	+	func TestReapDoesNotTrustARecycledPid(t *testing.T) {
			739	+	e := newEnv(t, "")
			740	+	d := e.driver(t)
			741	+	// A record pointing at this very test process: its command line carries no
			742	+	// VM id, so it must not be killed.
			743	+	_ = d.store.save(record{ID: "er00112233", Slot: 0, PID: os.Getpid()})
			744	+	if err := d.Reap(context.Background()); err != nil {
			745	+	t.Fatal(err)
			746	+	}
			747	+	if !pidAlive(os.Getpid()) {
			748	+	t.Fatal("Reap killed a process on the strength of a stale record")
			749	+	}
			750	+	}
			751	+
			752	+	// C5 + C8: when the VMM cannot be killed, the TAP, the firewall policy and the
			753	+	// files are left alone (the VM may still be running), the error wraps ErrLeaked,
			754	+	// the slot stays taken, and Reap (with a working kill) finishes the cleanup and
			755	+	// frees the slot.
			756	+	func TestStopKeepsNetworkWhenTheVMMSurvives(t *testing.T) {
			757	+	e := newEnv(t, "")
			758	+	stubborn := true
			759	+	d, err := New(context.Background(), e.cfg, Options{Run: e.run, Net: e.plumb, BootTimeout: 5 * time.Second,
			760	+	kill: func(pid int, to time.Duration) bool {
			761	+	if stubborn {
			762	+	return false
			763	+	}
			764	+	return killAndWait(pid, to)
			765	+	}})
			766	+	if err != nil {
			767	+	t.Fatal(err)
			768	+	}
			769	+	defer func() { _ = d.Close() }()
			770	+	sb, _ := d.New()
			771	+	conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
			772	+	if err != nil {
			773	+	t.Fatal(err)
			774	+	}
			775	+	runStub(t, conn)
			776	+
			777	+	err = sb.Stop(context.Background())
			778	+	if !errors.Is(err, sandbox.ErrLeaked) {
			779	+	t.Fatalf("Stop must report ErrLeaked, got %v", err)
			780	+	}
			781	+	if e.tapCount() != 1 \|\| e.plumb.Live() != 1 {
			782	+	t.Fatalf("the TAP and the policy must stay while the VMM may run: taps=%d attached=%d", e.tapCount(), e.plumb.Live())
			783	+	}
			784	+	if recs, _ := d.store.list(); len(recs) != 1 {
			785	+	t.Fatalf("the state record must stay for Reap: %v", recs)
			786	+	}
			787	+	if err := sb.Stop(context.Background()); !errors.Is(err, sandbox.ErrLeaked) {
			788	+	t.Fatalf("a second Stop must keep reporting the leak, got %v", err)
			789	+	}
			790	+	if d.slots.InUse() != 1 {
			791	+	t.Fatal("the slot of a leaked sandbox stays taken")
			792	+	}
			793	+	// Other jobs keep running on the free slot; the leaked one is never reused.
			794	+	other, err := d.New()
			795	+	if err != nil {
			796	+	t.Fatal(err)
			797	+	}
			798	+	if other.(vmSandbox).slot == sb.(vmSandbox).slot {
			799	+	t.Fatal("a leaked slot was handed out again")
			800	+	}
			801	+	// Reap refuses while a sandbox is running...
			802	+	if err := d.Reap(context.Background()); !errors.Is(err, sandbox.ErrBusy) {
			803	+	t.Fatalf("Reap with a handed-out sandbox: %v", err)
			804	+	}
			805	+	_ = other.Stop(context.Background())
			806	+	// ...and finishes the job once idle.
			807	+	stubborn = false
			808	+	if err := d.Reap(context.Background()); err != nil {
			809	+	t.Fatal(err)
			810	+	}
			811	+	if left := e.leftovers(t); len(left) != 0 {
			812	+	t.Fatalf("leftovers after Reap: %v", left)
			813	+	}
			814	+	if d.slots.InUse() != 0 {
			815	+	t.Fatal("Reap must free the slot of the leaked sandbox")
			816	+	}
			817	+	}
			818	+
			819	+	// C7: Stop from another goroutine aborts a boot in progress at once, instead of
			820	+	// waiting for the boot timeout, and leaves nothing behind.
			821	+	func TestStopAbortsABootInProgress(t *testing.T) {
			822	+	e := newEnv(t, "hang")
			823	+	d, err := New(context.Background(), e.cfg, Options{Run: e.run, Net: e.plumb, BootTimeout: time.Minute})
			824	+	if err != nil {
			825	+	t.Fatal(err)
			826	+	}
			827	+	defer func() { _ = d.Close() }()
			828	+	sb, _ := d.New()
			829	+	started := make(chan error, 1)
			830	+	go func() {
			831	+	_, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
			832	+	started <- err
			833	+	}()
			834	+	time.Sleep(700 * time.Millisecond) // the fake is now waiting for the executor
			835	+	t0 := time.Now()
			836	+	if err := sb.Stop(context.Background()); err != nil {
			837	+	t.Fatal(err)
			838	+	}
			839	+	select {
			840	+	case err := <-started:
			841	+	if !errors.Is(err, sandbox.ErrAborted) {
			842	+	t.Fatalf("Start must report that Stop aborted it, got %v", err)
			843	+	}
			844	+	case <-time.After(10 * time.Second):
			845	+	t.Fatal("Start was not aborted by Stop")
			846	+	}
			847	+	if time.Since(t0) > 8*time.Second {
			848	+	t.Fatalf("Stop took %v", time.Since(t0))
			849	+	}
			850	+	if left := e.leftovers(t); len(left) != 0 {
			851	+	t.Fatalf("leftovers: %v", left)
			852	+	}
			853	+	// Stop before Start: Start must refuse.
			854	+	sb2, _ := d.New()
			855	+	_ = sb2.Stop(context.Background())
			856	+	if _, err := sb2.Start(context.Background(), sandbox.JobSpec{}); err == nil {
			857	+	t.Fatal("Start after Stop accepted")
			858	+	}
			859	+	}
			860	+
			861	+	// C16: the sandbox exposes its id, which names its files.
			862	+	func TestSandboxIDNamesItsFiles(t *testing.T) {
			863	+	e := newEnv(t, "")
			864	+	d := e.driver(t)
			865	+	sb, _ := d.New()
			866	+	if !sandbox.ValidID(sb.ID()) {
			867	+	t.Fatalf("id %q", sb.ID())
			868	+	}
			869	+	conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
			870	+	if err != nil {
			871	+	t.Fatal(err)
			872	+	}
			873	+	runStub(t, conn)
			874	+	defer func() { _ = sb.Stop(context.Background()) }()
			875	+	if _, err := os.Stat(filepath.Join(e.cfg.StateDir, "vm", sb.ID())); err != nil {
			876	+	t.Fatalf("the VM directory must be named after the sandbox id: %v", err)
			877	+	}
			878	+	if d.Isolation() != sandbox.IsolationVM \|\| d.Mode() != sandbox.VM {
			879	+	t.Fatal("isolation")
			880	+	}
			881	+	}
			882	+
			883	+	// S7: cgroup v2 limits, a new PID namespace, and the jailer's own options.
			884	+	func TestJailerArgs(t *testing.T) {
			885	+	args := jailerArgs("/usr/bin/jailer", "/usr/bin/firecracker", "er00112233", 900003, "/var/lib/er/jail",
			886	+	sandbox.JobSpec{CPUs: 4, MemoryMiB: 2048}, nil)
			887	+	line := " " + strings.Join(args, " ") + " "
			888	+	for _, want := range []string{
			889	+	" --id er00112233 ", " --exec-file /usr/bin/firecracker ", " --uid 900003 ", " --gid 900003 ",
			890	+	" --chroot-base-dir /var/lib/er/jail ", " --cgroup-version 2 ", " --new-pid-ns ",
			891	+	" --cgroup memory.max=" + fmt.Sprint((2048+vmmOverheadMiB)<<20) + " ",
			892	+	" --cgroup pids.max=64 ", " --resource-limit no-file=4096 ", " -- ",
			893	+	} {
			894	+	if !strings.Contains(line, want) {
			895	+	t.Errorf("jailer args lack %q: %s", want, line)
			896	+	}
			897	+	}
			898	+	if args[len(args)-1] != "--" {
			899	+	t.Errorf("the jailer separator must come last when there are no extra args: %v", args)
			900	+	}
			901	+	more := jailerArgs("j", "f", "er00112233", 1, "/b", sandbox.JobSpec{CPUs: 1, MemoryMiB: 512}, []string{"--x"})
			902	+	if more[len(more)-2] != "--" \|\| more[len(more)-1] != "--x" {
			903	+	t.Errorf("extra args follow the separator: %v", more)
			904	+	}
			905	+	}
			906	+
			907	+	// S7: sane rate limiters on the scratch disk and both directions of eth0, off
			908	+	// when configured to zero, and only on the writable disk.
			909	+	func TestRateLimitersInTheVMConfig(t *testing.T) {
			910	+	e := newEnv(t, "")
			911	+	d := e.driver(t)
			912	+	sb, _ := d.New()
			913	+	conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
			914	+	if err != nil {
			915	+	t.Fatal(err)
			916	+	}
			917	+	runStub(t, conn)
			918	+	defer func() { _ = sb.Stop(context.Background()) }()
			919	+	b, _ := os.ReadFile(e.log)
			920	+	log := string(b)
			921	+	for _, want := range []string{
			922	+	`"rate_limiter":{"bandwidth":{"size":419430400,"one_time_burst":419430400,"refill_time":1000},"ops":{"size":20000,"one_time_burst":20000,"refill_time":1000}}`,
			923	+	`"rx_rate_limiter":{"bandwidth":{"size":125000000,"one_time_burst":125000000,"refill_time":1000}}`,
			924	+	`"tx_rate_limiter":{"bandwidth":{"size":125000000,"one_time_burst":125000000,"refill_time":1000}}`,
			925	+	} {
			926	+	if !strings.Contains(log, want) {
			927	+	t.Errorf("API log lacks %s\n%s", want, log)
			928	+	}
			929	+	}
			930	+	if n := strings.Count(log, `"rate_limiter"`); n != 1 {
			931	+	t.Errorf("only the scratch drive is rate limited, got %d rate_limiter fields", n)
			932	+	}
			933	+	if diskLimiter(0, 0) != nil \|\| netLimiter(0) != nil {
			934	+	t.Error("zero must disable the limiters")
			935	+	}
			936	+	if l := diskLimiter(0, 100); l.Bandwidth != nil \|\| l.Ops == nil {
			937	+	t.Errorf("%+v", l)
			938	+	}
			939	+	}
			940	+
			941	+	// S8: the free-space check warns, and only warns.
			942	+	func TestNewWarnsOnLowFreeSpace(t *testing.T) {
			943	+	e := newEnv(t, "")
			944	+	var buf bytes.Buffer
			945	+	ctx := zerowrap.WithCtx(context.Background(), zerowrap.New(zerowrap.Config{Level: "debug", Format: "json", Output: &buf}))
			946	+	d, err := New(ctx, e.cfg, Options{Run: e.run, Net: e.plumb, FreeSpace: func(string) (int64, error) { return 1, nil }})
			947	+	if err != nil {
			948	+	t.Fatalf("low space must not fail New: %v", err)
			949	+	}
			950	+	defer func() { _ = d.Close() }()
			951	+	if !strings.Contains(buf.String(), "less free space") {
			952	+	t.Fatalf("no warning: %s", buf.String())
			953	+	}
			954	+	}
			955	+
			956	+	// S11: the scratch disk is created without following symlinks and chowned
			957	+	// through its descriptor.
			958	+	func TestMakeScratchRefusesSymlinksAndExistingFiles(t *testing.T) {
			959	+	dir := t.TempDir()
			960	+	target := filepath.Join(dir, "victim")
			961	+	_ = os.WriteFile(target, []byte("precious"), 0o600)
			962	+	link := filepath.Join(dir, "scratch.ext4")
			963	+	if err := os.Symlink(target, link); err != nil {
			964	+	t.Fatal(err)
			965	+	}
			966	+	run := &cmdrun.Fake{}
			967	+	if err := makeScratch(context.Background(), run, link, 8, -1); err == nil {
			968	+	t.Fatal("a planted symlink must be refused")
			969	+	}
			970	+	if b, _ := os.ReadFile(target); string(b) != "precious" {
			971	+	t.Fatalf("the symlink target was modified: %q", b)
			972	+	}
			973	+	if len(run.Calls) != 0 {
			974	+	t.Fatalf("mkfs must not run: %v", run.Lines())
			975	+	}
			976	+	ok := filepath.Join(dir, "fresh.ext4")
			977	+	if err := makeScratch(context.Background(), run, ok, 8, os.Getuid()); err != nil {
			978	+	t.Fatal(err)
			979	+	}
			980	+	if st, err := os.Stat(ok); err != nil \|\| st.Size() != 8<<20 \|\| st.Mode().Perm() != 0o600 {
			981	+	t.Fatalf("%v %v", st, err)
			982	+	}
			983	+	if err := makeScratch(context.Background(), run, ok, 8, -1); err == nil {
			984	+	t.Fatal("an existing file must be refused (O_EXCL)")
			985	+	}
			986	+	}
			987	+
			988	+	func TestOwnUsesLchown(t *testing.T) {
			989	+	dir := t.TempDir()
			990	+	l := newLayout(dir, "firecracker", "er00112233", true, os.Getuid())
			991	+	_ = os.MkdirAll(l.root, 0o700)
			992	+	target := filepath.Join(dir, "victim")
			993	+	_ = os.WriteFile(target, nil, 0o600)
			994	+	if err := os.Symlink(target, l.host("link")); err != nil {
			995	+	t.Fatal(err)
			996	+	}
			997	+	if err := l.own("link"); err != nil {
			998	+	t.Fatalf("Lchown on a link must act on the link itself: %v", err)
			999	+	}
			1000	+	if err := l.own("missing"); err == nil {
			1001	+	t.Fatal("a missing file is an error")
			1002	+	}
			1003	+	if u := newLayout(dir, "f", "er00112233", false, 1); u.own("anything") != nil {
			1004	+	t.Fatal("no-op without jailer")
			1005	+	}
			1006	+	}
			1007	+
			1008	+	func TestRemoveCgroup(t *testing.T) {
			1009	+	root := t.TempDir()
			1010	+	_ = os.MkdirAll(filepath.Join(root, "firecracker", "er00112233"), 0o755)
			1011	+	if err := removeCgroup(root, "firecracker", "er00112233"); err != nil {
			1012	+	t.Fatal(err)
			1013	+	}
			1014	+	if err := removeCgroup(root, "firecracker", "er00112233"); err != nil {
			1015	+	t.Fatalf("a missing cgroup is fine: %v", err)
			1016	+	}
			1017	+	_ = os.MkdirAll(filepath.Join(root, "firecracker", "er44556677", "child"), 0o755)
			1018	+	if err := removeCgroup(root, "firecracker", "er44556677"); err == nil {
			1019	+	t.Fatal("a non-empty cgroup cannot be removed and must be reported")
			1020	+	}
			1021	+	}