feat(sandbox): host-wide lock, ErrLeaked/ErrBusy, boot gate, ID and Isolation, free-space warning, new limits in config

commit: 1905b3977824ac48adbbf3a0fb0b8517677dd7f3
author: brice · 10d ago · 2026-10-12 16:10 UTC
parent: 8784b11
signature: GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email

7 files · +447 -12

M internal/sandbox/config.go +60 -0


@@ -29,6 +29,13 @@ const (
29	29		EnvJailer = "ERMITE_RUNNER_JAILER"
30	30		EnvJailerMode = "ERMITE_RUNNER_JAILER_MODE"
31	31		EnvKernel = "ERMITE_RUNNER_KERNEL"
	32	+
	33	+	EnvJobPids = "ERMITE_RUNNER_JOB_PIDS"
	34	+	EnvDenyPrivileged = "ERMITE_RUNNER_DENY_PRIVILEGED"
	35	+	EnvLockFile = "ERMITE_RUNNER_LOCK_FILE"
	36	+	EnvJobDiskBW = "ERMITE_RUNNER_JOB_DISK_BANDWIDTH"
	37	+	EnvJobDiskIOPS = "ERMITE_RUNNER_JOB_DISK_IOPS"
	38	+	EnvJobNetBandwidth = "ERMITE_RUNNER_JOB_NET_BANDWIDTH"
32	39		)
33	40
34	41		// Defaults.

@@ -40,6 +47,17 @@ const (
40	47		DefaultCPUs = 2
41	48		DefaultMemoryMiB = 4096
42	49		DefaultDiskMiB = 64 * 1024
	50	+	DefaultJobPids = 32768
	51	+	// DefaultLockFile is the single-instance lock of the host: reaping removes
	52	+	// everything the runner labels as its own, so two runners (even with
	53	+	// different state directories, or one in a container) must never coexist.
	54	+	DefaultLockFile = "/run/ermite-runner.lock"
	55	+	// Rate limits of a micro-VM, generous so that they only stop a runaway job
	56	+	// from starving the host: 400 MiB/s and 20000 IOPS per disk, 1 Gbit/s per
	57	+	// network direction. Zero disables a limit.
	58	+	DefaultDiskBandwidthMiB = 400
	59	+	DefaultDiskIOPS = 20000
	60	+	DefaultNetMbit = 1000
43	61		)
44	62
45	63		// Config is the sandbox part of the runner configuration.

@@ -81,6 +99,17 @@ type Config struct {
81	99		// DNSPort is the port of the DNS forwarder; 53 in production. Tests that
82	100		// cannot bind it use another port with no guest.
83	101		DNSPort int
	102	+
	103	+	// JobPids is the process limit of a job (container PidsLimit, VM pids.max
	104	+	// is derived from the vCPU count and is not affected).
	105	+	JobPids int
	106	+	// DenyPrivileged refuses the container mode when it would have to run
	107	+	// privileged (no sysbox).
	108	+	DenyPrivileged bool
	109	+	// LockFile is the host-wide single-instance lock.
	110	+	LockFile string
	111	+	// Rate limits of micro-VMs; zero disables.
	112	+	DiskBandwidthMiB, DiskIOPS, NetMbit int
84	113		}
85	114
86	115		// LoadConfig reads the sandbox settings from getenv (os.Getenv in production).

@@ -95,6 +124,12 @@ func LoadConfig(getenv func(string) string) (Config, error) {
95	124		UIDBase: DefaultUIDBase,
96	125		KVMPath: "/dev/kvm",
97	126		DNSPort: 53,
	127	+	JobPids: DefaultJobPids,
	128	+	LockFile: DefaultLockFile,
	129	+
	130	+	DiskBandwidthMiB: DefaultDiskBandwidthMiB,
	131	+	DiskIOPS: DefaultDiskIOPS,
	132	+	NetMbit: DefaultNetMbit,
98	133		}
99	134		var err error
100	135		if v := getenv(EnvIsolation); v != "" {

@@ -144,6 +179,31 @@ func LoadConfig(getenv func(string) string) (Config, error) {
144	179		if 1<<(30-c.Network.Bits()) < c.Capacity {
145	180		return c, fmt.Errorf("%s: %s has fewer /30 subnets than the capacity %d", EnvNetwork, c.Network, c.Capacity)
146	181		}
	182	+	if c.JobPids, err = intEnv(getenv, EnvJobPids, c.JobPids, 64, 4_194_304); err != nil {
	183	+	return c, err
	184	+	}
	185	+	switch strings.ToLower(strings.TrimSpace(getenv(EnvDenyPrivileged))) {
	186	+	case "", "0", "false", "no":
	187	+	case "1", "true", "yes":
	188	+	c.DenyPrivileged = true
	189	+	default:
	190	+	return c, fmt.Errorf("%s: %q is not 0 or 1", EnvDenyPrivileged, getenv(EnvDenyPrivileged))
	191	+	}
	192	+	if v := getenv(EnvLockFile); v != "" {
	193	+	if !filepath.IsAbs(v) {
	194	+	return c, fmt.Errorf("%s: %q must be an absolute path", EnvLockFile, v)
	195	+	}
	196	+	c.LockFile = v
	197	+	}
	198	+	if c.DiskBandwidthMiB, err = intEnv(getenv, EnvJobDiskBW, c.DiskBandwidthMiB, 0, 1_000_000); err != nil {
	199	+	return c, err
	200	+	}
	201	+	if c.DiskIOPS, err = intEnv(getenv, EnvJobDiskIOPS, c.DiskIOPS, 0, 100_000_000); err != nil {
	202	+	return c, err
	203	+	}
	204	+	if c.NetMbit, err = intEnv(getenv, EnvJobNetBandwidth, c.NetMbit, 0, 1_000_000); err != nil {
	205	+	return c, err
	206	+	}
147	207		c.Firecracker = getenv(EnvFirecracker)
148	208		c.Jailer = getenv(EnvJailer)
149	209		c.Kernel = getenv(EnvKernel)


@@ -29,6 +29,13 @@ const (
29	EnvJailer = "ERMITE_RUNNER_JAILER"	29		EnvJailer = "ERMITE_RUNNER_JAILER"
30	EnvJailerMode = "ERMITE_RUNNER_JAILER_MODE"	30		EnvJailerMode = "ERMITE_RUNNER_JAILER_MODE"
31	EnvKernel = "ERMITE_RUNNER_KERNEL"	31		EnvKernel = "ERMITE_RUNNER_KERNEL"
		32	+
		33	+	EnvJobPids = "ERMITE_RUNNER_JOB_PIDS"
		34	+	EnvDenyPrivileged = "ERMITE_RUNNER_DENY_PRIVILEGED"
		35	+	EnvLockFile = "ERMITE_RUNNER_LOCK_FILE"
		36	+	EnvJobDiskBW = "ERMITE_RUNNER_JOB_DISK_BANDWIDTH"
		37	+	EnvJobDiskIOPS = "ERMITE_RUNNER_JOB_DISK_IOPS"
		38	+	EnvJobNetBandwidth = "ERMITE_RUNNER_JOB_NET_BANDWIDTH"
32	)	39		)
33		40
34	// Defaults.	41		// Defaults.

@@ -40,6 +47,17 @@ const (
40	DefaultCPUs = 2	47		DefaultCPUs = 2
41	DefaultMemoryMiB = 4096	48		DefaultMemoryMiB = 4096
42	DefaultDiskMiB = 64 * 1024	49		DefaultDiskMiB = 64 * 1024
		50	+	DefaultJobPids = 32768
		51	+	// DefaultLockFile is the single-instance lock of the host: reaping removes
		52	+	// everything the runner labels as its own, so two runners (even with
		53	+	// different state directories, or one in a container) must never coexist.
		54	+	DefaultLockFile = "/run/ermite-runner.lock"
		55	+	// Rate limits of a micro-VM, generous so that they only stop a runaway job
		56	+	// from starving the host: 400 MiB/s and 20000 IOPS per disk, 1 Gbit/s per
		57	+	// network direction. Zero disables a limit.
		58	+	DefaultDiskBandwidthMiB = 400
		59	+	DefaultDiskIOPS = 20000
		60	+	DefaultNetMbit = 1000
43	)	61		)
44		62
45	// Config is the sandbox part of the runner configuration.	63		// Config is the sandbox part of the runner configuration.

@@ -81,6 +99,17 @@ type Config struct {
81	// DNSPort is the port of the DNS forwarder; 53 in production. Tests that	99		// DNSPort is the port of the DNS forwarder; 53 in production. Tests that
82	// cannot bind it use another port with no guest.	100		// cannot bind it use another port with no guest.
83	DNSPort int	101		DNSPort int
		102	+
		103	+	// JobPids is the process limit of a job (container PidsLimit, VM pids.max
		104	+	// is derived from the vCPU count and is not affected).
		105	+	JobPids int
		106	+	// DenyPrivileged refuses the container mode when it would have to run
		107	+	// privileged (no sysbox).
		108	+	DenyPrivileged bool
		109	+	// LockFile is the host-wide single-instance lock.
		110	+	LockFile string
		111	+	// Rate limits of micro-VMs; zero disables.
		112	+	DiskBandwidthMiB, DiskIOPS, NetMbit int
84	}	113		}
85		114
86	// LoadConfig reads the sandbox settings from getenv (os.Getenv in production).	115		// LoadConfig reads the sandbox settings from getenv (os.Getenv in production).

@@ -95,6 +124,12 @@ func LoadConfig(getenv func(string) string) (Config, error) {
95	UIDBase: DefaultUIDBase,	124		UIDBase: DefaultUIDBase,
96	KVMPath: "/dev/kvm",	125		KVMPath: "/dev/kvm",
97	DNSPort: 53,	126		DNSPort: 53,
		127	+	JobPids: DefaultJobPids,
		128	+	LockFile: DefaultLockFile,
		129	+
		130	+	DiskBandwidthMiB: DefaultDiskBandwidthMiB,
		131	+	DiskIOPS: DefaultDiskIOPS,
		132	+	NetMbit: DefaultNetMbit,
98	}	133		}
99	var err error	134		var err error
100	if v := getenv(EnvIsolation); v != "" {	135		if v := getenv(EnvIsolation); v != "" {

@@ -144,6 +179,31 @@ func LoadConfig(getenv func(string) string) (Config, error) {
144	if 1<<(30-c.Network.Bits()) < c.Capacity {	179		if 1<<(30-c.Network.Bits()) < c.Capacity {
145	return c, fmt.Errorf("%s: %s has fewer /30 subnets than the capacity %d", EnvNetwork, c.Network, c.Capacity)	180		return c, fmt.Errorf("%s: %s has fewer /30 subnets than the capacity %d", EnvNetwork, c.Network, c.Capacity)
146	}	181		}
		182	+	if c.JobPids, err = intEnv(getenv, EnvJobPids, c.JobPids, 64, 4_194_304); err != nil {
		183	+	return c, err
		184	+	}
		185	+	switch strings.ToLower(strings.TrimSpace(getenv(EnvDenyPrivileged))) {
		186	+	case "", "0", "false", "no":
		187	+	case "1", "true", "yes":
		188	+	c.DenyPrivileged = true
		189	+	default:
		190	+	return c, fmt.Errorf("%s: %q is not 0 or 1", EnvDenyPrivileged, getenv(EnvDenyPrivileged))
		191	+	}
		192	+	if v := getenv(EnvLockFile); v != "" {
		193	+	if !filepath.IsAbs(v) {
		194	+	return c, fmt.Errorf("%s: %q must be an absolute path", EnvLockFile, v)
		195	+	}
		196	+	c.LockFile = v
		197	+	}
		198	+	if c.DiskBandwidthMiB, err = intEnv(getenv, EnvJobDiskBW, c.DiskBandwidthMiB, 0, 1_000_000); err != nil {
		199	+	return c, err
		200	+	}
		201	+	if c.DiskIOPS, err = intEnv(getenv, EnvJobDiskIOPS, c.DiskIOPS, 0, 100_000_000); err != nil {
		202	+	return c, err
		203	+	}
		204	+	if c.NetMbit, err = intEnv(getenv, EnvJobNetBandwidth, c.NetMbit, 0, 1_000_000); err != nil {
		205	+	return c, err
		206	+	}
147	c.Firecracker = getenv(EnvFirecracker)	207		c.Firecracker = getenv(EnvFirecracker)
148	c.Jailer = getenv(EnvJailer)	208		c.Jailer = getenv(EnvJailer)
149	c.Kernel = getenv(EnvKernel)	209		c.Kernel = getenv(EnvKernel)

A internal/sandbox/core_test.go +161 -0

@@ -0,0 +1,161 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package sandbox
	4	+
	5	+	import (
	6	+	"bytes"
	7	+	"context"
	8	+	"errors"
	9	+	"path/filepath"
	10	+	"strings"
	11	+	"testing"
	12	+	"time"
	13	+	)
	14	+
	15	+	// S3: one runner per host, whatever its state directory.
	16	+	func TestHostLockIsExclusiveAndReleasable(t *testing.T) {
	17	+	p := filepath.Join(t.TempDir(), "run", "ermite-runner.lock")
	18	+	a, err := AcquireHostLock(p)
	19	+	if err != nil {
	20	+	t.Fatal(err)
	21	+	}
	22	+	if _, err := AcquireHostLock(p); err == nil \|\| !strings.Contains(err.Error(), "another ermite-runner is running on this host") {
	23	+	t.Fatalf("a second runner must be refused with a clear message, got %v", err)
	24	+	}
	25	+	if err := a.Release(); err != nil {
	26	+	t.Fatal(err)
	27	+	}
	28	+	if err := a.Release(); err != nil {
	29	+	t.Fatalf("double release: %v", err)
	30	+	}
	31	+	b, err := AcquireHostLock(p)
	32	+	if err != nil {
	33	+	t.Fatalf("the lock must be free after Release: %v", err)
	34	+	}
	35	+	_ = b.Release()
	36	+	var nilLock *HostLock
	37	+	if err := nilLock.Release(); err != nil {
	38	+	t.Fatal(err)
	39	+	}
	40	+	}
	41	+
	42	+	func TestHostLockErrorNamesTheOverride(t *testing.T) {
	43	+	_, err := AcquireHostLock("/proc/nope/ermite.lock")
	44	+	if err == nil \|\| !strings.Contains(err.Error(), EnvLockFile) {
	45	+	t.Fatalf("got %v", err)
	46	+	}
	47	+	}
	48	+
	49	+	func TestConfigNewSettings(t *testing.T) {
	50	+	c, err := LoadConfig(env(nil))
	51	+	if err != nil {
	52	+	t.Fatal(err)
	53	+	}
	54	+	if c.JobPids != 32768 \|\| c.DenyPrivileged \|\| c.LockFile != "/run/ermite-runner.lock" \|\|
	55	+	c.DiskBandwidthMiB != 400 \|\| c.DiskIOPS != 20000 \|\| c.NetMbit != 1000 {
	56	+	t.Fatalf("%+v", c)
	57	+	}
	58	+	c, err = LoadConfig(env(map[string]string{
	59	+	EnvJobPids: "1000", EnvDenyPrivileged: "1", EnvLockFile: "/tmp/x.lock",
	60	+	EnvJobDiskBW: "0", EnvJobDiskIOPS: "5000", EnvJobNetBandwidth: "100",
	61	+	}))
	62	+	if err != nil {
	63	+	t.Fatal(err)
	64	+	}
	65	+	if c.JobPids != 1000 \|\| !c.DenyPrivileged \|\| c.LockFile != "/tmp/x.lock" \|\| c.DiskBandwidthMiB != 0 \|\| c.DiskIOPS != 5000 \|\| c.NetMbit != 100 {
	66	+	t.Fatalf("%+v", c)
	67	+	}
	68	+	for name, m := range map[string]map[string]string{
	69	+	"pids low": {EnvJobPids: "3"},
	70	+	"deny": {EnvDenyPrivileged: "maybe"},
	71	+	"lock rel": {EnvLockFile: "run/x.lock"},
	72	+	"bw negative": {EnvJobDiskBW: "-1"},
	73	+	"net not int": {EnvJobNetBandwidth: "fast"},
	74	+	} {
	75	+	if _, err := LoadConfig(env(m)); err == nil {
	76	+	t.Errorf("%s accepted", name)
	77	+	}
	78	+	}
	79	+	}
	80	+
	81	+	// C8: leaked slots stay taken until ReleaseLeaked; Busy ignores them.
	82	+	func TestSlotsLeakedLifecycle(t *testing.T) {
	83	+	s := NewSlots(3)
	84	+	a, _ := s.Acquire()
	85	+	b, _ := s.Acquire()
	86	+	s.MarkLeaked(a)
	87	+	if s.InUse() != 2 \|\| s.Busy() != 1 {
	88	+	t.Fatalf("InUse=%d Busy=%d", s.InUse(), s.Busy())
	89	+	}
	90	+	c, _ := s.Acquire()
	91	+	if c == a \|\| c == b {
	92	+	t.Fatalf("a leaked slot must not be handed out again: %d", c)
	93	+	}
	94	+	s.Release(a) // a normal release also clears the leak flag
	95	+	if s.InUse() != 2 {
	96	+	t.Fatal(s.InUse())
	97	+	}
	98	+	s.MarkLeaked(b)
	99	+	s.ReleaseLeaked()
	100	+	if s.InUse() != 1 \|\| s.Busy() != 1 {
	101	+	t.Fatalf("after ReleaseLeaked InUse=%d Busy=%d", s.InUse(), s.Busy())
	102	+	}
	103	+	}
	104	+
	105	+	func TestErrLeakedIsMatchable(t *testing.T) {
	106	+	err := errors.Join(errors.New("remove container: busy"), ErrLeaked)
	107	+	if !errors.Is(err, ErrLeaked) {
	108	+	t.Fatal("errors.Is must see ErrLeaked through a join")
	109	+	}
	110	+	}
	111	+
	112	+	// C7: Abort cancels a boot in progress and forbids a later one.
	113	+	func TestBootGate(t *testing.T) {
	114	+	var g BootGate
	115	+	ctx, err := g.Begin(context.Background())
	116	+	if err != nil {
	117	+	t.Fatal(err)
	118	+	}
	119	+	go func() { time.Sleep(20 * time.Millisecond); g.Abort() }()
	120	+	select {
	121	+	case <-ctx.Done():
	122	+	if !errors.Is(context.Cause(ctx), ErrAborted) {
	123	+	t.Fatalf("cause %v", context.Cause(ctx))
	124	+	}
	125	+	case <-time.After(2 * time.Second):
	126	+	t.Fatal("Abort did not cancel the boot context")
	127	+	}
	128	+	g.End()
	129	+	if _, err := g.Begin(context.Background()); !errors.Is(err, ErrAborted) {
	130	+	t.Fatalf("Begin after Abort: %v", err)
	131	+	}
	132	+	// Parent cancellation still works, and End releases the context.
	133	+	var g2 BootGate
	134	+	parent, cancel := context.WithCancel(context.Background())
	135	+	c2, _ := g2.Begin(parent)
	136	+	cancel()
	137	+	<-c2.Done()
	138	+	g2.End()
	139	+	g2.End()
	140	+	}
	141	+
	142	+	func TestWarnIfLowSpace(t *testing.T) {
	143	+	var buf bytes.Buffer
	144	+	ctx := logCtx(&buf)
	145	+	WarnIfLowSpace(ctx, "/x", 2, 641024, func(string) (int64, error) { return 10 1024, nil })
	146	+	if !strings.Contains(buf.String(), "less free space") \|\| !strings.Contains(buf.String(), `"worst_case_mib":131072`) {
	147	+	t.Fatalf("expected a warning, got %q", buf.String())
	148	+	}
	149	+	buf.Reset()
	150	+	WarnIfLowSpace(ctx, "/x", 2, 1024, func(string) (int64, error) { return 10 * 1024, nil })
	151	+	if buf.Len() != 0 {
	152	+	t.Fatalf("no warning expected: %q", buf.String())
	153	+	}
	154	+	WarnIfLowSpace(ctx, "/x", 1, 1, func(string) (int64, error) { return 0, errors.New("statfs failed") })
	155	+	if !strings.Contains(buf.String(), "cannot check") {
	156	+	t.Fatalf("a statfs error must be logged: %q", buf.String())
	157	+	}
	158	+	if n, err := FreeMiB(t.TempDir()); err != nil \|\| n <= 0 {
	159	+	t.Fatalf("FreeMiB: %d %v", n, err)
	160	+	}
	161	+	}

@@ -0,0 +1,161 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package sandbox
	4	+
	5	+	import (
	6	+	"bytes"
	7	+	"context"
	8	+	"errors"
	9	+	"path/filepath"
	10	+	"strings"
	11	+	"testing"
	12	+	"time"
	13	+	)
	14	+
	15	+	// S3: one runner per host, whatever its state directory.
	16	+	func TestHostLockIsExclusiveAndReleasable(t *testing.T) {
	17	+	p := filepath.Join(t.TempDir(), "run", "ermite-runner.lock")
	18	+	a, err := AcquireHostLock(p)
	19	+	if err != nil {
	20	+	t.Fatal(err)
	21	+	}
	22	+	if _, err := AcquireHostLock(p); err == nil \|\| !strings.Contains(err.Error(), "another ermite-runner is running on this host") {
	23	+	t.Fatalf("a second runner must be refused with a clear message, got %v", err)
	24	+	}
	25	+	if err := a.Release(); err != nil {
	26	+	t.Fatal(err)
	27	+	}
	28	+	if err := a.Release(); err != nil {
	29	+	t.Fatalf("double release: %v", err)
	30	+	}
	31	+	b, err := AcquireHostLock(p)
	32	+	if err != nil {
	33	+	t.Fatalf("the lock must be free after Release: %v", err)
	34	+	}
	35	+	_ = b.Release()
	36	+	var nilLock *HostLock
	37	+	if err := nilLock.Release(); err != nil {
	38	+	t.Fatal(err)
	39	+	}
	40	+	}
	41	+
	42	+	func TestHostLockErrorNamesTheOverride(t *testing.T) {
	43	+	_, err := AcquireHostLock("/proc/nope/ermite.lock")
	44	+	if err == nil \|\| !strings.Contains(err.Error(), EnvLockFile) {
	45	+	t.Fatalf("got %v", err)
	46	+	}
	47	+	}
	48	+
	49	+	func TestConfigNewSettings(t *testing.T) {
	50	+	c, err := LoadConfig(env(nil))
	51	+	if err != nil {
	52	+	t.Fatal(err)
	53	+	}
	54	+	if c.JobPids != 32768 \|\| c.DenyPrivileged \|\| c.LockFile != "/run/ermite-runner.lock" \|\|
	55	+	c.DiskBandwidthMiB != 400 \|\| c.DiskIOPS != 20000 \|\| c.NetMbit != 1000 {
	56	+	t.Fatalf("%+v", c)
	57	+	}
	58	+	c, err = LoadConfig(env(map[string]string{
	59	+	EnvJobPids: "1000", EnvDenyPrivileged: "1", EnvLockFile: "/tmp/x.lock",
	60	+	EnvJobDiskBW: "0", EnvJobDiskIOPS: "5000", EnvJobNetBandwidth: "100",
	61	+	}))
	62	+	if err != nil {
	63	+	t.Fatal(err)
	64	+	}
	65	+	if c.JobPids != 1000 \|\| !c.DenyPrivileged \|\| c.LockFile != "/tmp/x.lock" \|\| c.DiskBandwidthMiB != 0 \|\| c.DiskIOPS != 5000 \|\| c.NetMbit != 100 {
	66	+	t.Fatalf("%+v", c)
	67	+	}
	68	+	for name, m := range map[string]map[string]string{
	69	+	"pids low": {EnvJobPids: "3"},
	70	+	"deny": {EnvDenyPrivileged: "maybe"},
	71	+	"lock rel": {EnvLockFile: "run/x.lock"},
	72	+	"bw negative": {EnvJobDiskBW: "-1"},
	73	+	"net not int": {EnvJobNetBandwidth: "fast"},
	74	+	} {
	75	+	if _, err := LoadConfig(env(m)); err == nil {
	76	+	t.Errorf("%s accepted", name)
	77	+	}
	78	+	}
	79	+	}
	80	+
	81	+	// C8: leaked slots stay taken until ReleaseLeaked; Busy ignores them.
	82	+	func TestSlotsLeakedLifecycle(t *testing.T) {
	83	+	s := NewSlots(3)
	84	+	a, _ := s.Acquire()
	85	+	b, _ := s.Acquire()
	86	+	s.MarkLeaked(a)
	87	+	if s.InUse() != 2 \|\| s.Busy() != 1 {
	88	+	t.Fatalf("InUse=%d Busy=%d", s.InUse(), s.Busy())
	89	+	}
	90	+	c, _ := s.Acquire()
	91	+	if c == a \|\| c == b {
	92	+	t.Fatalf("a leaked slot must not be handed out again: %d", c)
	93	+	}
	94	+	s.Release(a) // a normal release also clears the leak flag
	95	+	if s.InUse() != 2 {
	96	+	t.Fatal(s.InUse())
	97	+	}
	98	+	s.MarkLeaked(b)
	99	+	s.ReleaseLeaked()
	100	+	if s.InUse() != 1 \|\| s.Busy() != 1 {
	101	+	t.Fatalf("after ReleaseLeaked InUse=%d Busy=%d", s.InUse(), s.Busy())
	102	+	}
	103	+	}
	104	+
	105	+	func TestErrLeakedIsMatchable(t *testing.T) {
	106	+	err := errors.Join(errors.New("remove container: busy"), ErrLeaked)
	107	+	if !errors.Is(err, ErrLeaked) {
	108	+	t.Fatal("errors.Is must see ErrLeaked through a join")
	109	+	}
	110	+	}
	111	+
	112	+	// C7: Abort cancels a boot in progress and forbids a later one.
	113	+	func TestBootGate(t *testing.T) {
	114	+	var g BootGate
	115	+	ctx, err := g.Begin(context.Background())
	116	+	if err != nil {
	117	+	t.Fatal(err)
	118	+	}
	119	+	go func() { time.Sleep(20 * time.Millisecond); g.Abort() }()
	120	+	select {
	121	+	case <-ctx.Done():
	122	+	if !errors.Is(context.Cause(ctx), ErrAborted) {
	123	+	t.Fatalf("cause %v", context.Cause(ctx))
	124	+	}
	125	+	case <-time.After(2 * time.Second):
	126	+	t.Fatal("Abort did not cancel the boot context")
	127	+	}
	128	+	g.End()
	129	+	if _, err := g.Begin(context.Background()); !errors.Is(err, ErrAborted) {
	130	+	t.Fatalf("Begin after Abort: %v", err)
	131	+	}
	132	+	// Parent cancellation still works, and End releases the context.
	133	+	var g2 BootGate
	134	+	parent, cancel := context.WithCancel(context.Background())
	135	+	c2, _ := g2.Begin(parent)
	136	+	cancel()
	137	+	<-c2.Done()
	138	+	g2.End()
	139	+	g2.End()
	140	+	}
	141	+
	142	+	func TestWarnIfLowSpace(t *testing.T) {
	143	+	var buf bytes.Buffer
	144	+	ctx := logCtx(&buf)
	145	+	WarnIfLowSpace(ctx, "/x", 2, 641024, func(string) (int64, error) { return 10 1024, nil })
	146	+	if !strings.Contains(buf.String(), "less free space") \|\| !strings.Contains(buf.String(), `"worst_case_mib":131072`) {
	147	+	t.Fatalf("expected a warning, got %q", buf.String())
	148	+	}
	149	+	buf.Reset()
	150	+	WarnIfLowSpace(ctx, "/x", 2, 1024, func(string) (int64, error) { return 10 * 1024, nil })
	151	+	if buf.Len() != 0 {
	152	+	t.Fatalf("no warning expected: %q", buf.String())
	153	+	}
	154	+	WarnIfLowSpace(ctx, "/x", 1, 1, func(string) (int64, error) { return 0, errors.New("statfs failed") })
	155	+	if !strings.Contains(buf.String(), "cannot check") {
	156	+	t.Fatalf("a statfs error must be logged: %q", buf.String())
	157	+	}
	158	+	if n, err := FreeMiB(t.TempDir()); err != nil \|\| n <= 0 {
	159	+	t.Fatalf("FreeMiB: %d %v", n, err)
	160	+	}
	161	+	}

A internal/sandbox/gate.go +55 -0

@@ -0,0 +1,55 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package sandbox
	4	+
	5	+	import (
	6	+	"context"
	7	+	"errors"
	8	+	"sync"
	9	+	)
	10	+
	11	+	// BootGate lets Stop abort a Start in progress. Start holds the sandbox's own
	12	+	// mutex for the whole boot (up to the boot timeout), so Stop could otherwise
	13	+	// only wait for it; with a gate, Stop first calls Abort, which cancels the
	14	+	// context Start runs its boot with, and then takes the mutex.
	15	+	type BootGate struct {
	16	+	mu sync.Mutex
	17	+	cancel context.CancelFunc
	18	+	aborted bool
	19	+	}
	20	+
	21	+	// ErrAborted is the cause of a boot cancelled by Stop.
	22	+	var ErrAborted = errors.New("sandbox: start aborted by Stop")
	23	+
	24	+	// Begin returns the context to boot with: parent, cancelled by Abort. It fails
	25	+	// when Abort was already called (Stop before Start).
	26	+	func (g *BootGate) Begin(parent context.Context) (context.Context, error) {
	27	+	g.mu.Lock()
	28	+	defer g.mu.Unlock()
	29	+	if g.aborted {
	30	+	return nil, ErrAborted
	31	+	}
	32	+	ctx, cancel := context.WithCancelCause(parent)
	33	+	g.cancel = func() { cancel(ErrAborted) }
	34	+	return ctx, nil
	35	+	}
	36	+
	37	+	// End must be called when the boot finished, successfully or not.
	38	+	func (g *BootGate) End() {
	39	+	g.mu.Lock()
	40	+	defer g.mu.Unlock()
	41	+	if g.cancel != nil {
	42	+	g.cancel()
	43	+	g.cancel = nil
	44	+	}
	45	+	}
	46	+
	47	+	// Abort cancels a boot in progress and forbids a later Begin. It never blocks.
	48	+	func (g *BootGate) Abort() {
	49	+	g.mu.Lock()
	50	+	defer g.mu.Unlock()
	51	+	g.aborted = true
	52	+	if g.cancel != nil {
	53	+	g.cancel()
	54	+	}
	55	+	}

@@ -0,0 +1,55 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package sandbox
	4	+
	5	+	import (
	6	+	"context"
	7	+	"errors"
	8	+	"sync"
	9	+	)
	10	+
	11	+	// BootGate lets Stop abort a Start in progress. Start holds the sandbox's own
	12	+	// mutex for the whole boot (up to the boot timeout), so Stop could otherwise
	13	+	// only wait for it; with a gate, Stop first calls Abort, which cancels the
	14	+	// context Start runs its boot with, and then takes the mutex.
	15	+	type BootGate struct {
	16	+	mu sync.Mutex
	17	+	cancel context.CancelFunc
	18	+	aborted bool
	19	+	}
	20	+
	21	+	// ErrAborted is the cause of a boot cancelled by Stop.
	22	+	var ErrAborted = errors.New("sandbox: start aborted by Stop")
	23	+
	24	+	// Begin returns the context to boot with: parent, cancelled by Abort. It fails
	25	+	// when Abort was already called (Stop before Start).
	26	+	func (g *BootGate) Begin(parent context.Context) (context.Context, error) {
	27	+	g.mu.Lock()
	28	+	defer g.mu.Unlock()
	29	+	if g.aborted {
	30	+	return nil, ErrAborted
	31	+	}
	32	+	ctx, cancel := context.WithCancelCause(parent)
	33	+	g.cancel = func() { cancel(ErrAborted) }
	34	+	return ctx, nil
	35	+	}
	36	+
	37	+	// End must be called when the boot finished, successfully or not.
	38	+	func (g *BootGate) End() {
	39	+	g.mu.Lock()
	40	+	defer g.mu.Unlock()
	41	+	if g.cancel != nil {
	42	+	g.cancel()
	43	+	g.cancel = nil
	44	+	}
	45	+	}
	46	+
	47	+	// Abort cancels a boot in progress and forbids a later Begin. It never blocks.
	48	+	func (g *BootGate) Abort() {
	49	+	g.mu.Lock()
	50	+	defer g.mu.Unlock()
	51	+	g.aborted = true
	52	+	if g.cancel != nil {
	53	+	g.cancel()
	54	+	}
	55	+	}

A internal/sandbox/lock.go +49 -0

@@ -0,0 +1,49 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package sandbox
	4	+
	5	+	import (
	6	+	"fmt"
	7	+	"os"
	8	+	"path/filepath"
	9	+
	10	+	"golang.org/x/sys/unix"
	11	+	)
	12	+
	13	+	// HostLock is the host-wide single-instance lock. Reaping removes every process,
	14	+	// device, firewall rule and Docker object labelled as the runner's own, so it is
	15	+	// only safe when no other runner of the same host is alive: the lock lives at a
	16	+	// fixed path of the host (DefaultLockFile), not in a state directory, which two
	17	+	// runners could choose differently, and it is taken by both isolation modes.
	18	+	type HostLock struct{ f *os.File }
	19	+
	20	+	// AcquireHostLock takes the lock at path (the default when empty) without
	21	+	// waiting. The kernel releases it when the process dies, so a crash never leaves
	22	+	// it stuck.
	23	+	func AcquireHostLock(path string) (*HostLock, error) {
	24	+	if path == "" {
	25	+	path = DefaultLockFile
	26	+	}
	27	+	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
	28	+	return nil, fmt.Errorf("host lock %s: %w (set %s for another path)", path, err, EnvLockFile)
	29	+	}
	30	+	f, err := os.OpenFile(path, os.O_CREATE\|os.O_RDWR, 0o600)
	31	+	if err != nil {
	32	+	return nil, fmt.Errorf("host lock %s: %w (set %s for another path)", path, err, EnvLockFile)
	33	+	}
	34	+	if err := unix.Flock(int(f.Fd()), unix.LOCK_EX\|unix.LOCK_NB); err != nil {
	35	+	_ = f.Close()
	36	+	return nil, fmt.Errorf("another ermite-runner is running on this host (lock %s): only one runner per host is supported, because reaping removes everything the runner owns: %w", path, err)
	37	+	}
	38	+	return &HostLock{f: f}, nil
	39	+	}
	40	+
	41	+	// Release drops the lock; releasing twice is fine.
	42	+	func (l *HostLock) Release() error {
	43	+	if l == nil \|\| l.f == nil {
	44	+	return nil
	45	+	}
	46	+	err := l.f.Close()
	47	+	l.f = nil
	48	+	return err
	49	+	}

@@ -0,0 +1,49 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package sandbox
	4	+
	5	+	import (
	6	+	"fmt"
	7	+	"os"
	8	+	"path/filepath"
	9	+
	10	+	"golang.org/x/sys/unix"
	11	+	)
	12	+
	13	+	// HostLock is the host-wide single-instance lock. Reaping removes every process,
	14	+	// device, firewall rule and Docker object labelled as the runner's own, so it is
	15	+	// only safe when no other runner of the same host is alive: the lock lives at a
	16	+	// fixed path of the host (DefaultLockFile), not in a state directory, which two
	17	+	// runners could choose differently, and it is taken by both isolation modes.
	18	+	type HostLock struct{ f *os.File }
	19	+
	20	+	// AcquireHostLock takes the lock at path (the default when empty) without
	21	+	// waiting. The kernel releases it when the process dies, so a crash never leaves
	22	+	// it stuck.
	23	+	func AcquireHostLock(path string) (*HostLock, error) {
	24	+	if path == "" {
	25	+	path = DefaultLockFile
	26	+	}
	27	+	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
	28	+	return nil, fmt.Errorf("host lock %s: %w (set %s for another path)", path, err, EnvLockFile)
	29	+	}
	30	+	f, err := os.OpenFile(path, os.O_CREATE\|os.O_RDWR, 0o600)
	31	+	if err != nil {
	32	+	return nil, fmt.Errorf("host lock %s: %w (set %s for another path)", path, err, EnvLockFile)
	33	+	}
	34	+	if err := unix.Flock(int(f.Fd()), unix.LOCK_EX\|unix.LOCK_NB); err != nil {
	35	+	_ = f.Close()
	36	+	return nil, fmt.Errorf("another ermite-runner is running on this host (lock %s): only one runner per host is supported, because reaping removes everything the runner owns: %w", path, err)
	37	+	}
	38	+	return &HostLock{f: f}, nil
	39	+	}
	40	+
	41	+	// Release drops the lock; releasing twice is fine.
	42	+	func (l *HostLock) Release() error {
	43	+	if l == nil \|\| l.f == nil {
	44	+	return nil
	45	+	}
	46	+	err := l.f.Close()
	47	+	l.f = nil
	48	+	return err
	49	+	}

M internal/sandbox/sandbox.go +78 -12


@@ -24,9 +24,31 @@ const (
24	24		Container Mode = "container"
25	25		)
26	26
	27	+	// Isolation is the effective isolation of a driver, reported by the host at
	28	+	// registration and shown on the job page.
	29	+	type Isolation string
	30	+
	31	+	// Effective isolations.
	32	+	const (
	33	+	IsolationVM Isolation = "vm"
	34	+	IsolationContainerSysbox Isolation = "container_sysbox"
	35	+	IsolationContainerPrivileged Isolation = "container_privileged"
	36	+	)
	37	+
27	38		// ErrCapacity is returned by Driver.New when every slot is in use.
28	39		var ErrCapacity = errors.New("sandbox: no free slot")
29	40
	41	+	// ErrLeaked is wrapped by the error of Start or Stop when the cleanup of a
	42	+	// sandbox is incomplete: something it created (process, network device,
	43	+	// firewall rules, files, containers) may still exist and its slot stays taken.
	44	+	// The host should stop assigning jobs to this runner slot, let running jobs
	45	+	// finish and call Driver.Reap, which retries the cleanup and frees the slot.
	46	+	var ErrLeaked = errors.New("sandbox: cleanup incomplete, resources leaked")
	47	+
	48	+	// ErrBusy is returned by Driver.Reap while sandboxes are running: reaping
	49	+	// removes everything labelled as ours and would destroy them.
	50	+	var ErrBusy = errors.New("sandbox: cannot reap while sandboxes are running")
	51	+
30	52		// JobSpec describes the sandbox of one job. Zero resource fields take the
31	53		// configured defaults.
32	54		type JobSpec struct {

@@ -45,12 +67,20 @@ type Conn io.ReadWriteCloser
45	67
46	68		// Sandbox isolates one job. A Sandbox is single use: Start once, then Stop.
47	69		type Sandbox interface {
	70	+	// ID is the sandbox's identifier ("er" + 8 hex digits), assigned by
	71	+	// Driver.New. It names the sandbox's files and Docker objects, so logs and
	72	+	// leftover reports can be matched with them.
	73	+	ID() string
48	74		// Start creates the sandbox, boots it and waits for the executor inside to
49		-	// connect back. On error everything created so far has been removed.
	75	+	// connect back. It aborts when ctx ends or when Stop is called from another
	76	+	// goroutine. On error everything created so far has been removed, or the
	77	+	// error wraps ErrLeaked.
50	78		Start(ctx context.Context, spec JobSpec) (Conn, error)
51	79		// Stop destroys the sandbox and everything it created (processes, network
52	80		// devices, firewall rules, files). It is idempotent, safe after a failed or
53		-	// never-made Start, and runs to completion even when ctx is already done.
	81	+	// never-made Start, runs to completion even when ctx is already done, and
	82	+	// aborts a Start in progress. When something could not be removed the error
	83	+	// wraps ErrLeaked.
54	84		Stop(ctx context.Context) error
55	85		}
56	86

@@ -58,26 +88,34 @@ type Sandbox interface {
58	88		type Driver interface {
59	89		// Mode is VM or Container.
60	90		Mode() Mode
	91	+	// Isolation is the effective isolation (vm, container_sysbox,
	92	+	// container_privileged), for registration with ermite.
	93	+	Isolation() Isolation
61	94		// New returns a fresh Sandbox bound to a free slot (ErrCapacity if none).
62		-	// The slot is released by Stop.
	95	+	// The slot is released by Stop, unless the cleanup leaked (ErrLeaked).
63	96		New() (Sandbox, error)
64	97		// Reap removes everything a previous run of the runner left behind
65		-	// (leftover processes, devices, firewall rules, containers, files). It must
66		-	// be called once at startup, before any New.
	98	+	// (leftover processes, devices, firewall rules, containers, files) and
	99	+	// frees the slots of leaked sandboxes. Call it once at startup, before any
	100	+	// New, and after ErrLeaked when no sandbox is running; it returns ErrBusy
	101	+	// while one is.
67	102		Reap(ctx context.Context) error
68		-	// Close releases the driver's own resources (DNS forwarder, API client).
69		-	// Sandboxes still running are not stopped.
	103	+	// Close releases the driver's own resources (DNS forwarder, API client,
	104	+	// host lock). Sandboxes still running are not stopped.
70	105		Close() error
71	106		}
72	107
73	108		// Slots hands out small integers, lowest first. Slots identify a sandbox's uid,
74		-	// network device and /30 subnet, so two live sandboxes never collide.
	109	+	// network device and /30 subnet, so two live sandboxes never collide. A slot
	110	+	// whose sandbox leaked stays taken (its device name and subnet may still be in
	111	+	// use) until ReleaseLeaked.
75	112		type Slots struct {
76		-	used []bool
	113	+	used []bool
	114	+	leaked []bool
77	115		}
78	116
79	117		// NewSlots returns a pool of n slots.
80		-	func NewSlots(n int) *Slots { return &Slots{used: make([]bool, n)} }
	118	+	func NewSlots(n int) *Slots { return &Slots{used: make([]bool, n), leaked: make([]bool, n)} }
81	119
82	120		// Acquire returns the lowest free slot, or ErrCapacity. Callers serialise
83	121		// access (the drivers hold their mutex).

@@ -94,11 +132,27 @@ func (s *Slots) Acquire() (int, error) {
94	132		// Release frees a slot; releasing a free or unknown slot is a no-op.
95	133		func (s *Slots) Release(i int) {
96	134		if i >= 0 && i < len(s.used) {
97		-	s.used[i] = false
	135	+	s.used[i], s.leaked[i] = false, false
	136	+	}
	137	+	}
	138	+
	139	+	// MarkLeaked keeps slot i taken after a failed cleanup.
	140	+	func (s *Slots) MarkLeaked(i int) {
	141	+	if i >= 0 && i < len(s.used) {
	142	+	s.used[i], s.leaked[i] = true, true
	143	+	}
	144	+	}
	145	+
	146	+	// ReleaseLeaked frees every leaked slot: the caller has reaped what they held.
	147	+	func (s *Slots) ReleaseLeaked() {
	148	+	for i := range s.used {
	149	+	if s.leaked[i] {
	150	+	s.used[i], s.leaked[i] = false, false
	151	+	}
98	152		}
99	153		}
100	154
101		-	// InUse reports how many slots are taken.
	155	+	// InUse reports how many slots are taken, leaked ones included.
102	156		func (s *Slots) InUse() int {
103	157		n := 0
104	158		for _, u := range s.used {

@@ -108,3 +162,15 @@ func (s *Slots) InUse() int {
108	162		}
109	163		return n
110	164		}
	165	+
	166	+	// Busy reports how many slots belong to a sandbox that is not leaked: running,
	167	+	// starting, or handed out by New and not yet stopped.
	168	+	func (s *Slots) Busy() int {
	169	+	n := 0
	170	+	for i, u := range s.used {
	171	+	if u && !s.leaked[i] {
	172	+	n++
	173	+	}
	174	+	}
	175	+	return n
	176	+	}


@@ -24,9 +24,31 @@ const (
24		Container Mode = "container"	24		Container Mode = "container"
25		)	25		)
26			26
			27	+	// Isolation is the effective isolation of a driver, reported by the host at
			28	+	// registration and shown on the job page.
			29	+	type Isolation string
			30	+
			31	+	// Effective isolations.
			32	+	const (
			33	+	IsolationVM Isolation = "vm"
			34	+	IsolationContainerSysbox Isolation = "container_sysbox"
			35	+	IsolationContainerPrivileged Isolation = "container_privileged"
			36	+	)
			37	+
27		// ErrCapacity is returned by Driver.New when every slot is in use.	38		// ErrCapacity is returned by Driver.New when every slot is in use.
28		var ErrCapacity = errors.New("sandbox: no free slot")	39		var ErrCapacity = errors.New("sandbox: no free slot")
29			40
			41	+	// ErrLeaked is wrapped by the error of Start or Stop when the cleanup of a
			42	+	// sandbox is incomplete: something it created (process, network device,
			43	+	// firewall rules, files, containers) may still exist and its slot stays taken.
			44	+	// The host should stop assigning jobs to this runner slot, let running jobs
			45	+	// finish and call Driver.Reap, which retries the cleanup and frees the slot.
			46	+	var ErrLeaked = errors.New("sandbox: cleanup incomplete, resources leaked")
			47	+
			48	+	// ErrBusy is returned by Driver.Reap while sandboxes are running: reaping
			49	+	// removes everything labelled as ours and would destroy them.
			50	+	var ErrBusy = errors.New("sandbox: cannot reap while sandboxes are running")
			51	+
30		// JobSpec describes the sandbox of one job. Zero resource fields take the	52		// JobSpec describes the sandbox of one job. Zero resource fields take the
31		// configured defaults.	53		// configured defaults.
32		type JobSpec struct {	54		type JobSpec struct {

@@ -45,12 +67,20 @@ type Conn io.ReadWriteCloser
45			67
46		// Sandbox isolates one job. A Sandbox is single use: Start once, then Stop.	68		// Sandbox isolates one job. A Sandbox is single use: Start once, then Stop.
47		type Sandbox interface {	69		type Sandbox interface {
			70	+	// ID is the sandbox's identifier ("er" + 8 hex digits), assigned by
			71	+	// Driver.New. It names the sandbox's files and Docker objects, so logs and
			72	+	// leftover reports can be matched with them.
			73	+	ID() string
48		// Start creates the sandbox, boots it and waits for the executor inside to	74		// Start creates the sandbox, boots it and waits for the executor inside to
49	-	// connect back. On error everything created so far has been removed.	75	+	// connect back. It aborts when ctx ends or when Stop is called from another
			76	+	// goroutine. On error everything created so far has been removed, or the
			77	+	// error wraps ErrLeaked.
50		Start(ctx context.Context, spec JobSpec) (Conn, error)	78		Start(ctx context.Context, spec JobSpec) (Conn, error)
51		// Stop destroys the sandbox and everything it created (processes, network	79		// Stop destroys the sandbox and everything it created (processes, network
52		// devices, firewall rules, files). It is idempotent, safe after a failed or	80		// devices, firewall rules, files). It is idempotent, safe after a failed or
53	-	// never-made Start, and runs to completion even when ctx is already done.	81	+	// never-made Start, runs to completion even when ctx is already done, and
			82	+	// aborts a Start in progress. When something could not be removed the error
			83	+	// wraps ErrLeaked.
54		Stop(ctx context.Context) error	84		Stop(ctx context.Context) error
55		}	85		}
56			86

@@ -58,26 +88,34 @@ type Sandbox interface {
58		type Driver interface {	88		type Driver interface {
59		// Mode is VM or Container.	89		// Mode is VM or Container.
60		Mode() Mode	90		Mode() Mode
			91	+	// Isolation is the effective isolation (vm, container_sysbox,
			92	+	// container_privileged), for registration with ermite.
			93	+	Isolation() Isolation
61		// New returns a fresh Sandbox bound to a free slot (ErrCapacity if none).	94		// New returns a fresh Sandbox bound to a free slot (ErrCapacity if none).
62	-	// The slot is released by Stop.	95	+	// The slot is released by Stop, unless the cleanup leaked (ErrLeaked).
63		New() (Sandbox, error)	96		New() (Sandbox, error)
64		// Reap removes everything a previous run of the runner left behind	97		// Reap removes everything a previous run of the runner left behind
65	-	// (leftover processes, devices, firewall rules, containers, files). It must	98	+	// (leftover processes, devices, firewall rules, containers, files) and
66	-	// be called once at startup, before any New.	99	+	// frees the slots of leaked sandboxes. Call it once at startup, before any
			100	+	// New, and after ErrLeaked when no sandbox is running; it returns ErrBusy
			101	+	// while one is.
67		Reap(ctx context.Context) error	102		Reap(ctx context.Context) error
68	-	// Close releases the driver's own resources (DNS forwarder, API client).	103	+	// Close releases the driver's own resources (DNS forwarder, API client,
69	-	// Sandboxes still running are not stopped.	104	+	// host lock). Sandboxes still running are not stopped.
70		Close() error	105		Close() error
71		}	106		}
72			107
73		// Slots hands out small integers, lowest first. Slots identify a sandbox's uid,	108		// Slots hands out small integers, lowest first. Slots identify a sandbox's uid,
74	-	// network device and /30 subnet, so two live sandboxes never collide.	109	+	// network device and /30 subnet, so two live sandboxes never collide. A slot
			110	+	// whose sandbox leaked stays taken (its device name and subnet may still be in
			111	+	// use) until ReleaseLeaked.
75		type Slots struct {	112		type Slots struct {
76	-	used []bool	113	+	used []bool
			114	+	leaked []bool
77		}	115		}
78			116
79		// NewSlots returns a pool of n slots.	117		// NewSlots returns a pool of n slots.
80	-	func NewSlots(n int) *Slots { return &Slots{used: make([]bool, n)} }	118	+	func NewSlots(n int) *Slots { return &Slots{used: make([]bool, n), leaked: make([]bool, n)} }
81			119
82		// Acquire returns the lowest free slot, or ErrCapacity. Callers serialise	120		// Acquire returns the lowest free slot, or ErrCapacity. Callers serialise
83		// access (the drivers hold their mutex).	121		// access (the drivers hold their mutex).

@@ -94,11 +132,27 @@ func (s *Slots) Acquire() (int, error) {
94		// Release frees a slot; releasing a free or unknown slot is a no-op.	132		// Release frees a slot; releasing a free or unknown slot is a no-op.
95		func (s *Slots) Release(i int) {	133		func (s *Slots) Release(i int) {
96		if i >= 0 && i < len(s.used) {	134		if i >= 0 && i < len(s.used) {
97	-	s.used[i] = false	135	+	s.used[i], s.leaked[i] = false, false
			136	+	}
			137	+	}
			138	+
			139	+	// MarkLeaked keeps slot i taken after a failed cleanup.
			140	+	func (s *Slots) MarkLeaked(i int) {
			141	+	if i >= 0 && i < len(s.used) {
			142	+	s.used[i], s.leaked[i] = true, true
			143	+	}
			144	+	}
			145	+
			146	+	// ReleaseLeaked frees every leaked slot: the caller has reaped what they held.
			147	+	func (s *Slots) ReleaseLeaked() {
			148	+	for i := range s.used {
			149	+	if s.leaked[i] {
			150	+	s.used[i], s.leaked[i] = false, false
			151	+	}
98		}	152		}
99		}	153		}
100			154
101	-	// InUse reports how many slots are taken.	155	+	// InUse reports how many slots are taken, leaked ones included.
102		func (s *Slots) InUse() int {	156		func (s *Slots) InUse() int {
103		n := 0	157		n := 0
104		for _, u := range s.used {	158		for _, u := range s.used {

@@ -108,3 +162,15 @@ func (s *Slots) InUse() int {
108		}	162		}
109		return n	163		return n
110		}	164		}
			165	+
			166	+	// Busy reports how many slots belong to a sandbox that is not leaked: running,
			167	+	// starting, or handed out by New and not yet stopped.
			168	+	func (s *Slots) Busy() int {
			169	+	n := 0
			170	+	for i, u := range s.used {
			171	+	if u && !s.leaked[i] {
			172	+	n++
			173	+	}
			174	+	}
			175	+	return n
			176	+	}

M internal/sandbox/sandbox_test.go +1 -0


@@ -123,6 +123,7 @@ func TestIDs(t *testing.T) {
123	123		type fakeDriver struct{ mode Mode }
124	124
125	125		func (f fakeDriver) Mode() Mode { return f.mode }
	126	+	func (f fakeDriver) Isolation() Isolation { return Isolation(f.mode) }
126	127		func (fakeDriver) New() (Sandbox, error) { return nil, nil }
127	128		func (fakeDriver) Reap(context.Context) error { return nil }
128	129		func (fakeDriver) Close() error { return nil }


@@ -123,6 +123,7 @@ func TestIDs(t *testing.T) {
123	type fakeDriver struct{ mode Mode }	123		type fakeDriver struct{ mode Mode }
124		124
125	func (f fakeDriver) Mode() Mode { return f.mode }	125		func (f fakeDriver) Mode() Mode { return f.mode }
		126	+	func (f fakeDriver) Isolation() Isolation { return Isolation(f.mode) }
126	func (fakeDriver) New() (Sandbox, error) { return nil, nil }	127		func (fakeDriver) New() (Sandbox, error) { return nil, nil }
127	func (fakeDriver) Reap(context.Context) error { return nil }	128		func (fakeDriver) Reap(context.Context) error { return nil }
128	func (fakeDriver) Close() error { return nil }	129		func (fakeDriver) Close() error { return nil }

A internal/sandbox/space.go +43 -0

@@ -0,0 +1,43 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package sandbox
	4	+
	5	+	import (
	6	+	"context"
	7	+
	8	+	"github.com/bnema/zerowrap"
	9	+	"golang.org/x/sys/unix"
	10	+	)
	11	+
	12	+	// FreeMiB returns the space available to unprivileged users on the filesystem
	13	+	// of path, in MiB.
	14	+	func FreeMiB(path string) (int64, error) {
	15	+	var st unix.Statfs_t
	16	+	if err := unix.Statfs(path, &st); err != nil {
	17	+	return 0, err
	18	+	}
	19	+	return int64(st.Bavail) * int64(st.Bsize) >> 20, nil
	20	+	}
	21	+
	22	+	// WarnIfLowSpace logs a warning when the filesystem of dir has less free space
	23	+	// than capacity sandboxes of diskMiB would need if they all filled their sparse
	24	+	// scratch disk. It is only a warning: scratch disks are sparse and jobs rarely
	25	+	// fill them, so a smaller filesystem works until it does not. free is FreeMiB in
	26	+	// production.
	27	+	func WarnIfLowSpace(ctx context.Context, dir string, capacity, diskMiB int, free func(string) (int64, error)) {
	28	+	if free == nil {
	29	+	free = FreeMiB
	30	+	}
	31	+	log := zerowrap.FromCtx(ctx)
	32	+	have, err := free(dir)
	33	+	if err != nil {
	34	+	log.Warn().Err(err).Str("dir", dir).Msg("cannot check the free space of the state directory")
	35	+	return
	36	+	}
	37	+	need := int64(capacity) * int64(diskMiB)
	38	+	if have < need {
	39	+	log.Warn().Str("dir", dir).Int64("free_mib", have).Int64("worst_case_mib", need).
	40	+	Int("capacity", capacity).Int("job_disk_mib", diskMiB).
	41	+	Msg("the state directory has less free space than capacity x job disk: jobs that fill their scratch disk will fail with 'no space left on device'")
	42	+	}
	43	+	}

@@ -0,0 +1,43 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package sandbox
	4	+
	5	+	import (
	6	+	"context"
	7	+
	8	+	"github.com/bnema/zerowrap"
	9	+	"golang.org/x/sys/unix"
	10	+	)
	11	+
	12	+	// FreeMiB returns the space available to unprivileged users on the filesystem
	13	+	// of path, in MiB.
	14	+	func FreeMiB(path string) (int64, error) {
	15	+	var st unix.Statfs_t
	16	+	if err := unix.Statfs(path, &st); err != nil {
	17	+	return 0, err
	18	+	}
	19	+	return int64(st.Bavail) * int64(st.Bsize) >> 20, nil
	20	+	}
	21	+
	22	+	// WarnIfLowSpace logs a warning when the filesystem of dir has less free space
	23	+	// than capacity sandboxes of diskMiB would need if they all filled their sparse
	24	+	// scratch disk. It is only a warning: scratch disks are sparse and jobs rarely
	25	+	// fill them, so a smaller filesystem works until it does not. free is FreeMiB in
	26	+	// production.
	27	+	func WarnIfLowSpace(ctx context.Context, dir string, capacity, diskMiB int, free func(string) (int64, error)) {
	28	+	if free == nil {
	29	+	free = FreeMiB
	30	+	}
	31	+	log := zerowrap.FromCtx(ctx)
	32	+	have, err := free(dir)
	33	+	if err != nil {
	34	+	log.Warn().Err(err).Str("dir", dir).Msg("cannot check the free space of the state directory")
	35	+	return
	36	+	}
	37	+	need := int64(capacity) * int64(diskMiB)
	38	+	if have < need {
	39	+	log.Warn().Str("dir", dir).Int64("free_mib", have).Int64("worst_case_mib", need).
	40	+	Int("capacity", capacity).Int("job_disk_mib", diskMiB).
	41	+	Msg("the state directory has less free space than capacity x job disk: jobs that fill their scratch disk will fail with 'no space left on device'")
	42	+	}
	43	+	}