fix(sandbox/container): host lock, deny-privileged and per-job warning, pids and swap limits, loop-mounted scratch disk, keep policy while the container may run, abortable boot, ErrLeaked

commit: 146655af157e03c0dfed78040290dbe4c38ec371
author: brice · 10d ago · 2026-10-12 16:19 UTC
parent: 2f75a56
signature: GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email

10 files · +1192 -225

M internal/sandbox/container/container_test.go +498 -42


@@ -3,6 +3,7 @@
3	3		package container
4	4
5	5		import (
	6	+	"bytes"
6	7		"context"
7	8		"errors"
8	9		"fmt"

@@ -16,9 +17,14 @@ import (
16	17		"testing"
17	18		"time"
18	19
	20	+	"github.com/bnema/zerowrap"
	21	+	"github.com/moby/moby/api/types/container"
	22	+
	23	+	"bnema.dev/ermite-runner/internal/cmdrun"
19	24		"bnema.dev/ermite-runner/internal/execstream"
20	25		"bnema.dev/ermite-runner/internal/execstream/execpb"
21	26		"bnema.dev/ermite-runner/internal/sandbox"
	27	+	"bnema.dev/ermite-runner/internal/sandbox/disk"
22	28		"bnema.dev/ermite-runner/internal/sandbox/sandboxtest"
23	29		)
24	30

@@ -30,7 +36,6 @@ type fakeEngine struct {
30	36		infoErr error
31	37		imageErr error
32	38		networks map[string]networkSpec
33		-	volumes map[string]map[string]string
34	39		containers map[string]containerSpec
35	40		failStart error
36	41		noConnect bool

@@ -45,7 +50,6 @@ func newFake() *fakeEngine {
45	50		return &fakeEngine{
46	51		info: daemonInfo{OSType: "linux", Runtimes: []string{"runc"}},
47	52		networks: map[string]networkSpec{},
48		-	volumes: map[string]map[string]string{},
49	53		containers: map[string]containerSpec{},
50	54		waiters: map[string]chan exitResult{},
51	55		}

@@ -81,27 +85,6 @@ func (f *fakeEngine) ListNetworks(context.Context, string) ([]string, error) {
81	85		}
82	86		return out, nil
83	87		}
84		-	func (f *fakeEngine) CreateVolume(_ context.Context, name string, labels map[string]string) error {
85		-	f.mu.Lock()
86		-	defer f.mu.Unlock()
87		-	f.volumes[name] = labels
88		-	return nil
89		-	}
90		-	func (f *fakeEngine) RemoveVolume(_ context.Context, name string) error {
91		-	f.mu.Lock()
92		-	defer f.mu.Unlock()
93		-	delete(f.volumes, name)
94		-	return nil
95		-	}
96		-	func (f *fakeEngine) ListVolumes(context.Context, string) ([]string, error) {
97		-	f.mu.Lock()
98		-	defer f.mu.Unlock()
99		-	var out []string
100		-	for n := range f.volumes {
101		-	out = append(out, n)
102		-	}
103		-	return out, nil
104		-	}
105	88		func (f *fakeEngine) CreateContainer(_ context.Context, s containerSpec) (string, error) {
106	89		f.mu.Lock()
107	90		defer f.mu.Unlock()

@@ -189,37 +172,92 @@ func (f *fakeEngine) objects() []string {
189	172		for id := range f.networks {
190	173		out = append(out, "network "+id)
191	174		}
192		-	for n := range f.volumes {
193		-	out = append(out, "volume "+n)
194		-	}
195	175		return out
196	176		}
197	177
	178	+	// fakeMounts plays mount/umount: it records mounts and writes a mountinfo file
	179	+	// that disk.Mounter reads.
	180	+	type fakeMounts struct {
	181	+	mu sync.Mutex
	182	+	mounted map[string]bool
	183	+	file string
	184	+	failUm error
	185	+	}
	186	+
	187	+	func (m *fakeMounts) sync() {
	188	+	var b strings.Builder
	189	+	for d := range m.mounted {
	190	+	fmt.Fprintf(&b, "40 28 7:0 / %s rw - ext4 /dev/loop0 rw\n", d)
	191	+	}
	192	+	_ = os.WriteFile(m.file, []byte(b.String()), 0o644)
	193	+	}
	194	+
	195	+	func (m *fakeMounts) handle(c cmdrun.Call) ([]byte, error) {
	196	+	m.mu.Lock()
	197	+	defer m.mu.Unlock()
	198	+	switch c.Name {
	199	+	case "mkfs.ext4":
	200	+	if len(c.Args) == 1 && c.Args[0] == "-V" {
	201	+	return []byte("mke2fs 1.47.4 (6-Mar-2025)\n"), nil
	202	+	}
	203	+	case "mount":
	204	+	m.mounted[c.Args[len(c.Args)-1]] = true
	205	+	m.sync()
	206	+	case "umount":
	207	+	if m.failUm != nil {
	208	+	return nil, m.failUm
	209	+	}
	210	+	delete(m.mounted, c.Args[0])
	211	+	m.sync()
	212	+	}
	213	+	return nil, nil
	214	+	}
	215	+
	216	+	func (m *fakeMounts) count() int { m.mu.Lock(); defer m.mu.Unlock(); return len(m.mounted) }
	217	+
198	218		type cenv struct {
199		-	cfg sandbox.Config
200		-	eng *fakeEngine
201		-	plumb *sandboxtest.Plumbing
	219	+	cfg sandbox.Config
	220	+	eng *fakeEngine
	221	+	plumb *sandboxtest.Plumbing
	222	+	mounts *fakeMounts
	223	+	run *cmdrun.Fake
202	224		}
203	225
204	226		func newCEnv(t testing.T) cenv {
205	227		t.Helper()
206		-	dir := t.TempDir()
	228	+	// A short directory: the executor socket path is limited to 107 bytes and
	229	+	// t.TempDir embeds the (long) test name.
	230	+	dir, err := os.MkdirTemp("", "erc")
	231	+	if err != nil {
	232	+	t.Fatal(err)
	233	+	}
	234	+	t.Cleanup(func() { _ = os.RemoveAll(dir) })
207	235		exe := filepath.Join(dir, "runner")
208	236		sandboxtest.WriteStaticELF(t, exe)
	237	+	fm := &fakeMounts{mounted: map[string]bool{}, file: filepath.Join(dir, "mountinfo")}
	238	+	fm.sync()
209	239		return &cenv{
210		-	eng: newFake(),
211		-	plumb: sandboxtest.NewPlumbing(),
	240	+	eng: newFake(),
	241	+	plumb: sandboxtest.NewPlumbing(),
	242	+	mounts: fm,
	243	+	run: &cmdrun.Fake{Handle: fm.handle},
212	244		cfg: sandbox.Config{
213	245		Isolation: sandbox.Container, StateDir: filepath.Join(dir, "state"), Capacity: 2,
214	246		JobCPUs: 2, JobMemoryMiB: 1024, JobDiskMiB: 1024, UIDBase: 900000,
215	247		Network: netip.MustParsePrefix("10.200.0.0/16"), Executable: exe, DNSPort: 0,
	248	+	LockFile: filepath.Join(dir, "host.lock"), JobPids: 32768,
216	249		},
217	250		}
218	251		}
219	252
	253	+	func (e *cenv) opts() Options {
	254	+	return Options{engine: e.eng, Net: e.plumb, ConnectTimeout: 3 * time.Second, Run: e.run,
	255	+	Mounter: &disk.Mounter{Run: e.run, Mountinfo: e.mounts.file}}
	256	+	}
	257	+
220	258		func (e cenv) driver(t testing.T) *Driver {
221	259		t.Helper()
222		-	d, err := New(context.Background(), e.cfg, Options{engine: e.eng, Net: e.plumb, ConnectTimeout: 3 * time.Second})
	260	+	d, err := New(context.Background(), e.cfg, e.opts())
223	261		if err != nil {
224	262		t.Fatal(err)
225	263		}

@@ -227,8 +265,13 @@ func (e cenv) driver(t testing.T) *Driver {
227	265		return d
228	266		}
229	267
	268	+	func withTimeout(o Options, d time.Duration) Options { o.ConnectTimeout = d; return o }
	269	+
230	270		func (e *cenv) leftovers() []string {
231	271		out := e.eng.objects()
	272	+	if n := e.mounts.count(); n != 0 {
	273	+	out = append(out, fmt.Sprintf("%d scratch mounts", n))
	274	+	}
232	275		if n := e.plumb.Live(); n != 0 {
233	276		out = append(out, fmt.Sprintf("%d attached guests", n))
234	277		}

@@ -252,7 +295,7 @@ func stub(t *testing.T, conn sandbox.Conn) (string, int) {
252	295		func TestRootlessDockerIsRefused(t *testing.T) {
253	296		e := newCEnv(t)
254	297		e.eng.info.Rootless = true
255		-	_, err := New(context.Background(), e.cfg, Options{engine: e.eng, Net: e.plumb})
	298	+	_, err := New(context.Background(), e.cfg, e.opts())
256	299		if err == nil \|\| !strings.Contains(err.Error(), "rootless") \|\| !strings.Contains(err.Error(), "rootful") {
257	300		t.Fatalf("want a clear rootless error, got %v", err)
258	301		}

@@ -275,22 +318,22 @@ func TestRuntimeSelection(t *testing.T) {
275	318		func TestNewErrors(t *testing.T) {
276	319		e := newCEnv(t)
277	320		e.eng.infoErr = errors.New("connection refused")
278		-	if _, err := New(context.Background(), e.cfg, Options{engine: e.eng, Net: e.plumb}); err == nil \|\| !strings.Contains(err.Error(), "DOCKER_HOST") {
	321	+	if _, err := New(context.Background(), e.cfg, e.opts()); err == nil \|\| !strings.Contains(err.Error(), "DOCKER_HOST") {
279	322		t.Fatalf("got %v", err)
280	323		}
281	324		e = newCEnv(t)
282	325		e.eng.imageErr = errors.New("no such image")
283		-	if _, err := New(context.Background(), e.cfg, Options{engine: e.eng, Net: e.plumb}); err == nil {
	326	+	if _, err := New(context.Background(), e.cfg, e.opts()); err == nil {
284	327		t.Fatal("missing image accepted")
285	328		}
286	329		e = newCEnv(t)
287	330		e.eng.info.OSType = "windows"
288		-	if _, err := New(context.Background(), e.cfg, Options{engine: e.eng, Net: e.plumb}); err == nil {
	331	+	if _, err := New(context.Background(), e.cfg, e.opts()); err == nil {
289	332		t.Fatal("windows daemon accepted")
290	333		}
291	334		e = newCEnv(t)
292	335		e.cfg.Executable = "/bin/sh"
293		-	if _, err := New(context.Background(), e.cfg, Options{engine: e.eng, Net: e.plumb}); err == nil {
	336	+	if _, err := New(context.Background(), e.cfg, e.opts()); err == nil {
294	337		t.Skip("/bin/sh is static")
295	338		}
296	339		}

@@ -346,8 +389,19 @@ func TestStartRunStop(t *testing.T) {
346	389		if !roExe \|\| !runDir {
347	390		t.Errorf("binds: %+v", spec.Binds)
348	391		}
349		-	if len(spec.Volumes) != 1 \|\| spec.Volumes[0].Target != sandbox.GuestScratch {
350		-	t.Errorf("scratch volume: %+v", spec.Volumes)
	392	+	var scratch, work, tmp bool
	393	+	for _, b := range spec.Binds {
	394	+	switch b.Target {
	395	+	case sandbox.GuestScratch:
	396	+	scratch = strings.HasSuffix(b.Source, "/scratch") && !b.ReadOnly
	397	+	case "/home/runner/work":
	398	+	work = strings.HasSuffix(b.Source, "/scratch/work")
	399	+	case "/tmp":
	400	+	tmp = strings.HasSuffix(b.Source, "/scratch/tmp")
	401	+	}
	402	+	}
	403	+	if !scratch \|\| !work \|\| !tmp {
	404	+	t.Errorf("scratch, workspace and /tmp must come from the loop-mounted disk: %+v", spec.Binds)
351	405		}
352	406		if spec.DNS.String() != "10.200.0.1" \|\| spec.IPv4.String() != "10.200.0.2" \|\| netSpec.Bridge != "erbr0" \|\| netSpec.Subnet.String() != "10.200.0.0/30" {
353	407		t.Errorf("network: %+v %+v", spec, netSpec)

@@ -406,7 +460,7 @@ func TestStartFailuresCleanEverything(t *testing.T) {
406	460		t.Run(name, func(t *testing.T) {
407	461		e := newCEnv(t)
408	462		setup(e)
409		-	d, err := New(context.Background(), e.cfg, Options{engine: e.eng, Net: e.plumb, ConnectTimeout: 300 * time.Millisecond})
	463	+	d, err := New(context.Background(), e.cfg, withTimeout(e.opts(), 300*time.Millisecond))
410	464		if err != nil {
411	465		t.Fatal(err)
412	466		}

@@ -470,7 +524,6 @@ func TestReapRemovesLabelledObjectsOnly(t *testing.T) {
470	524		_, _ = e.eng.CreateContainer(ctx, containerSpec{Name: "old", Labels: map[string]string{sandbox.LabelJob: "x"}})
471	525		_, _ = e.eng.CreateNetwork(ctx, networkSpec{Name: "old", Labels: map[string]string{sandbox.LabelJob: "x"}})
472	526		_, _ = e.eng.CreateNetwork(ctx, networkSpec{Name: "mine", Labels: map[string]string{"app": "other"}})
473		-	_ = e.eng.CreateVolume(ctx, "old", map[string]string{sandbox.LabelJob: "x"})
474	527		_ = os.MkdirAll(filepath.Join(e.cfg.StateDir, "containers", "er00112233"), 0o700)
475	528		_ = os.MkdirAll(filepath.Join(e.cfg.StateDir, "containers", "keep-me"), 0o700)
476	529

@@ -554,3 +607,406 @@ func TestDoubleStartRefused(t *testing.T) {
554	607		}
555	608
556	609		func mustPrefix(s string) netip.Prefix { return netip.MustParsePrefix(s) }
	610	+
	611	+	func logger(buf *bytes.Buffer) context.Context {
	612	+	return zerowrap.WithCtx(context.Background(), zerowrap.New(zerowrap.Config{Level: "debug", Format: "json", Output: buf}))
	613	+	}
	614	+
	615	+	// S4: the privileged fallback is allowed by default, warned about at the start
	616	+	// of EVERY job, reported as the effective isolation, and refused on demand.
	617	+	func TestPrivilegedWarnedAtEveryJobAndReportedAsIsolation(t *testing.T) {
	618	+	e := newCEnv(t)
	619	+	var buf bytes.Buffer
	620	+	ctx := logger(&buf)
	621	+	d, err := New(ctx, e.cfg, e.opts())
	622	+	if err != nil {
	623	+	t.Fatal(err)
	624	+	}
	625	+	defer func() { _ = d.Close() }()
	626	+	if d.Isolation() != sandbox.IsolationContainerPrivileged \|\| d.Mode() != sandbox.Container {
	627	+	t.Fatalf("isolation %q", d.Isolation())
	628	+	}
	629	+	const msg = "privileged container: host-root equivalent, network policy best effort"
	630	+	if n := strings.Count(buf.String(), msg); n != 1 {
	631	+	t.Fatalf("one warning at driver start, got %d", n)
	632	+	}
	633	+	for i := range 3 {
	634	+	sb, _ := d.New()
	635	+	conn, err := sb.Start(ctx, sandbox.JobSpec{JobID: fmt.Sprint("job", i)})
	636	+	if err != nil {
	637	+	t.Fatal(err)
	638	+	}
	639	+	stub(t, conn)
	640	+	_ = sb.Stop(ctx)
	641	+	}
	642	+	if n := strings.Count(buf.String(), msg); n != 4 {
	643	+	t.Fatalf("the warning must be logged at every job start (1 + 3), got %d:\n%s", n, buf.String())
	644	+	}
	645	+	if !strings.Contains(buf.String(), `"job":"job2"`) {
	646	+	t.Fatal("the warning must name the job")
	647	+	}
	648	+	}
	649	+
	650	+	func TestSysboxIsReportedAndSilent(t *testing.T) {
	651	+	e := newCEnv(t)
	652	+	e.eng.info.Runtimes = []string{"runc", "sysbox-runc"}
	653	+	var buf bytes.Buffer
	654	+	ctx := logger(&buf)
	655	+	d, err := New(ctx, e.cfg, e.opts())
	656	+	if err != nil {
	657	+	t.Fatal(err)
	658	+	}
	659	+	defer func() { _ = d.Close() }()
	660	+	if d.Isolation() != sandbox.IsolationContainerSysbox {
	661	+	t.Fatalf("isolation %q", d.Isolation())
	662	+	}
	663	+	sb, _ := d.New()
	664	+	conn, _ := sb.Start(ctx, sandbox.JobSpec{JobID: "j"})
	665	+	stub(t, conn)
	666	+	_ = sb.Stop(ctx)
	667	+	if strings.Contains(buf.String(), "host-root equivalent") {
	668	+	t.Fatal("no privileged warning with sysbox")
	669	+	}
	670	+	}
	671	+
	672	+	func TestDenyPrivileged(t *testing.T) {
	673	+	e := newCEnv(t)
	674	+	e.cfg.DenyPrivileged = true
	675	+	_, err := New(context.Background(), e.cfg, e.opts())
	676	+	if err == nil \|\| !strings.Contains(err.Error(), "ERMITE_RUNNER_DENY_PRIVILEGED") \|\| !strings.Contains(err.Error(), "sysbox") {
	677	+	t.Fatalf("got %v", err)
	678	+	}
	679	+	// The host lock must be free again after a refused start.
	680	+	if l, err := sandbox.AcquireHostLock(e.cfg.LockFile); err != nil {
	681	+	t.Fatalf("a failed New must release the host lock: %v", err)
	682	+	} else {
	683	+	_ = l.Release()
	684	+	}
	685	+	// With sysbox available the setting does nothing.
	686	+	e2 := newCEnv(t)
	687	+	e2.cfg.DenyPrivileged = true
	688	+	e2.eng.info.Runtimes = []string{"sysbox-runc"}
	689	+	d, err := New(context.Background(), e2.cfg, e2.opts())
	690	+	if err != nil {
	691	+	t.Fatalf("deny-privileged with sysbox must work: %v", err)
	692	+	}
	693	+	_ = d.Close()
	694	+	}
	695	+
	696	+	// S5: pids limit from the config and no swap.
	697	+	func TestHostConfigPidsAndSwap(t *testing.T) {
	698	+	hc := hostConfigFor(containerSpec{Network: "n", MemoryMiB: 2048, CPUs: 3, Pids: 4096, DNS: netip.MustParseAddr("10.200.0.1")})
	699	+	if hc.Memory != 2048<<20 \|\| hc.MemorySwap != hc.Memory {
	700	+	t.Errorf("memory %d swap %d: swap must equal memory (no swap)", hc.Memory, hc.MemorySwap)
	701	+	}
	702	+	if hc.PidsLimit == nil \|\| *hc.PidsLimit != 4096 {
	703	+	t.Errorf("pids limit %v", hc.PidsLimit)
	704	+	}
	705	+	if hc.NanoCPUs != 3_000_000_000 {
	706	+	t.Errorf("cpus %d", hc.NanoCPUs)
	707	+	}
	708	+	if hc.Sysctls["net.ipv6.conf.all.disable_ipv6"] != "1" {
	709	+	t.Error("IPv6 must be disabled in the container")
	710	+	}
	711	+	if hostConfigFor(containerSpec{Network: "n"}).PidsLimit != nil {
	712	+	t.Error("no pids limit configured: leave it unset")
	713	+	}
	714	+	}
	715	+
	716	+	func TestJobPidsFlowsFromConfig(t *testing.T) {
	717	+	e := newCEnv(t)
	718	+	e.cfg.JobPids = 777
	719	+	d := e.driver(t)
	720	+	sb, _ := d.New()
	721	+	conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
	722	+	if err != nil {
	723	+	t.Fatal(err)
	724	+	}
	725	+	stub(t, conn)
	726	+	defer func() { _ = sb.Stop(context.Background()) }()
	727	+	e.eng.mu.Lock()
	728	+	defer e.eng.mu.Unlock()
	729	+	for _, s := range e.eng.containers {
	730	+	if s.Pids != 777 {
	731	+	t.Fatalf("spec pids %d", s.Pids)
	732	+	}
	733	+	}
	734	+	}
	735	+
	736	+	// C4: the disk is enforced by a loop-mounted ext4 file of ERMITE_RUNNER_JOB_DISK,
	737	+	// mounted while the job runs and unmounted (before the files are removed) after.
	738	+	func TestScratchDiskIsLoopMountedAndSized(t *testing.T) {
	739	+	e := newCEnv(t)
	740	+	d := e.driver(t)
	741	+	sb, _ := d.New()
	742	+	conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j", DiskMiB: 2048})
	743	+	if err != nil {
	744	+	t.Fatal(err)
	745	+	}
	746	+	stub(t, conn)
	747	+	if e.mounts.count() != 1 {
	748	+	t.Fatalf("mounts: %d", e.mounts.count())
	749	+	}
	750	+	var mkfs, mount string
	751	+	for _, l := range e.run.Lines() {
	752	+	if strings.HasPrefix(l, "mkfs.ext4") && strings.Contains(l, "scratch.ext4") {
	753	+	mkfs = l
	754	+	}
	755	+	if strings.HasPrefix(l, "mount ") {
	756	+	mount = l
	757	+	}
	758	+	}
	759	+	if !strings.Contains(mkfs, "ermite-scratch") \|\| !strings.Contains(mount, "loop") \|\| !strings.Contains(mount, "scratch.ext4") {
	760	+	t.Fatalf("mkfs=%q mount=%q", mkfs, mount)
	761	+	}
	762	+	// The scratch file has the configured size (sparse) and is not visible to the job.
	763	+	matches, _ := filepath.Glob(filepath.Join(e.cfg.StateDir, "containers", sb.ID(), "scratch.ext4"))
	764	+	if len(matches) != 1 {
	765	+	t.Fatal("no scratch file")
	766	+	}
	767	+	if st, _ := os.Stat(matches[0]); st.Size() != 2048<<20 {
	768	+	t.Fatalf("size %d", st.Size())
	769	+	}
	770	+	e.eng.mu.Lock()
	771	+	for _, s := range e.eng.containers {
	772	+	for _, b := range s.Binds {
	773	+	if strings.HasSuffix(b.Source, "scratch.ext4") \|\| b.Source == filepath.Join(e.cfg.StateDir, "containers", sb.ID()) {
	774	+	t.Errorf("the job must not see the disk file or the whole run directory: %+v", b)
	775	+	}
	776	+	}
	777	+	}
	778	+	e.eng.mu.Unlock()
	779	+	if err := sb.Stop(context.Background()); err != nil {
	780	+	t.Fatal(err)
	781	+	}
	782	+	// Order: umount comes before the directory removal (nothing below a mount is deleted).
	783	+	lines := e.run.Lines()
	784	+	if lines[len(lines)-1] != "umount "+filepath.Join(e.cfg.StateDir, "containers", sb.ID(), "scratch") {
	785	+	t.Fatalf("last command: %s", lines[len(lines)-1])
	786	+	}
	787	+	if left := e.leftovers(); len(left) != 0 {
	788	+	t.Fatalf("leftovers: %v", left)
	789	+	}
	790	+	}
	791	+
	792	+	// C5: when the container cannot be removed it may still run: keep the firewall
	793	+	// policy, the bridge, the mount and the files; report ErrLeaked; keep the slot;
	794	+	// let Reap finish.
	795	+	func TestStopKeepsPolicyWhenTheContainerMayStillRun(t *testing.T) {
	796	+	e := newCEnv(t)
	797	+	d := e.driver(t)
	798	+	sb, _ := d.New()
	799	+	conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
	800	+	if err != nil {
	801	+	t.Fatal(err)
	802	+	}
	803	+	stub(t, conn)
	804	+	e.eng.removeErr = errors.New("daemon busy")
	805	+	err = sb.Stop(context.Background())
	806	+	if !errors.Is(err, sandbox.ErrLeaked) {
	807	+	t.Fatalf("want ErrLeaked, got %v", err)
	808	+	}
	809	+	if e.plumb.Live() != 1 {
	810	+	t.Fatal("the firewall policy must stay while the container may run")
	811	+	}
	812	+	e.eng.mu.Lock()
	813	+	nets := len(e.eng.networks)
	814	+	e.eng.mu.Unlock()
	815	+	if nets != 1 \|\| e.mounts.count() != 1 {
	816	+	t.Fatalf("the bridge and the scratch mount must stay: nets=%d mounts=%d", nets, e.mounts.count())
	817	+	}
	818	+	if _, err := os.Stat(filepath.Join(e.cfg.StateDir, "containers", sb.ID(), "scratch.ext4")); err != nil {
	819	+	t.Fatal("the scratch file must stay")
	820	+	}
	821	+	if err := sb.Stop(context.Background()); !errors.Is(err, sandbox.ErrLeaked) {
	822	+	t.Fatalf("second Stop: %v", err)
	823	+	}
	824	+	// C8: the slot is taken, a new sandbox gets another slot, Reap is busy while it runs.
	825	+	other, err := d.New()
	826	+	if err != nil {
	827	+	t.Fatal(err)
	828	+	}
	829	+	if err := d.Reap(context.Background()); !errors.Is(err, sandbox.ErrBusy) {
	830	+	t.Fatalf("Reap while a sandbox exists: %v", err)
	831	+	}
	832	+	_ = other.Stop(context.Background())
	833	+	e.eng.removeErr = nil
	834	+	if err := d.Reap(context.Background()); err != nil {
	835	+	t.Fatal(err)
	836	+	}
	837	+	if left := e.leftovers(); len(left) != 0 {
	838	+	t.Fatalf("leftovers after Reap: %v", left)
	839	+	}
	840	+	if d.slots.InUse() != 0 {
	841	+	t.Fatal("Reap must free the leaked slot")
	842	+	}
	843	+	}
	844	+
	845	+	// A scratch disk that cannot be unmounted keeps its files: nothing is deleted
	846	+	// below a live mount.
	847	+	func TestStopNeverDeletesBelowALiveMount(t *testing.T) {
	848	+	e := newCEnv(t)
	849	+	d := e.driver(t)
	850	+	sb, _ := d.New()
	851	+	conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
	852	+	if err != nil {
	853	+	t.Fatal(err)
	854	+	}
	855	+	stub(t, conn)
	856	+	e.mounts.failUm = errors.New("target is busy")
	857	+	if err := sb.Stop(context.Background()); !errors.Is(err, sandbox.ErrLeaked) {
	858	+	t.Fatalf("got %v", err)
	859	+	}
	860	+	if _, err := os.Stat(filepath.Join(e.cfg.StateDir, "containers", sb.ID(), "scratch.ext4")); err != nil {
	861	+	t.Fatal("files below a mount that is still there were deleted")
	862	+	}
	863	+	e.mounts.failUm = nil
	864	+	if err := d.Reap(context.Background()); err != nil {
	865	+	t.Fatal(err)
	866	+	}
	867	+	if e.mounts.count() != 0 {
	868	+	t.Fatal("Reap must unmount leftover scratch disks")
	869	+	}
	870	+	if left := e.leftovers(); len(left) != 0 {
	871	+	t.Fatalf("leftovers: %v", left)
	872	+	}
	873	+	}
	874	+
	875	+	// C7: Stop aborts a boot in progress instead of waiting for the timeout.
	876	+	func TestStopAbortsABootInProgress(t *testing.T) {
	877	+	e := newCEnv(t)
	878	+	e.eng.noConnect = true
	879	+	o := e.opts()
	880	+	o.ConnectTimeout = time.Minute
	881	+	d, err := New(context.Background(), e.cfg, o)
	882	+	if err != nil {
	883	+	t.Fatal(err)
	884	+	}
	885	+	defer func() { _ = d.Close() }()
	886	+	sb, _ := d.New()
	887	+	started := make(chan error, 1)
	888	+	go func() { _, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"}); started <- err }()
	889	+	time.Sleep(500 * time.Millisecond)
	890	+	t0 := time.Now()
	891	+	if err := sb.Stop(context.Background()); err != nil {
	892	+	t.Fatal(err)
	893	+	}
	894	+	select {
	895	+	case err := <-started:
	896	+	if !errors.Is(err, sandbox.ErrAborted) {
	897	+	t.Fatalf("Start must report the abort, got %v", err)
	898	+	}
	899	+	case <-time.After(10 * time.Second):
	900	+	t.Fatal("Start was not aborted")
	901	+	}
	902	+	if time.Since(t0) > 5*time.Second {
	903	+	t.Fatalf("Stop took %v", time.Since(t0))
	904	+	}
	905	+	if left := e.leftovers(); len(left) != 0 {
	906	+	t.Fatalf("leftovers: %v", left)
	907	+	}
	908	+	}
	909	+
	910	+	// C16: ID names the Docker objects.
	911	+	func TestSandboxIDNamesDockerObjects(t *testing.T) {
	912	+	e := newCEnv(t)
	913	+	d := e.driver(t)
	914	+	sb, _ := d.New()
	915	+	if !sandbox.ValidID(sb.ID()) {
	916	+	t.Fatal(sb.ID())
	917	+	}
	918	+	conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
	919	+	if err != nil {
	920	+	t.Fatal(err)
	921	+	}
	922	+	stub(t, conn)
	923	+	defer func() { _ = sb.Stop(context.Background()) }()
	924	+	e.eng.mu.Lock()
	925	+	defer e.eng.mu.Unlock()
	926	+	for _, s := range e.eng.containers {
	927	+	if s.Name != "ermite-"+sb.ID() {
	928	+	t.Fatalf("container name %q", s.Name)
	929	+	}
	930	+	}
	931	+	}
	932	+
	933	+	// S3: a second runner on the same host is refused whatever its state directory.
	934	+	func TestHostLockAcrossStateDirs(t *testing.T) {
	935	+	e := newCEnv(t)
	936	+	d := e.driver(t)
	937	+	_ = d
	938	+	e2 := newCEnv(t)
	939	+	e2.cfg.LockFile = e.cfg.LockFile // same host, another state directory
	940	+	_, err := New(context.Background(), e2.cfg, e2.opts())
	941	+	if err == nil \|\| !strings.Contains(err.Error(), "another ermite-runner is running on this host") {
	942	+	t.Fatalf("got %v", err)
	943	+	}
	944	+	}
	945	+
	946	+	// C10: a cancelled wait is reported as cancelled, never as a clean exit.
	947	+	func TestAwaitExit(t *testing.T) {
	948	+	res := make(chan container.WaitResponse, 1)
	949	+	errc := make(chan error, 1)
	950	+	res <- container.WaitResponse{StatusCode: 3}
	951	+	if r := <-awaitExit(context.Background(), res, errc); r.Code != 3 \|\| r.Err != nil {
	952	+	t.Fatalf("%+v", r)
	953	+	}
	954	+	errc <- errors.New("daemon gone")
	955	+	if r := <-awaitExit(context.Background(), make(chan container.WaitResponse), errc); r.Code != -1 \|\| r.Err == nil {
	956	+	t.Fatalf("%+v", r)
	957	+	}
	958	+	// Both ready and the context already cancelled: the cancellation wins.
	959	+	ctx, cancel := context.WithCancel(context.Background())
	960	+	cancel()
	961	+	for range 50 {
	962	+	res2 := make(chan container.WaitResponse, 1)
	963	+	res2 <- container.WaitResponse{StatusCode: 0}
	964	+	if r := <-awaitExit(ctx, res2, make(chan error)); r.Err == nil \|\| !errors.Is(r.Err, context.Canceled) \|\| r.Code != -1 {
	965	+	t.Fatalf("a cancelled wait must not look like a clean exit: %+v", r)
	966	+	}
	967	+	}
	968	+	}
	969	+
	970	+	func TestFreeSpaceWarningInContainerMode(t *testing.T) {
	971	+	e := newCEnv(t)
	972	+	var buf bytes.Buffer
	973	+	o := e.opts()
	974	+	o.FreeSpace = func(string) (int64, error) { return 1, nil }
	975	+	d, err := New(logger(&buf), e.cfg, o)
	976	+	if err != nil {
	977	+	t.Fatal(err)
	978	+	}
	979	+	defer func() { _ = d.Close() }()
	980	+	if !strings.Contains(buf.String(), "less free space") {
	981	+	t.Fatalf("no warning: %s", buf.String())
	982	+	}
	983	+	}
	984	+
	985	+	func TestMkfsTooOldRefusedAtStart(t *testing.T) {
	986	+	e := newCEnv(t)
	987	+	run := &cmdrun.Fake{Handle: func(c cmdrun.Call) ([]byte, error) {
	988	+	if c.Name == "mkfs.ext4" {
	989	+	return []byte("mke2fs 1.42.9 (4-Feb-2014)\n"), nil
	990	+	}
	991	+	return nil, nil
	992	+	}}
	993	+	o := e.opts()
	994	+	o.Run = run
	995	+	if _, err := New(context.Background(), e.cfg, o); err == nil \|\| !strings.Contains(err.Error(), "1.43") {
	996	+	t.Fatalf("got %v", err)
	997	+	}
	998	+	}
	999	+
	1000	+	func TestSocketPathTooLongIsExplained(t *testing.T) {
	1001	+	e := newCEnv(t)
	1002	+	e.cfg.StateDir = filepath.Join(e.cfg.StateDir, strings.Repeat("d", 100))
	1003	+	d := e.driver(t)
	1004	+	sb, _ := d.New()
	1005	+	_, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
	1006	+	if err == nil \|\| !strings.Contains(err.Error(), "108-byte limit") && !strings.Contains(err.Error(), "limit of a Unix socket") {
	1007	+	t.Fatalf("got %v", err)
	1008	+	}
	1009	+	if left := e.leftovers(); len(left) != 0 {
	1010	+	t.Fatalf("leftovers: %v", left)
	1011	+	}
	1012	+	}


@@ -3,6 +3,7 @@
3		package container	3		package container
4			4
5		import (	5		import (
			6	+	"bytes"
6		"context"	7		"context"
7		"errors"	8		"errors"
8		"fmt"	9		"fmt"

@@ -16,9 +17,14 @@ import (
16		"testing"	17		"testing"
17		"time"	18		"time"
18			19
			20	+	"github.com/bnema/zerowrap"
			21	+	"github.com/moby/moby/api/types/container"
			22	+
			23	+	"bnema.dev/ermite-runner/internal/cmdrun"
19		"bnema.dev/ermite-runner/internal/execstream"	24		"bnema.dev/ermite-runner/internal/execstream"
20		"bnema.dev/ermite-runner/internal/execstream/execpb"	25		"bnema.dev/ermite-runner/internal/execstream/execpb"
21		"bnema.dev/ermite-runner/internal/sandbox"	26		"bnema.dev/ermite-runner/internal/sandbox"
			27	+	"bnema.dev/ermite-runner/internal/sandbox/disk"
22		"bnema.dev/ermite-runner/internal/sandbox/sandboxtest"	28		"bnema.dev/ermite-runner/internal/sandbox/sandboxtest"
23		)	29		)
24			30

@@ -30,7 +36,6 @@ type fakeEngine struct {
30		infoErr error	36		infoErr error
31		imageErr error	37		imageErr error
32		networks map[string]networkSpec	38		networks map[string]networkSpec
33	-	volumes map[string]map[string]string
34		containers map[string]containerSpec	39		containers map[string]containerSpec
35		failStart error	40		failStart error
36		noConnect bool	41		noConnect bool

@@ -45,7 +50,6 @@ func newFake() *fakeEngine {
45		return &fakeEngine{	50		return &fakeEngine{
46		info: daemonInfo{OSType: "linux", Runtimes: []string{"runc"}},	51		info: daemonInfo{OSType: "linux", Runtimes: []string{"runc"}},
47		networks: map[string]networkSpec{},	52		networks: map[string]networkSpec{},
48	-	volumes: map[string]map[string]string{},
49		containers: map[string]containerSpec{},	53		containers: map[string]containerSpec{},
50		waiters: map[string]chan exitResult{},	54		waiters: map[string]chan exitResult{},
51		}	55		}

@@ -81,27 +85,6 @@ func (f *fakeEngine) ListNetworks(context.Context, string) ([]string, error) {
81		}	85		}
82		return out, nil	86		return out, nil
83		}	87		}
84	-	func (f *fakeEngine) CreateVolume(_ context.Context, name string, labels map[string]string) error {
85	-	f.mu.Lock()
86	-	defer f.mu.Unlock()
87	-	f.volumes[name] = labels
88	-	return nil
89	-	}
90	-	func (f *fakeEngine) RemoveVolume(_ context.Context, name string) error {
91	-	f.mu.Lock()
92	-	defer f.mu.Unlock()
93	-	delete(f.volumes, name)
94	-	return nil
95	-	}
96	-	func (f *fakeEngine) ListVolumes(context.Context, string) ([]string, error) {
97	-	f.mu.Lock()
98	-	defer f.mu.Unlock()
99	-	var out []string
100	-	for n := range f.volumes {
101	-	out = append(out, n)
102	-	}
103	-	return out, nil
104	-	}
105		func (f *fakeEngine) CreateContainer(_ context.Context, s containerSpec) (string, error) {	88		func (f *fakeEngine) CreateContainer(_ context.Context, s containerSpec) (string, error) {
106		f.mu.Lock()	89		f.mu.Lock()
107		defer f.mu.Unlock()	90		defer f.mu.Unlock()

@@ -189,37 +172,92 @@ func (f *fakeEngine) objects() []string {
189		for id := range f.networks {	172		for id := range f.networks {
190		out = append(out, "network "+id)	173		out = append(out, "network "+id)
191		}	174		}
192	-	for n := range f.volumes {
193	-	out = append(out, "volume "+n)
194	-	}
195		return out	175		return out
196		}	176		}
197			177
			178	+	// fakeMounts plays mount/umount: it records mounts and writes a mountinfo file
			179	+	// that disk.Mounter reads.
			180	+	type fakeMounts struct {
			181	+	mu sync.Mutex
			182	+	mounted map[string]bool
			183	+	file string
			184	+	failUm error
			185	+	}
			186	+
			187	+	func (m *fakeMounts) sync() {
			188	+	var b strings.Builder
			189	+	for d := range m.mounted {
			190	+	fmt.Fprintf(&b, "40 28 7:0 / %s rw - ext4 /dev/loop0 rw\n", d)
			191	+	}
			192	+	_ = os.WriteFile(m.file, []byte(b.String()), 0o644)
			193	+	}
			194	+
			195	+	func (m *fakeMounts) handle(c cmdrun.Call) ([]byte, error) {
			196	+	m.mu.Lock()
			197	+	defer m.mu.Unlock()
			198	+	switch c.Name {
			199	+	case "mkfs.ext4":
			200	+	if len(c.Args) == 1 && c.Args[0] == "-V" {
			201	+	return []byte("mke2fs 1.47.4 (6-Mar-2025)\n"), nil
			202	+	}
			203	+	case "mount":
			204	+	m.mounted[c.Args[len(c.Args)-1]] = true
			205	+	m.sync()
			206	+	case "umount":
			207	+	if m.failUm != nil {
			208	+	return nil, m.failUm
			209	+	}
			210	+	delete(m.mounted, c.Args[0])
			211	+	m.sync()
			212	+	}
			213	+	return nil, nil
			214	+	}
			215	+
			216	+	func (m *fakeMounts) count() int { m.mu.Lock(); defer m.mu.Unlock(); return len(m.mounted) }
			217	+
198		type cenv struct {	218		type cenv struct {
199	-	cfg sandbox.Config	219	+	cfg sandbox.Config
200	-	eng *fakeEngine	220	+	eng *fakeEngine
201	-	plumb *sandboxtest.Plumbing	221	+	plumb *sandboxtest.Plumbing
			222	+	mounts *fakeMounts
			223	+	run *cmdrun.Fake
202		}	224		}
203			225
204		func newCEnv(t testing.T) cenv {	226		func newCEnv(t testing.T) cenv {
205		t.Helper()	227		t.Helper()
206	-	dir := t.TempDir()	228	+	// A short directory: the executor socket path is limited to 107 bytes and
			229	+	// t.TempDir embeds the (long) test name.
			230	+	dir, err := os.MkdirTemp("", "erc")
			231	+	if err != nil {
			232	+	t.Fatal(err)
			233	+	}
			234	+	t.Cleanup(func() { _ = os.RemoveAll(dir) })
207		exe := filepath.Join(dir, "runner")	235		exe := filepath.Join(dir, "runner")
208		sandboxtest.WriteStaticELF(t, exe)	236		sandboxtest.WriteStaticELF(t, exe)
			237	+	fm := &fakeMounts{mounted: map[string]bool{}, file: filepath.Join(dir, "mountinfo")}
			238	+	fm.sync()
209		return &cenv{	239		return &cenv{
210	-	eng: newFake(),	240	+	eng: newFake(),
211	-	plumb: sandboxtest.NewPlumbing(),	241	+	plumb: sandboxtest.NewPlumbing(),
			242	+	mounts: fm,
			243	+	run: &cmdrun.Fake{Handle: fm.handle},
212		cfg: sandbox.Config{	244		cfg: sandbox.Config{
213		Isolation: sandbox.Container, StateDir: filepath.Join(dir, "state"), Capacity: 2,	245		Isolation: sandbox.Container, StateDir: filepath.Join(dir, "state"), Capacity: 2,
214		JobCPUs: 2, JobMemoryMiB: 1024, JobDiskMiB: 1024, UIDBase: 900000,	246		JobCPUs: 2, JobMemoryMiB: 1024, JobDiskMiB: 1024, UIDBase: 900000,
215		Network: netip.MustParsePrefix("10.200.0.0/16"), Executable: exe, DNSPort: 0,	247		Network: netip.MustParsePrefix("10.200.0.0/16"), Executable: exe, DNSPort: 0,
			248	+	LockFile: filepath.Join(dir, "host.lock"), JobPids: 32768,
216		},	249		},
217		}	250		}
218		}	251		}
219			252
			253	+	func (e *cenv) opts() Options {
			254	+	return Options{engine: e.eng, Net: e.plumb, ConnectTimeout: 3 * time.Second, Run: e.run,
			255	+	Mounter: &disk.Mounter{Run: e.run, Mountinfo: e.mounts.file}}
			256	+	}
			257	+
220		func (e cenv) driver(t testing.T) *Driver {	258		func (e cenv) driver(t testing.T) *Driver {
221		t.Helper()	259		t.Helper()
222	-	d, err := New(context.Background(), e.cfg, Options{engine: e.eng, Net: e.plumb, ConnectTimeout: 3 * time.Second})	260	+	d, err := New(context.Background(), e.cfg, e.opts())
223		if err != nil {	261		if err != nil {
224		t.Fatal(err)	262		t.Fatal(err)
225		}	263		}

@@ -227,8 +265,13 @@ func (e cenv) driver(t testing.T) *Driver {
227		return d	265		return d
228		}	266		}
229			267
			268	+	func withTimeout(o Options, d time.Duration) Options { o.ConnectTimeout = d; return o }
			269	+
230		func (e *cenv) leftovers() []string {	270		func (e *cenv) leftovers() []string {
231		out := e.eng.objects()	271		out := e.eng.objects()
			272	+	if n := e.mounts.count(); n != 0 {
			273	+	out = append(out, fmt.Sprintf("%d scratch mounts", n))
			274	+	}
232		if n := e.plumb.Live(); n != 0 {	275		if n := e.plumb.Live(); n != 0 {
233		out = append(out, fmt.Sprintf("%d attached guests", n))	276		out = append(out, fmt.Sprintf("%d attached guests", n))
234		}	277		}

@@ -252,7 +295,7 @@ func stub(t *testing.T, conn sandbox.Conn) (string, int) {
252		func TestRootlessDockerIsRefused(t *testing.T) {	295		func TestRootlessDockerIsRefused(t *testing.T) {
253		e := newCEnv(t)	296		e := newCEnv(t)
254		e.eng.info.Rootless = true	297		e.eng.info.Rootless = true
255	-	_, err := New(context.Background(), e.cfg, Options{engine: e.eng, Net: e.plumb})	298	+	_, err := New(context.Background(), e.cfg, e.opts())
256		if err == nil \|\| !strings.Contains(err.Error(), "rootless") \|\| !strings.Contains(err.Error(), "rootful") {	299		if err == nil \|\| !strings.Contains(err.Error(), "rootless") \|\| !strings.Contains(err.Error(), "rootful") {
257		t.Fatalf("want a clear rootless error, got %v", err)	300		t.Fatalf("want a clear rootless error, got %v", err)
258		}	301		}

@@ -275,22 +318,22 @@ func TestRuntimeSelection(t *testing.T) {
275		func TestNewErrors(t *testing.T) {	318		func TestNewErrors(t *testing.T) {
276		e := newCEnv(t)	319		e := newCEnv(t)
277		e.eng.infoErr = errors.New("connection refused")	320		e.eng.infoErr = errors.New("connection refused")
278	-	if _, err := New(context.Background(), e.cfg, Options{engine: e.eng, Net: e.plumb}); err == nil \|\| !strings.Contains(err.Error(), "DOCKER_HOST") {	321	+	if _, err := New(context.Background(), e.cfg, e.opts()); err == nil \|\| !strings.Contains(err.Error(), "DOCKER_HOST") {
279		t.Fatalf("got %v", err)	322		t.Fatalf("got %v", err)
280		}	323		}
281		e = newCEnv(t)	324		e = newCEnv(t)
282		e.eng.imageErr = errors.New("no such image")	325		e.eng.imageErr = errors.New("no such image")
283	-	if _, err := New(context.Background(), e.cfg, Options{engine: e.eng, Net: e.plumb}); err == nil {	326	+	if _, err := New(context.Background(), e.cfg, e.opts()); err == nil {
284		t.Fatal("missing image accepted")	327		t.Fatal("missing image accepted")
285		}	328		}
286		e = newCEnv(t)	329		e = newCEnv(t)
287		e.eng.info.OSType = "windows"	330		e.eng.info.OSType = "windows"
288	-	if _, err := New(context.Background(), e.cfg, Options{engine: e.eng, Net: e.plumb}); err == nil {	331	+	if _, err := New(context.Background(), e.cfg, e.opts()); err == nil {
289		t.Fatal("windows daemon accepted")	332		t.Fatal("windows daemon accepted")
290		}	333		}
291		e = newCEnv(t)	334		e = newCEnv(t)
292		e.cfg.Executable = "/bin/sh"	335		e.cfg.Executable = "/bin/sh"
293	-	if _, err := New(context.Background(), e.cfg, Options{engine: e.eng, Net: e.plumb}); err == nil {	336	+	if _, err := New(context.Background(), e.cfg, e.opts()); err == nil {
294		t.Skip("/bin/sh is static")	337		t.Skip("/bin/sh is static")
295		}	338		}
296		}	339		}

@@ -346,8 +389,19 @@ func TestStartRunStop(t *testing.T) {
346		if !roExe \|\| !runDir {	389		if !roExe \|\| !runDir {
347		t.Errorf("binds: %+v", spec.Binds)	390		t.Errorf("binds: %+v", spec.Binds)
348		}	391		}
349	-	if len(spec.Volumes) != 1 \|\| spec.Volumes[0].Target != sandbox.GuestScratch {	392	+	var scratch, work, tmp bool
350	-	t.Errorf("scratch volume: %+v", spec.Volumes)	393	+	for _, b := range spec.Binds {
			394	+	switch b.Target {
			395	+	case sandbox.GuestScratch:
			396	+	scratch = strings.HasSuffix(b.Source, "/scratch") && !b.ReadOnly
			397	+	case "/home/runner/work":
			398	+	work = strings.HasSuffix(b.Source, "/scratch/work")
			399	+	case "/tmp":
			400	+	tmp = strings.HasSuffix(b.Source, "/scratch/tmp")
			401	+	}
			402	+	}
			403	+	if !scratch \|\| !work \|\| !tmp {
			404	+	t.Errorf("scratch, workspace and /tmp must come from the loop-mounted disk: %+v", spec.Binds)
351		}	405		}
352		if spec.DNS.String() != "10.200.0.1" \|\| spec.IPv4.String() != "10.200.0.2" \|\| netSpec.Bridge != "erbr0" \|\| netSpec.Subnet.String() != "10.200.0.0/30" {	406		if spec.DNS.String() != "10.200.0.1" \|\| spec.IPv4.String() != "10.200.0.2" \|\| netSpec.Bridge != "erbr0" \|\| netSpec.Subnet.String() != "10.200.0.0/30" {
353		t.Errorf("network: %+v %+v", spec, netSpec)	407		t.Errorf("network: %+v %+v", spec, netSpec)

@@ -406,7 +460,7 @@ func TestStartFailuresCleanEverything(t *testing.T) {
406		t.Run(name, func(t *testing.T) {	460		t.Run(name, func(t *testing.T) {
407		e := newCEnv(t)	461		e := newCEnv(t)
408		setup(e)	462		setup(e)
409	-	d, err := New(context.Background(), e.cfg, Options{engine: e.eng, Net: e.plumb, ConnectTimeout: 300 * time.Millisecond})	463	+	d, err := New(context.Background(), e.cfg, withTimeout(e.opts(), 300*time.Millisecond))
410		if err != nil {	464		if err != nil {
411		t.Fatal(err)	465		t.Fatal(err)
412		}	466		}

@@ -470,7 +524,6 @@ func TestReapRemovesLabelledObjectsOnly(t *testing.T) {
470		_, _ = e.eng.CreateContainer(ctx, containerSpec{Name: "old", Labels: map[string]string{sandbox.LabelJob: "x"}})	524		_, _ = e.eng.CreateContainer(ctx, containerSpec{Name: "old", Labels: map[string]string{sandbox.LabelJob: "x"}})
471		_, _ = e.eng.CreateNetwork(ctx, networkSpec{Name: "old", Labels: map[string]string{sandbox.LabelJob: "x"}})	525		_, _ = e.eng.CreateNetwork(ctx, networkSpec{Name: "old", Labels: map[string]string{sandbox.LabelJob: "x"}})
472		_, _ = e.eng.CreateNetwork(ctx, networkSpec{Name: "mine", Labels: map[string]string{"app": "other"}})	526		_, _ = e.eng.CreateNetwork(ctx, networkSpec{Name: "mine", Labels: map[string]string{"app": "other"}})
473	-	_ = e.eng.CreateVolume(ctx, "old", map[string]string{sandbox.LabelJob: "x"})
474		_ = os.MkdirAll(filepath.Join(e.cfg.StateDir, "containers", "er00112233"), 0o700)	527		_ = os.MkdirAll(filepath.Join(e.cfg.StateDir, "containers", "er00112233"), 0o700)
475		_ = os.MkdirAll(filepath.Join(e.cfg.StateDir, "containers", "keep-me"), 0o700)	528		_ = os.MkdirAll(filepath.Join(e.cfg.StateDir, "containers", "keep-me"), 0o700)
476			529

@@ -554,3 +607,406 @@ func TestDoubleStartRefused(t *testing.T) {
554		}	607		}
555			608
556		func mustPrefix(s string) netip.Prefix { return netip.MustParsePrefix(s) }	609		func mustPrefix(s string) netip.Prefix { return netip.MustParsePrefix(s) }
			610	+
			611	+	func logger(buf *bytes.Buffer) context.Context {
			612	+	return zerowrap.WithCtx(context.Background(), zerowrap.New(zerowrap.Config{Level: "debug", Format: "json", Output: buf}))
			613	+	}
			614	+
			615	+	// S4: the privileged fallback is allowed by default, warned about at the start
			616	+	// of EVERY job, reported as the effective isolation, and refused on demand.
			617	+	func TestPrivilegedWarnedAtEveryJobAndReportedAsIsolation(t *testing.T) {
			618	+	e := newCEnv(t)
			619	+	var buf bytes.Buffer
			620	+	ctx := logger(&buf)
			621	+	d, err := New(ctx, e.cfg, e.opts())
			622	+	if err != nil {
			623	+	t.Fatal(err)
			624	+	}
			625	+	defer func() { _ = d.Close() }()
			626	+	if d.Isolation() != sandbox.IsolationContainerPrivileged \|\| d.Mode() != sandbox.Container {
			627	+	t.Fatalf("isolation %q", d.Isolation())
			628	+	}
			629	+	const msg = "privileged container: host-root equivalent, network policy best effort"
			630	+	if n := strings.Count(buf.String(), msg); n != 1 {
			631	+	t.Fatalf("one warning at driver start, got %d", n)
			632	+	}
			633	+	for i := range 3 {
			634	+	sb, _ := d.New()
			635	+	conn, err := sb.Start(ctx, sandbox.JobSpec{JobID: fmt.Sprint("job", i)})
			636	+	if err != nil {
			637	+	t.Fatal(err)
			638	+	}
			639	+	stub(t, conn)
			640	+	_ = sb.Stop(ctx)
			641	+	}
			642	+	if n := strings.Count(buf.String(), msg); n != 4 {
			643	+	t.Fatalf("the warning must be logged at every job start (1 + 3), got %d:\n%s", n, buf.String())
			644	+	}
			645	+	if !strings.Contains(buf.String(), `"job":"job2"`) {
			646	+	t.Fatal("the warning must name the job")
			647	+	}
			648	+	}
			649	+
			650	+	func TestSysboxIsReportedAndSilent(t *testing.T) {
			651	+	e := newCEnv(t)
			652	+	e.eng.info.Runtimes = []string{"runc", "sysbox-runc"}
			653	+	var buf bytes.Buffer
			654	+	ctx := logger(&buf)
			655	+	d, err := New(ctx, e.cfg, e.opts())
			656	+	if err != nil {
			657	+	t.Fatal(err)
			658	+	}
			659	+	defer func() { _ = d.Close() }()
			660	+	if d.Isolation() != sandbox.IsolationContainerSysbox {
			661	+	t.Fatalf("isolation %q", d.Isolation())
			662	+	}
			663	+	sb, _ := d.New()
			664	+	conn, _ := sb.Start(ctx, sandbox.JobSpec{JobID: "j"})
			665	+	stub(t, conn)
			666	+	_ = sb.Stop(ctx)
			667	+	if strings.Contains(buf.String(), "host-root equivalent") {
			668	+	t.Fatal("no privileged warning with sysbox")
			669	+	}
			670	+	}
			671	+
			672	+	func TestDenyPrivileged(t *testing.T) {
			673	+	e := newCEnv(t)
			674	+	e.cfg.DenyPrivileged = true
			675	+	_, err := New(context.Background(), e.cfg, e.opts())
			676	+	if err == nil \|\| !strings.Contains(err.Error(), "ERMITE_RUNNER_DENY_PRIVILEGED") \|\| !strings.Contains(err.Error(), "sysbox") {
			677	+	t.Fatalf("got %v", err)
			678	+	}
			679	+	// The host lock must be free again after a refused start.
			680	+	if l, err := sandbox.AcquireHostLock(e.cfg.LockFile); err != nil {
			681	+	t.Fatalf("a failed New must release the host lock: %v", err)
			682	+	} else {
			683	+	_ = l.Release()
			684	+	}
			685	+	// With sysbox available the setting does nothing.
			686	+	e2 := newCEnv(t)
			687	+	e2.cfg.DenyPrivileged = true
			688	+	e2.eng.info.Runtimes = []string{"sysbox-runc"}
			689	+	d, err := New(context.Background(), e2.cfg, e2.opts())
			690	+	if err != nil {
			691	+	t.Fatalf("deny-privileged with sysbox must work: %v", err)
			692	+	}
			693	+	_ = d.Close()
			694	+	}
			695	+
			696	+	// S5: pids limit from the config and no swap.
			697	+	func TestHostConfigPidsAndSwap(t *testing.T) {
			698	+	hc := hostConfigFor(containerSpec{Network: "n", MemoryMiB: 2048, CPUs: 3, Pids: 4096, DNS: netip.MustParseAddr("10.200.0.1")})
			699	+	if hc.Memory != 2048<<20 \|\| hc.MemorySwap != hc.Memory {
			700	+	t.Errorf("memory %d swap %d: swap must equal memory (no swap)", hc.Memory, hc.MemorySwap)
			701	+	}
			702	+	if hc.PidsLimit == nil \|\| *hc.PidsLimit != 4096 {
			703	+	t.Errorf("pids limit %v", hc.PidsLimit)
			704	+	}
			705	+	if hc.NanoCPUs != 3_000_000_000 {
			706	+	t.Errorf("cpus %d", hc.NanoCPUs)
			707	+	}
			708	+	if hc.Sysctls["net.ipv6.conf.all.disable_ipv6"] != "1" {
			709	+	t.Error("IPv6 must be disabled in the container")
			710	+	}
			711	+	if hostConfigFor(containerSpec{Network: "n"}).PidsLimit != nil {
			712	+	t.Error("no pids limit configured: leave it unset")
			713	+	}
			714	+	}
			715	+
			716	+	func TestJobPidsFlowsFromConfig(t *testing.T) {
			717	+	e := newCEnv(t)
			718	+	e.cfg.JobPids = 777
			719	+	d := e.driver(t)
			720	+	sb, _ := d.New()
			721	+	conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
			722	+	if err != nil {
			723	+	t.Fatal(err)
			724	+	}
			725	+	stub(t, conn)
			726	+	defer func() { _ = sb.Stop(context.Background()) }()
			727	+	e.eng.mu.Lock()
			728	+	defer e.eng.mu.Unlock()
			729	+	for _, s := range e.eng.containers {
			730	+	if s.Pids != 777 {
			731	+	t.Fatalf("spec pids %d", s.Pids)
			732	+	}
			733	+	}
			734	+	}
			735	+
			736	+	// C4: the disk is enforced by a loop-mounted ext4 file of ERMITE_RUNNER_JOB_DISK,
			737	+	// mounted while the job runs and unmounted (before the files are removed) after.
			738	+	func TestScratchDiskIsLoopMountedAndSized(t *testing.T) {
			739	+	e := newCEnv(t)
			740	+	d := e.driver(t)
			741	+	sb, _ := d.New()
			742	+	conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j", DiskMiB: 2048})
			743	+	if err != nil {
			744	+	t.Fatal(err)
			745	+	}
			746	+	stub(t, conn)
			747	+	if e.mounts.count() != 1 {
			748	+	t.Fatalf("mounts: %d", e.mounts.count())
			749	+	}
			750	+	var mkfs, mount string
			751	+	for _, l := range e.run.Lines() {
			752	+	if strings.HasPrefix(l, "mkfs.ext4") && strings.Contains(l, "scratch.ext4") {
			753	+	mkfs = l
			754	+	}
			755	+	if strings.HasPrefix(l, "mount ") {
			756	+	mount = l
			757	+	}
			758	+	}
			759	+	if !strings.Contains(mkfs, "ermite-scratch") \|\| !strings.Contains(mount, "loop") \|\| !strings.Contains(mount, "scratch.ext4") {
			760	+	t.Fatalf("mkfs=%q mount=%q", mkfs, mount)
			761	+	}
			762	+	// The scratch file has the configured size (sparse) and is not visible to the job.
			763	+	matches, _ := filepath.Glob(filepath.Join(e.cfg.StateDir, "containers", sb.ID(), "scratch.ext4"))
			764	+	if len(matches) != 1 {
			765	+	t.Fatal("no scratch file")
			766	+	}
			767	+	if st, _ := os.Stat(matches[0]); st.Size() != 2048<<20 {
			768	+	t.Fatalf("size %d", st.Size())
			769	+	}
			770	+	e.eng.mu.Lock()
			771	+	for _, s := range e.eng.containers {
			772	+	for _, b := range s.Binds {
			773	+	if strings.HasSuffix(b.Source, "scratch.ext4") \|\| b.Source == filepath.Join(e.cfg.StateDir, "containers", sb.ID()) {
			774	+	t.Errorf("the job must not see the disk file or the whole run directory: %+v", b)
			775	+	}
			776	+	}
			777	+	}
			778	+	e.eng.mu.Unlock()
			779	+	if err := sb.Stop(context.Background()); err != nil {
			780	+	t.Fatal(err)
			781	+	}
			782	+	// Order: umount comes before the directory removal (nothing below a mount is deleted).
			783	+	lines := e.run.Lines()
			784	+	if lines[len(lines)-1] != "umount "+filepath.Join(e.cfg.StateDir, "containers", sb.ID(), "scratch") {
			785	+	t.Fatalf("last command: %s", lines[len(lines)-1])
			786	+	}
			787	+	if left := e.leftovers(); len(left) != 0 {
			788	+	t.Fatalf("leftovers: %v", left)
			789	+	}
			790	+	}
			791	+
			792	+	// C5: when the container cannot be removed it may still run: keep the firewall
			793	+	// policy, the bridge, the mount and the files; report ErrLeaked; keep the slot;
			794	+	// let Reap finish.
			795	+	func TestStopKeepsPolicyWhenTheContainerMayStillRun(t *testing.T) {
			796	+	e := newCEnv(t)
			797	+	d := e.driver(t)
			798	+	sb, _ := d.New()
			799	+	conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
			800	+	if err != nil {
			801	+	t.Fatal(err)
			802	+	}
			803	+	stub(t, conn)
			804	+	e.eng.removeErr = errors.New("daemon busy")
			805	+	err = sb.Stop(context.Background())
			806	+	if !errors.Is(err, sandbox.ErrLeaked) {
			807	+	t.Fatalf("want ErrLeaked, got %v", err)
			808	+	}
			809	+	if e.plumb.Live() != 1 {
			810	+	t.Fatal("the firewall policy must stay while the container may run")
			811	+	}
			812	+	e.eng.mu.Lock()
			813	+	nets := len(e.eng.networks)
			814	+	e.eng.mu.Unlock()
			815	+	if nets != 1 \|\| e.mounts.count() != 1 {
			816	+	t.Fatalf("the bridge and the scratch mount must stay: nets=%d mounts=%d", nets, e.mounts.count())
			817	+	}
			818	+	if _, err := os.Stat(filepath.Join(e.cfg.StateDir, "containers", sb.ID(), "scratch.ext4")); err != nil {
			819	+	t.Fatal("the scratch file must stay")
			820	+	}
			821	+	if err := sb.Stop(context.Background()); !errors.Is(err, sandbox.ErrLeaked) {
			822	+	t.Fatalf("second Stop: %v", err)
			823	+	}
			824	+	// C8: the slot is taken, a new sandbox gets another slot, Reap is busy while it runs.
			825	+	other, err := d.New()
			826	+	if err != nil {
			827	+	t.Fatal(err)
			828	+	}
			829	+	if err := d.Reap(context.Background()); !errors.Is(err, sandbox.ErrBusy) {
			830	+	t.Fatalf("Reap while a sandbox exists: %v", err)
			831	+	}
			832	+	_ = other.Stop(context.Background())
			833	+	e.eng.removeErr = nil
			834	+	if err := d.Reap(context.Background()); err != nil {
			835	+	t.Fatal(err)
			836	+	}
			837	+	if left := e.leftovers(); len(left) != 0 {
			838	+	t.Fatalf("leftovers after Reap: %v", left)
			839	+	}
			840	+	if d.slots.InUse() != 0 {
			841	+	t.Fatal("Reap must free the leaked slot")
			842	+	}
			843	+	}
			844	+
			845	+	// A scratch disk that cannot be unmounted keeps its files: nothing is deleted
			846	+	// below a live mount.
			847	+	func TestStopNeverDeletesBelowALiveMount(t *testing.T) {
			848	+	e := newCEnv(t)
			849	+	d := e.driver(t)
			850	+	sb, _ := d.New()
			851	+	conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
			852	+	if err != nil {
			853	+	t.Fatal(err)
			854	+	}
			855	+	stub(t, conn)
			856	+	e.mounts.failUm = errors.New("target is busy")
			857	+	if err := sb.Stop(context.Background()); !errors.Is(err, sandbox.ErrLeaked) {
			858	+	t.Fatalf("got %v", err)
			859	+	}
			860	+	if _, err := os.Stat(filepath.Join(e.cfg.StateDir, "containers", sb.ID(), "scratch.ext4")); err != nil {
			861	+	t.Fatal("files below a mount that is still there were deleted")
			862	+	}
			863	+	e.mounts.failUm = nil
			864	+	if err := d.Reap(context.Background()); err != nil {
			865	+	t.Fatal(err)
			866	+	}
			867	+	if e.mounts.count() != 0 {
			868	+	t.Fatal("Reap must unmount leftover scratch disks")
			869	+	}
			870	+	if left := e.leftovers(); len(left) != 0 {
			871	+	t.Fatalf("leftovers: %v", left)
			872	+	}
			873	+	}
			874	+
			875	+	// C7: Stop aborts a boot in progress instead of waiting for the timeout.
			876	+	func TestStopAbortsABootInProgress(t *testing.T) {
			877	+	e := newCEnv(t)
			878	+	e.eng.noConnect = true
			879	+	o := e.opts()
			880	+	o.ConnectTimeout = time.Minute
			881	+	d, err := New(context.Background(), e.cfg, o)
			882	+	if err != nil {
			883	+	t.Fatal(err)
			884	+	}
			885	+	defer func() { _ = d.Close() }()
			886	+	sb, _ := d.New()
			887	+	started := make(chan error, 1)
			888	+	go func() { _, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"}); started <- err }()
			889	+	time.Sleep(500 * time.Millisecond)
			890	+	t0 := time.Now()
			891	+	if err := sb.Stop(context.Background()); err != nil {
			892	+	t.Fatal(err)
			893	+	}
			894	+	select {
			895	+	case err := <-started:
			896	+	if !errors.Is(err, sandbox.ErrAborted) {
			897	+	t.Fatalf("Start must report the abort, got %v", err)
			898	+	}
			899	+	case <-time.After(10 * time.Second):
			900	+	t.Fatal("Start was not aborted")
			901	+	}
			902	+	if time.Since(t0) > 5*time.Second {
			903	+	t.Fatalf("Stop took %v", time.Since(t0))
			904	+	}
			905	+	if left := e.leftovers(); len(left) != 0 {
			906	+	t.Fatalf("leftovers: %v", left)
			907	+	}
			908	+	}
			909	+
			910	+	// C16: ID names the Docker objects.
			911	+	func TestSandboxIDNamesDockerObjects(t *testing.T) {
			912	+	e := newCEnv(t)
			913	+	d := e.driver(t)
			914	+	sb, _ := d.New()
			915	+	if !sandbox.ValidID(sb.ID()) {
			916	+	t.Fatal(sb.ID())
			917	+	}
			918	+	conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
			919	+	if err != nil {
			920	+	t.Fatal(err)
			921	+	}
			922	+	stub(t, conn)
			923	+	defer func() { _ = sb.Stop(context.Background()) }()
			924	+	e.eng.mu.Lock()
			925	+	defer e.eng.mu.Unlock()
			926	+	for _, s := range e.eng.containers {
			927	+	if s.Name != "ermite-"+sb.ID() {
			928	+	t.Fatalf("container name %q", s.Name)
			929	+	}
			930	+	}
			931	+	}
			932	+
			933	+	// S3: a second runner on the same host is refused whatever its state directory.
			934	+	func TestHostLockAcrossStateDirs(t *testing.T) {
			935	+	e := newCEnv(t)
			936	+	d := e.driver(t)
			937	+	_ = d
			938	+	e2 := newCEnv(t)
			939	+	e2.cfg.LockFile = e.cfg.LockFile // same host, another state directory
			940	+	_, err := New(context.Background(), e2.cfg, e2.opts())
			941	+	if err == nil \|\| !strings.Contains(err.Error(), "another ermite-runner is running on this host") {
			942	+	t.Fatalf("got %v", err)
			943	+	}
			944	+	}
			945	+
			946	+	// C10: a cancelled wait is reported as cancelled, never as a clean exit.
			947	+	func TestAwaitExit(t *testing.T) {
			948	+	res := make(chan container.WaitResponse, 1)
			949	+	errc := make(chan error, 1)
			950	+	res <- container.WaitResponse{StatusCode: 3}
			951	+	if r := <-awaitExit(context.Background(), res, errc); r.Code != 3 \|\| r.Err != nil {
			952	+	t.Fatalf("%+v", r)
			953	+	}
			954	+	errc <- errors.New("daemon gone")
			955	+	if r := <-awaitExit(context.Background(), make(chan container.WaitResponse), errc); r.Code != -1 \|\| r.Err == nil {
			956	+	t.Fatalf("%+v", r)
			957	+	}
			958	+	// Both ready and the context already cancelled: the cancellation wins.
			959	+	ctx, cancel := context.WithCancel(context.Background())
			960	+	cancel()
			961	+	for range 50 {
			962	+	res2 := make(chan container.WaitResponse, 1)
			963	+	res2 <- container.WaitResponse{StatusCode: 0}
			964	+	if r := <-awaitExit(ctx, res2, make(chan error)); r.Err == nil \|\| !errors.Is(r.Err, context.Canceled) \|\| r.Code != -1 {
			965	+	t.Fatalf("a cancelled wait must not look like a clean exit: %+v", r)
			966	+	}
			967	+	}
			968	+	}
			969	+
			970	+	func TestFreeSpaceWarningInContainerMode(t *testing.T) {
			971	+	e := newCEnv(t)
			972	+	var buf bytes.Buffer
			973	+	o := e.opts()
			974	+	o.FreeSpace = func(string) (int64, error) { return 1, nil }
			975	+	d, err := New(logger(&buf), e.cfg, o)
			976	+	if err != nil {
			977	+	t.Fatal(err)
			978	+	}
			979	+	defer func() { _ = d.Close() }()
			980	+	if !strings.Contains(buf.String(), "less free space") {
			981	+	t.Fatalf("no warning: %s", buf.String())
			982	+	}
			983	+	}
			984	+
			985	+	func TestMkfsTooOldRefusedAtStart(t *testing.T) {
			986	+	e := newCEnv(t)
			987	+	run := &cmdrun.Fake{Handle: func(c cmdrun.Call) ([]byte, error) {
			988	+	if c.Name == "mkfs.ext4" {
			989	+	return []byte("mke2fs 1.42.9 (4-Feb-2014)\n"), nil
			990	+	}
			991	+	return nil, nil
			992	+	}}
			993	+	o := e.opts()
			994	+	o.Run = run
			995	+	if _, err := New(context.Background(), e.cfg, o); err == nil \|\| !strings.Contains(err.Error(), "1.43") {
			996	+	t.Fatalf("got %v", err)
			997	+	}
			998	+	}
			999	+
			1000	+	func TestSocketPathTooLongIsExplained(t *testing.T) {
			1001	+	e := newCEnv(t)
			1002	+	e.cfg.StateDir = filepath.Join(e.cfg.StateDir, strings.Repeat("d", 100))
			1003	+	d := e.driver(t)
			1004	+	sb, _ := d.New()
			1005	+	_, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
			1006	+	if err == nil \|\| !strings.Contains(err.Error(), "108-byte limit") && !strings.Contains(err.Error(), "limit of a Unix socket") {
			1007	+	t.Fatalf("got %v", err)
			1008	+	}
			1009	+	if left := e.leftovers(); len(left) != 0 {
			1010	+	t.Fatalf("leftovers: %v", left)
			1011	+	}
			1012	+	}

M internal/sandbox/container/docker_api_test.go +22 -6


@@ -4,6 +4,7 @@ package container
4	4
5	5		import (
6	6		"context"
	7	+	"io"
7	8		"os"
8	9		"os/exec"
9	10		"path/filepath"

@@ -11,14 +12,16 @@ import (
11	12		"testing"
12	13		"time"
13	14
	15	+	"bnema.dev/ermite-runner/internal/cmdrun"
14	16		"bnema.dev/ermite-runner/internal/execstream"
15	17		"bnema.dev/ermite-runner/internal/execstream/execpb"
16	18		"bnema.dev/ermite-runner/internal/sandbox"
	19	+	"bnema.dev/ermite-runner/internal/sandbox/disk"
17	20		"bnema.dev/ermite-runner/internal/sandbox/sandboxtest"
18	21		)
19	22
20	23		// TestDockerAPIWiring runs the real Docker Engine calls (network with a named
21		-	// bridge, labelled volume and container, bind mounts, wait, removal, reaping by
	24	+	// bridge, labelled container, bind mounts, wait, removal, reaping by
22	25		// label) against whatever daemon DOCKER_HOST points at, including a rootless
23	26		// one. It does not test isolation (a rootless daemon cannot run the nested
24	27		// dockerd): the integration test in internal/sandbox/integration does.

@@ -52,9 +55,15 @@ func TestDockerAPIWiring(t *testing.T) {
52	55		Isolation: sandbox.Container, StateDir: filepath.Join(dir, "state"), Capacity: 1, Image: image,
53	56		JobCPUs: 1, JobMemoryMiB: 512, JobDiskMiB: 1024, UIDBase: 900000,
54	57		Network: mustPrefix("10.213.0.0/16"), Executable: helper, DNSPort: 0,
	58	+	LockFile: filepath.Join(dir, "host.lock"), JobPids: 4096,
55	59		}
56	60		ctx := context.Background()
57		-	d, err := New(ctx, cfg, Options{Net: plumb, allowRootless: true, ConnectTimeout: 60 * time.Second})
	61	+	// Loop mounts need root: replace them by plain directories (mkfs still runs).
	62	+	run := noMountRunner{}
	63	+	empty := filepath.Join(dir, "mountinfo")
	64	+	_ = os.WriteFile(empty, nil, 0o644)
	65	+	d, err := New(ctx, cfg, Options{Net: plumb, allowRootless: true, ConnectTimeout: 60 * time.Second, Run: run,
	66	+	Mounter: &disk.Mounter{Run: run, Mountinfo: empty}})
58	67		if err != nil {
59	68		t.Fatal(err)
60	69		}

@@ -88,9 +97,6 @@ func TestDockerAPIWiring(t *testing.T) {
88	97		if n := dockerLines(t, "network", "ls", "-q", "--filter", "label="+sandbox.LabelJob); len(n) != 1 {
89	98		t.Fatalf("expected one labelled network, got %v", n)
90	99		}
91		-	if v := dockerLines(t, "volume", "ls", "-q", "--filter", "label="+sandbox.LabelJob); len(v) != 1 {
92		-	t.Fatalf("expected one labelled volume, got %v", v)
93		-	}
94	100		}
95	101		if err := sb.Stop(ctx); err != nil {
96	102		t.Fatal(err)

@@ -99,7 +105,6 @@ func TestDockerAPIWiring(t *testing.T) {
99	105		for _, args := range [][]string{
100	106		{"ps", "-aq", "--filter", "label=" + sandbox.LabelJob},
101	107		{"network", "ls", "-q", "--filter", "label=" + sandbox.LabelJob},
102		-	{"volume", "ls", "-q", "--filter", "label=" + sandbox.LabelJob},
103	108		} {
104	109		if l := dockerLines(t, args...); len(l) != 0 {
105	110		t.Fatalf("docker %v left %v", args, l)

@@ -110,6 +115,17 @@ func TestDockerAPIWiring(t *testing.T) {
110	115		}
111	116		}
112	117
	118	+	// noMountRunner runs real commands except mount and umount, which succeed
	119	+	// without doing anything.
	120	+	type noMountRunner struct{}
	121	+
	122	+	func (noMountRunner) Run(ctx context.Context, in io.Reader, name string, args ...string) ([]byte, error) {
	123	+	if name == "mount" \|\| name == "umount" {
	124	+	return nil, nil
	125	+	}
	126	+	return cmdrun.Exec{}.Run(ctx, in, name, args...)
	127	+	}
	128	+
113	129		func dockerLines(t *testing.T, args ...string) []string {
114	130		t.Helper()
115	131		out, err := exec.Command("docker", args...).Output()

 import (
 	"context"
+	"io"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"testing"
 	"time"
+	"bnema.dev/ermite-runner/internal/cmdrun"
 	"bnema.dev/ermite-runner/internal/execstream"
 	"bnema.dev/ermite-runner/internal/execstream/execpb"
 	"bnema.dev/ermite-runner/internal/sandbox"
+	"bnema.dev/ermite-runner/internal/sandbox/disk"
 	"bnema.dev/ermite-runner/internal/sandbox/sandboxtest"
 )
 // TestDockerAPIWiring runs the real Docker Engine calls (network with a named
-// bridge, labelled volume and container, bind mounts, wait, removal, reaping by
+// bridge, labelled container, bind mounts, wait, removal, reaping by
 // label) against whatever daemon DOCKER_HOST points at, including a rootless
 // one. It does not test isolation (a rootless daemon cannot run the nested
 // dockerd): the integration test in internal/sandbox/integration does.
 		Isolation: sandbox.Container, StateDir: filepath.Join(dir, "state"), Capacity: 1, Image: image,
 		JobCPUs: 1, JobMemoryMiB: 512, JobDiskMiB: 1024, UIDBase: 900000,
 		Network: mustPrefix("10.213.0.0/16"), Executable: helper, DNSPort: 0,
+		LockFile: filepath.Join(dir, "host.lock"), JobPids: 4096,
 	}
 	ctx := context.Background()
-	d, err := New(ctx, cfg, Options{Net: plumb, allowRootless: true, ConnectTimeout: 60 * time.Second})
+	// Loop mounts need root: replace them by plain directories (mkfs still runs).
+	run := noMountRunner{}
+	empty := filepath.Join(dir, "mountinfo")
+	_ = os.WriteFile(empty, nil, 0o644)
+	d, err := New(ctx, cfg, Options{Net: plumb, allowRootless: true, ConnectTimeout: 60 * time.Second, Run: run,
+		Mounter: &disk.Mounter{Run: run, Mountinfo: empty}})
 	if err != nil {
 		t.Fatal(err)
 	}
 			if n := dockerLines(t, "network", "ls", "-q", "--filter", "label="+sandbox.LabelJob); len(n) != 1 {
 				t.Fatalf("expected one labelled network, got %v", n)
 			}
-			if v := dockerLines(t, "volume", "ls", "-q", "--filter", "label="+sandbox.LabelJob); len(v) != 1 {
-				t.Fatalf("expected one labelled volume, got %v", v)
-			}
 		}
 		if err := sb.Stop(ctx); err != nil {
 			t.Fatal(err)
 	for _, args := range [][]string{
 		{"ps", "-aq", "--filter", "label=" + sandbox.LabelJob},
 		{"network", "ls", "-q", "--filter", "label=" + sandbox.LabelJob},
-		{"volume", "ls", "-q", "--filter", "label=" + sandbox.LabelJob},
 	} {
 		if l := dockerLines(t, args...); len(l) != 0 {
 			t.Fatalf("docker %v left %v", args, l)
 	}
 }
+// noMountRunner runs real commands except mount and umount, which succeed
+// without doing anything.
+type noMountRunner struct{}
+func (noMountRunner) Run(ctx context.Context, in io.Reader, name string, args ...string) ([]byte, error) {
+	if name == "mount" || name == "umount" {
+		return nil, nil
+	}
+	return cmdrun.Exec{}.Run(ctx, in, name, args...)
+}
 func dockerLines(t *testing.T, args ...string) []string {
 	t.Helper()
 	out, err := exec.Command("docker", args...).Output()

M internal/sandbox/container/driver.go +292 -102


@@ -3,8 +3,14 @@
3	3		// Package container is the container sandbox, for hosts without KVM: one
4	4		// container per job with its own Docker daemon inside (the sysbox runtime when
5	5		// the daemon lists it, else a privileged container), on its own bridge network
6		-	// and with its own scratch volume. Everything Docker-side carries the label
7		-	// sandbox.LabelJob so that Reap finds it after a crash.
	6	+	// and with its own scratch disk. The scratch disk is a sparse ext4 file of
	7	+	// ERMITE_RUNNER_JOB_DISK, loop-mounted on the host (so it needs root, like the
	8	+	// rest of this mode) and bind-mounted at /scratch: the nested dockerd keeps its
	9	+	// data root there and the job workspace is a directory of it, so a job cannot
	10	+	// write more than its disk size to the host (the container's own writable layer
	11	+	// stays small: nothing big is written outside /scratch and /home/runner/work).
	12	+	// Everything Docker-side carries the label sandbox.LabelJob so that Reap finds
	13	+	// it after a crash.
8	14		//
9	15		// The executor stream is a Unix socket in a per-job directory of the runner's
10	16		// state directory, bind-mounted into the container. The runner executable is

@@ -21,6 +27,7 @@ import (
21	27		"os"
22	28		"path/filepath"
23	29		"slices"
	30	+	"strings"
24	31		"sync"
25	32		"time"
26	33

@@ -29,6 +36,7 @@ import (
29	36		"bnema.dev/ermite-runner/internal/cmdrun"
30	37		"bnema.dev/ermite-runner/internal/netpolicy"
31	38		"bnema.dev/ermite-runner/internal/sandbox"
	39	+	"bnema.dev/ermite-runner/internal/sandbox/disk"
32	40		"bnema.dev/ermite-runner/internal/sandbox/netsetup"
33	41		)
34	42

@@ -52,6 +60,10 @@ type Options struct {
52	60		// ConnectTimeout bounds container start + init + executor connection
53	61		// (default 90 s).
54	62		ConnectTimeout time.Duration
	63	+	// Mounter overrides the loop mounts (tests).
	64	+	Mounter *disk.Mounter
	65	+	// FreeSpace overrides statfs for the free-space warning (tests).
	66	+	FreeSpace func(string) (int64, error)
55	67		}
56	68
57	69		// Driver implements sandbox.Driver for containers.

@@ -60,6 +72,8 @@ type Driver struct {
60	72		opt Options
61	73		eng engine
62	74		net netsetup.Plumbing
	75	+	mnt disk.Mounter
	76	+	lock *sandbox.HostLock
63	77		image string
64	78		runtime string // "" when privileged
65	79		privileged bool

@@ -69,8 +83,11 @@ type Driver struct {
69	83		slots *sandbox.Slots
70	84		}
71	85
72		-	// New connects to the Docker daemon (DOCKER_HOST honoured) and checks it: not
73		-	// rootless, the job image present (pulled if missing), the runtime decided.
	86	+	// New takes the host-wide lock, connects to the Docker daemon (DOCKER_HOST
	87	+	// honoured) and checks it: not rootless, the job image present (pulled if
	88	+	// missing), the runtime decided (sysbox when listed; else privileged, unless
	89	+	// ERMITE_RUNNER_DENY_PRIVILEGED=1, which refuses). IPv4 forwarding is enabled
	90	+	// host-wide and stays on (see the README).
74	91		func New(ctx context.Context, cfg sandbox.Config, opt Options) (*Driver, error) {
75	92		if opt.Run == nil {
76	93		opt.Run = cmdrun.Exec{}

@@ -78,25 +95,39 @@ func New(ctx context.Context, cfg sandbox.Config, opt Options) (*Driver, error)
78	95		if opt.ConnectTimeout == 0 {
79	96		opt.ConnectTimeout = 90 * time.Second
80	97		}
	98	+	d := &Driver{cfg: cfg, opt: opt, image: cfg.OCIImage(), slots: sandbox.NewSlots(cfg.Capacity)}
	99	+	if opt.Mounter != nil {
	100	+	d.mnt = *opt.Mounter
	101	+	} else {
	102	+	d.mnt = disk.Mounter{Run: opt.Run}
	103	+	}
	104	+	exe, err := cfg.ExecutablePath()
	105	+	if err != nil {
	106	+	return nil, err
	107	+	}
	108	+	if err := sandbox.CheckStatic(exe); err != nil {
	109	+	return nil, err
	110	+	}
	111	+	d.exe = exe
	112	+
	113	+	if d.lock, err = sandbox.AcquireHostLock(cfg.LockFile); err != nil {
	114	+	return nil, err
	115	+	}
81	116		eng := opt.engine
82	117		if eng == nil {
83	118		e, err := newDockerEngine()
84	119		if err != nil {
	120	+	_ = d.lock.Release()
85	121		return nil, err
86	122		}
87	123		eng = e
88	124		}
89		-	d := &Driver{cfg: cfg, opt: opt, eng: eng, image: cfg.OCIImage(), slots: sandbox.NewSlots(cfg.Capacity)}
90		-	fail := func(err error) (*Driver, error) { _ = eng.Close(); return nil, err }
91		-
92		-	exe, err := cfg.ExecutablePath()
93		-	if err != nil {
94		-	return fail(err)
95		-	}
96		-	if err := sandbox.CheckStatic(exe); err != nil {
97		-	return fail(err)
	125	+	d.eng = eng
	126	+	fail := func(err error) (*Driver, error) {
	127	+	_ = eng.Close()
	128	+	_ = d.lock.Release()
	129	+	return nil, err
98	130		}
99		-	d.exe = exe
100	131
101	132		info, err := eng.Info(ctx)
102	133		if err != nil {

@@ -113,9 +144,19 @@ func New(ctx context.Context, cfg sandbox.Config, opt Options) (*Driver, error)
113	144		d.runtime = SysboxRuntime
114	145		log.Info().Str("runtime", SysboxRuntime).Msg("container isolation: using the sysbox runtime")
115	146		} else {
	147	+	if cfg.DenyPrivileged {
	148	+	return fail(fmt.Errorf("the sysbox runtime is not available and %s=1 forbids privileged containers (install sysbox, or unset the setting to accept host-root-equivalent jobs)", sandbox.EnvDenyPrivileged))
	149	+	}
116	150		d.privileged = true
117		-	log.Warn().Msg("container isolation: sysbox-runc not available, jobs run in PRIVILEGED containers, which is host-root equivalent; their network policy is best effort. Prefer vm isolation or install sysbox")
	151	+	log.Warn().Msg(privilegedWarning)
118	152		}
	153	+	if err := disk.CheckMkfs(ctx, opt.Run); err != nil {
	154	+	return fail(err)
	155	+	}
	156	+	if err := os.MkdirAll(cfg.StateDir, 0o755); err != nil {
	157	+	return fail(fmt.Errorf("state directory: %w", err))
	158	+	}
	159	+	sandbox.WarnIfLowSpace(ctx, cfg.StateDir, cfg.Capacity, cfg.JobDiskMiB, opt.FreeSpace)
119	160		if err := eng.EnsureImage(ctx, d.image); err != nil {
120	161		return fail(err)
121	162		}

@@ -129,9 +170,24 @@ func New(ctx context.Context, cfg sandbox.Config, opt Options) (*Driver, error)
129	170		return d, nil
130	171		}
131	172
	173	+	// maxSockPath is the longest Unix socket path Linux accepts (108 with the NUL).
	174	+	const maxSockPath = 107
	175	+
	176	+	// privilegedWarning is logged when the driver starts and again at every job
	177	+	// start: a privileged container is host-root equivalent.
	178	+	const privilegedWarning = "privileged container: host-root equivalent, network policy best effort"
	179	+
132	180		// Mode implements sandbox.Driver.
133	181		func (d *Driver) Mode() sandbox.Mode { return sandbox.Container }
134	182
	183	+	// Isolation implements sandbox.Driver.
	184	+	func (d *Driver) Isolation() sandbox.Isolation {
	185	+	if d.privileged {
	186	+	return sandbox.IsolationContainerPrivileged
	187	+	}
	188	+	return sandbox.IsolationContainerSysbox
	189	+	}
	190	+
135	191		// Privileged reports whether jobs run privileged (no sysbox).
136	192		func (d *Driver) Privileged() bool { return d.privileged }
137	193

@@ -140,18 +196,23 @@ func (d *Driver) Close() error {
140	196		if d.net != nil {
141	197		d.net.Close()
142	198		}
143		-	return d.eng.Close()
	199	+	err := d.eng.Close()
	200	+	return errors.Join(err, d.lock.Release())
144	201		}
145	202
146	203		// New implements sandbox.Driver.
147	204		func (d *Driver) New() (sandbox.Sandbox, error) {
	205	+	id, err := sandbox.NewID()
	206	+	if err != nil {
	207	+	return nil, err
	208	+	}
148	209		d.mu.Lock()
149	210		defer d.mu.Unlock()
150	211		slot, err := d.slots.Acquire()
151	212		if err != nil {
152	213		return nil, err
153	214		}
154		-	return &containerSandbox{d: d, slot: slot}, nil
	215	+	return &containerSandbox{d: d, slot: slot, id: id}, nil
155	216		}
156	217
157	218		func (d *Driver) release(slot int) {

@@ -160,19 +221,44 @@ func (d *Driver) release(slot int) {
160	221		d.mu.Unlock()
161	222		}
162	223
	224	+	func (d *Driver) markLeaked(slot int) {
	225	+	d.mu.Lock()
	226	+	d.slots.MarkLeaked(slot)
	227	+	d.mu.Unlock()
	228	+	}
	229	+
163	230		func (d *Driver) runDir(id string) string { return filepath.Join(d.cfg.StateDir, "containers", id) }
164	231
	232	+	// sockDir holds only the executor socket: it is the one directory of the job
	233	+	// that is bind-mounted (as /run/ermite), so the job never sees the scratch disk
	234	+	// file, nor the other host paths of its own sandbox.
	235	+	func (d *Driver) sockDir(id string) string { return filepath.Join(d.runDir(id), "sock") }
	236	+
	237	+	// scratchDir is where the loop-mounted scratch disk of a job is mounted.
	238	+	func (d *Driver) scratchDir(id string) string { return filepath.Join(d.runDir(id), "scratch") }
	239	+
	240	+	// workDir is the job workspace, a directory of the scratch disk.
	241	+	func (d *Driver) workDir(id string) string { return filepath.Join(d.scratchDir(id), "work") }
	242	+
	243	+	// Where the job workspace and its temporary files appear in the container.
	244	+	const (
	245	+	containerWorkspace = "/home/runner/work"
	246	+	containerTmp = "/tmp"
	247	+	)
	248	+
165	249		type containerSandbox struct {
166	250		d *Driver
167	251		slot int
	252	+	id string
	253	+	gate sandbox.BootGate
168	254
169	255		mu sync.Mutex
170	256		started bool
171	257		stopped bool
172		-	id string
	258	+	leakErr error
173	259		netID string
174		-	volume string
175	260		container string
	261	+	mounted bool
176	262		listener net.Listener
177	263		conn net.Conn
178	264		iface string

@@ -180,8 +266,18 @@ type containerSandbox struct {
180	266		attached bool
181	267		}
182	268
	269	+	// ID implements sandbox.Sandbox.
	270	+	func (s *containerSandbox) ID() string { return s.id }
	271	+
183	272		// Start implements sandbox.Sandbox.
184	273		func (s *containerSandbox) Start(ctx context.Context, spec sandbox.JobSpec) (sandbox.Conn, error) {
	274	+	// A Stop from another goroutine aborts the boot through the gate.
	275	+	bootCtx, err := s.gate.Begin(ctx)
	276	+	if err != nil {
	277	+	return nil, fmt.Errorf("container sandbox: %w", err)
	278	+	}
	279	+	defer s.gate.End()
	280	+
185	281		s.mu.Lock()
186	282		defer s.mu.Unlock()
187	283		if s.started \|\| s.stopped {

@@ -191,14 +287,12 @@ func (s *containerSandbox) Start(ctx context.Context, spec sandbox.JobSpec) (san
191	287		d := s.d
192	288		log := zerowrap.FromCtx(ctx)
193	289		spec = d.cfg.Resolve(spec)
194		-
195		-	id, err := sandbox.NewID()
196		-	if err != nil {
197		-	s.releaseLocked()
198		-	return nil, err
	290	+	if d.privileged {
	291	+	// At every job start, not just at driver start: the job log and the
	292	+	// operator's logs must say what kind of isolation this job got.
	293	+	log.Warn().Str("job", spec.JobID).Str("container", s.id).Msg(privilegedWarning)
199	294		}
200		-	s.id = id
201		-	conn, err := s.run(ctx, spec)
	295	+	conn, err := s.run(bootCtx, spec)
202	296		if err != nil {
203	297		var tail string
204	298		if s.container != "" {

@@ -214,25 +308,54 @@ func (s *containerSandbox) Start(ctx context.Context, spec sandbox.JobSpec) (san
214	308		return conn, nil
215	309		}
216	310
	311	+	// run runs the boot stages in order; each records what it created for cleanup.
217	312		func (s *containerSandbox) run(ctx context.Context, spec sandbox.JobSpec) (sandbox.Conn, error) {
218		-	d := s.d
219		-	netw, err := netpolicy.Subnet(d.cfg.Network, s.slot)
	313	+	netw, err := netpolicy.Subnet(s.d.cfg.Network, s.slot)
	314	+	if err != nil {
	315	+	return nil, err
	316	+	}
	317	+	accepted, err := s.runSocket()
220	318		if err != nil {
221	319		return nil, err
222	320		}
	321	+	if err := s.runScratch(ctx, spec); err != nil {
	322	+	return nil, err
	323	+	}
223	324		labels := map[string]string{sandbox.LabelJob: spec.JobID}
224		-	name := "ermite-" + s.id
225		-
226		-	// Per-job directory with the executor socket. It is created before anything
227		-	// on the Docker side so that a crash leaves a recognisable directory.
228		-	dir := d.runDir(s.id)
229		-	if err := os.MkdirAll(dir, 0o711); err != nil {
	325	+	if err := s.runNetwork(ctx, netw, labels); err != nil {
230	326		return nil, err
231	327		}
232		-	if err := os.Chmod(dir, 0o711); err != nil {
	328	+	exited, err := s.runContainer(ctx, spec, netw, labels)
	329	+	if err != nil {
233	330		return nil, err
234	331		}
	332	+	return s.waitExecutor(ctx, accepted, exited)
	333	+	}
	334	+
	335	+	type acceptResult struct {
	336	+	c net.Conn
	337	+	err error
	338	+	}
	339	+
	340	+	// runSocket creates the per-job directory and listens on the executor socket in
	341	+	// it. The directory exists before anything on the Docker side, so a crash
	342	+	// leaves a recognisable one.
	343	+	func (s *containerSandbox) runSocket() (<-chan acceptResult, error) {
	344	+	d := s.d
	345	+	for _, dir := range []string{d.runDir(s.id), d.sockDir(s.id)} {
	346	+	if err := os.MkdirAll(dir, 0o711); err != nil {
	347	+	return nil, err
	348	+	}
	349	+	if err := os.Chmod(dir, 0o711); err != nil {
	350	+	return nil, err
	351	+	}
	352	+	}
	353	+	dir := d.sockDir(s.id)
235	354		sock := filepath.Join(dir, sandbox.ContainerSocket)
	355	+	// sockaddr_un.sun_path holds 108 bytes: say so instead of "invalid argument".
	356	+	if len(sock) > maxSockPath {
	357	+	return nil, fmt.Errorf("executor socket path %q is %d bytes, over the %d-byte limit of a Unix socket: use a shorter %s", sock, len(sock), maxSockPath, sandbox.EnvStateDir)
	358	+	}
236	359		ln, err := net.Listen("unix", sock)
237	360		if err != nil {
238	361		return nil, fmt.Errorf("listen executor socket: %w", err)

@@ -243,55 +366,79 @@ func (s *containerSandbox) run(ctx context.Context, spec sandbox.JobSpec) (sandb
243	366		if err := os.Chmod(sock, 0o666); err != nil {
244	367		return nil, err
245	368		}
246		-	accepted := make(chan struct {
247		-	c net.Conn
248		-	err error
249		-	}, 1)
	369	+	accepted := make(chan acceptResult, 1)
250	370		go func() {
251	371		c, err := ln.Accept()
252		-	accepted <- struct {
253		-	c net.Conn
254		-	err error
255		-	}{c, err}
	372	+	accepted <- acceptResult{c, err}
256	373		}()
	374	+	return accepted, nil
	375	+	}
	376	+
	377	+	// runScratch creates the sparse ext4 scratch disk (ERMITE_RUNNER_JOB_DISK) and
	378	+	// loop-mounts it; the workspace directory lives on it.
	379	+	func (s *containerSandbox) runScratch(ctx context.Context, spec sandbox.JobSpec) error {
	380	+	d := s.d
	381	+	file := filepath.Join(d.runDir(s.id), "scratch.ext4")
	382	+	if err := disk.CreateScratch(ctx, d.opt.Run, file, spec.DiskMiB, -1); err != nil {
	383	+	return err
	384	+	}
	385	+	mnt := d.scratchDir(s.id)
	386	+	if err := os.MkdirAll(mnt, 0o755); err != nil {
	387	+	return err
	388	+	}
	389	+	if err := d.mnt.Mount(ctx, file, mnt); err != nil {
	390	+	return err
	391	+	}
	392	+	s.mounted = true
	393	+	// Workspace, temp and the nested dockerd's data root on the scratch disk. The
	394	+	// executor (phase G) sets the job layout and ownership inside the sandbox.
	395	+	for _, sub := range []string{"work", "tmp", "docker"} {
	396	+	if err := os.MkdirAll(filepath.Join(mnt, sub), 0o755); err != nil {
	397	+	return err
	398	+	}
	399	+	}
	400	+	return os.Chmod(filepath.Join(mnt, "tmp"), 0o1777)
	401	+	}
257	402
	403	+	// runNetwork creates the job's bridge and installs the firewall policy.
	404	+	func (s *containerSandbox) runNetwork(ctx context.Context, netw netpolicy.Net, labels map[string]string) error {
	405	+	d := s.d
258	406		s.iface = fmt.Sprintf("%s%d", bridgePrefix, s.slot)
259	407		s.gateway = netw.Gateway
260		-	netID, err := d.eng.CreateNetwork(ctx, networkSpec{Name: name, Bridge: s.iface, Subnet: netw.Prefix, Gateway: netw.Gateway, Labels: labels})
	408	+	netID, err := d.eng.CreateNetwork(ctx, networkSpec{Name: "ermite-" + s.id, Bridge: s.iface, Subnet: netw.Prefix, Gateway: netw.Gateway, Labels: labels})
261	409		if err != nil {
262		-	return nil, err
	410	+	return err
263	411		}
264	412		s.netID = netID
265	413		s.attached = true
266		-	if err := d.net.Attach(ctx, netpolicy.Guest{Iface: s.iface, Gateway: netw.Gateway, Addr: netw.Guest}); err != nil {
267		-	return nil, err
268		-	}
269		-
270		-	s.volume = name
271		-	if err := d.eng.CreateVolume(ctx, name, labels); err != nil {
272		-	s.volume = ""
273		-	return nil, fmt.Errorf("create volume: %w", err)
274		-	}
	414	+	return d.net.Attach(ctx, netpolicy.Guest{Iface: s.iface, Gateway: netw.Gateway, Addr: netw.Guest})
	415	+	}
275	416
	417	+	// runContainer creates and starts the container.
	418	+	func (s *containerSandbox) runContainer(ctx context.Context, spec sandbox.JobSpec, netw netpolicy.Net, labels map[string]string) (<-chan exitResult, error) {
	419	+	d := s.d
276	420		entry := sandbox.GuestExecDir + "/" + sandbox.GuestExecName
277	421		cid, err := d.eng.CreateContainer(ctx, containerSpec{
278		-	Name: name,
	422	+	Name: "ermite-" + s.id,
279	423		Image: d.image,
280	424		Entrypoint: []string{entry, "init"},
281	425		Env: []string{"ERMITE_CONNECT=unix:" + sandbox.ContainerRunDir + "/" + sandbox.ContainerSocket},
282	426		Labels: labels,
283	427		Runtime: d.runtime,
284	428		Privileged: d.privileged,
285		-	Network: name,
	429	+	Network: "ermite-" + s.id,
286	430		IPv4: netw.Guest,
287	431		DNS: netw.Gateway,
288	432		Binds: []bindMount{
289	433		{Source: d.exe, Target: entry, ReadOnly: true},
290		-	{Source: dir, Target: sandbox.ContainerRunDir},
	434	+	{Source: d.sockDir(s.id), Target: sandbox.ContainerRunDir},
	435	+	{Source: d.scratchDir(s.id), Target: sandbox.GuestScratch},
	436	+	{Source: d.workDir(s.id), Target: containerWorkspace},
	437	+	{Source: filepath.Join(d.scratchDir(s.id), "tmp"), Target: containerTmp},
291	438		},
292		-	Volumes: []volumeMount{{Name: name, Target: sandbox.GuestScratch}},
293	439		MemoryMiB: spec.MemoryMiB,
294	440		CPUs: spec.CPUs,
	441	+	Pids: d.cfg.JobPids,
295	442		})
296	443		if err != nil {
297	444		return nil, err

@@ -301,8 +448,14 @@ func (s *containerSandbox) run(ctx context.Context, spec sandbox.JobSpec) (sandb
301	448		if err := d.eng.StartContainer(ctx, cid); err != nil {
302	449		return nil, fmt.Errorf("start container: %w", err)
303	450		}
	451	+	return exited, nil
	452	+	}
304	453
305		-	timer := time.NewTimer(d.opt.ConnectTimeout)
	454	+	// waitExecutor waits for the executor to connect. Only ctx (the caller's
	455	+	// context, or Stop through the boot gate), the container exiting, and the
	456	+	// timeout end the wait.
	457	+	func (s *containerSandbox) waitExecutor(ctx context.Context, accepted <-chan acceptResult, exited <-chan exitResult) (sandbox.Conn, error) {
	458	+	timer := time.NewTimer(s.d.opt.ConnectTimeout)
306	459		defer timer.Stop()
307	460		select {
308	461		case r := <-accepted:

@@ -312,20 +465,25 @@ func (s *containerSandbox) run(ctx context.Context, spec sandbox.JobSpec) (sandb
312	465		s.conn = r.c
313	466		return r.c, nil
314	467		case ex := <-exited:
	468	+	if ex.Err != nil {
	469	+	return nil, fmt.Errorf("waiting for the container: %w", ex.Err)
	470	+	}
315	471		return nil, fmt.Errorf("container exited (code %d) before the executor connected", ex.Code)
316	472		case <-timer.C:
317		-	return nil, fmt.Errorf("executor did not connect within %s", d.opt.ConnectTimeout)
	473	+	return nil, fmt.Errorf("executor did not connect within %s", s.d.opt.ConnectTimeout)
318	474		case <-ctx.Done():
319		-	return nil, ctx.Err()
	475	+	return nil, context.Cause(ctx)
320	476		}
321	477		}
322	478
323	479		// Stop implements sandbox.Sandbox.
324	480		func (s *containerSandbox) Stop(ctx context.Context) error {
	481	+	// Abort a boot in progress first: Start holds s.mu for its whole duration.
	482	+	s.gate.Abort()
325	483		s.mu.Lock()
326	484		defer s.mu.Unlock()
327	485		if s.stopped {
328		-	return nil
	486	+	return s.leakErr
329	487		}
330	488		if !s.started {
331	489		s.releaseLocked()

@@ -339,10 +497,11 @@ func (s *containerSandbox) releaseLocked() {
339	497		s.d.release(s.slot)
340	498		}
341	499
342		-	// cleanupLocked removes the container, then its network, volume, firewall rules
343		-	// and directory, continuing after errors. The slot is released only when
344		-	// everything is gone (so its bridge name and subnet are not reused while the
345		-	// old ones still exist); leftovers are reaped by label at the next start.
	500	+	// cleanupLocked removes the container first; only when it is confirmed gone are
	501	+	// the firewall policy, the network, the scratch mount and the files removed. A
	502	+	// container that may still run keeps its policy and bridge: tearing the policy
	503	+	// down under it would remove the isolation it relies on. Failures mark the
	504	+	// sandbox as leaked (sandbox.ErrLeaked): its slot stays taken until Reap.
346	505		func (s *containerSandbox) cleanupLocked(ctx context.Context) error {
347	506		d := s.d
348	507		var errs []error

@@ -361,45 +520,59 @@ func (s *containerSandbox) cleanupLocked(ctx context.Context) error {
361	520		s.container = ""
362	521		}
363	522		}
364		-	if s.attached {
365		-	if err := d.net.Detach(ctx, s.iface, s.gateway); err != nil {
366		-	errs = append(errs, err)
367		-	} else {
368		-	s.attached = false
	523	+	if s.container == "" {
	524	+	if s.attached {
	525	+	if err := d.net.Detach(ctx, s.iface, s.gateway); err != nil {
	526	+	errs = append(errs, err)
	527	+	} else {
	528	+	s.attached = false
	529	+	}
369	530		}
370		-	}
371		-	if s.netID != "" && s.container == "" {
372		-	if err := d.eng.RemoveNetwork(ctx, s.netID); err != nil {
373		-	errs = append(errs, fmt.Errorf("remove network: %w", err))
374		-	} else {
375		-	s.netID = ""
	531	+	if s.netID != "" {
	532	+	if err := d.eng.RemoveNetwork(ctx, s.netID); err != nil {
	533	+	errs = append(errs, fmt.Errorf("remove network: %w", err))
	534	+	} else {
	535	+	s.netID = ""
	536	+	}
376	537		}
377		-	}
378		-	if s.volume != "" && s.container == "" {
379		-	if err := d.eng.RemoveVolume(ctx, s.volume); err != nil {
380		-	errs = append(errs, fmt.Errorf("remove volume: %w", err))
381		-	} else {
382		-	s.volume = ""
	538	+	if s.mounted {
	539	+	if err := d.mnt.Unmount(ctx, d.scratchDir(s.id)); err != nil {
	540	+	errs = append(errs, err)
	541	+	} else {
	542	+	s.mounted = false
	543	+	}
	544	+	}
	545	+	// Never delete files below a mount that is still there.
	546	+	if !s.mounted {
	547	+	if err := os.RemoveAll(d.runDir(s.id)); err != nil {
	548	+	errs = append(errs, err)
	549	+	}
383	550		}
384		-	}
385		-	if err := os.RemoveAll(d.runDir(s.id)); err != nil {
386		-	errs = append(errs, err)
387	551		}
388	552		if len(errs) == 0 {
389	553		s.releaseLocked()
390	554		return nil
391	555		}
392	556		s.stopped = true
393		-	err := errors.Join(errs...)
	557	+	d.markLeaked(s.slot)
	558	+	s.leakErr = fmt.Errorf("%w: container sandbox %s: %w", sandbox.ErrLeaked, s.id, errors.Join(errs...))
394	559		log := zerowrap.FromCtx(ctx)
395		-	log.Error().Err(err).Str("job", s.id).Msg("container cleanup incomplete; the next runner start will retry")
396		-	return err
	560	+	log.Error().Err(s.leakErr).Str("job", s.id).Msg("container cleanup incomplete; call Reap to finish it")
	561	+	return s.leakErr
397	562		}
398	563
399		-	// Reap implements sandbox.Driver: remove every container, network and volume
400		-	// carrying the job label, the nftables table, and the per-job directories.
	564	+	// Reap implements sandbox.Driver: remove every container and network carrying
	565	+	// the job label, the nftables table, the scratch mounts and the per-job
	566	+	// directories, and free the slots of leaked sandboxes. It returns ErrBusy while
	567	+	// a sandbox is running.
401	568		func (d *Driver) Reap(ctx context.Context) error {
402	569		log := zerowrap.FromCtx(ctx)
	570	+	d.mu.Lock()
	571	+	busy := d.slots.Busy()
	572	+	d.mu.Unlock()
	573	+	if busy > 0 {
	574	+	return fmt.Errorf("%w (%d running)", sandbox.ErrBusy, busy)
	575	+	}
403	576		var errs []error
404	577		ids, err := d.eng.ListContainers(ctx, sandbox.LabelJob)
405	578		if err != nil {

@@ -421,27 +594,44 @@ func (d *Driver) Reap(ctx context.Context) error {
421	594		errs = append(errs, err)
422	595		}
423	596		}
424		-	vols, err := d.eng.ListVolumes(ctx, sandbox.LabelJob)
425		-	if err != nil {
426		-	errs = append(errs, fmt.Errorf("list volumes: %w", err))
427		-	}
428		-	for _, v := range vols {
429		-	log.Warn().Str("volume", v).Msg("removing leftover job volume")
430		-	if err := d.eng.RemoveVolume(ctx, v); err != nil {
431		-	errs = append(errs, err)
432		-	}
433		-	}
434	597		if err := d.net.Reset(ctx); err != nil {
435	598		errs = append(errs, fmt.Errorf("reset nftables table: %w", err))
436	599		}
	600	+	// Unmount before deleting: a directory with a live mount below it must not
	601	+	// be removed (and EBUSY would be the best case).
437	602		root := filepath.Join(d.cfg.StateDir, "containers")
	603	+	mounts, merr := d.mnt.MountPoints()
	604	+	if merr != nil {
	605	+	errs = append(errs, fmt.Errorf("read mounts: %w", merr))
	606	+	}
	607	+	stuck := map[string]bool{}
	608	+	for _, mp := range mounts {
	609	+	rel, ok := strings.CutPrefix(mp, root+"/")
	610	+	if !ok {
	611	+	continue
	612	+	}
	613	+	id, _, _ := strings.Cut(rel, "/")
	614	+	if !sandbox.ValidID(id) {
	615	+	continue
	616	+	}
	617	+	log.Warn().Str("mount", mp).Msg("unmounting leftover scratch disk")
	618	+	if err := d.mnt.Unmount(ctx, mp); err != nil {
	619	+	stuck[id] = true
	620	+	errs = append(errs, err)
	621	+	}
	622	+	}
438	623		ents, _ := os.ReadDir(root)
439	624		for _, e := range ents {
440		-	if sandbox.ValidID(e.Name()) {
	625	+	if sandbox.ValidID(e.Name()) && !stuck[e.Name()] {
441	626		if err := os.RemoveAll(filepath.Join(root, e.Name())); err != nil {
442	627		errs = append(errs, err)
443	628		}
444	629		}
445	630		}
	631	+	if len(errs) == 0 {
	632	+	d.mu.Lock()
	633	+	d.slots.ReleaseLeaked()
	634	+	d.mu.Unlock()
	635	+	}
446	636		return errors.Join(errs...)
447	637		}


@@ -3,8 +3,14 @@
3		// Package container is the container sandbox, for hosts without KVM: one	3		// Package container is the container sandbox, for hosts without KVM: one
4		// container per job with its own Docker daemon inside (the sysbox runtime when	4		// container per job with its own Docker daemon inside (the sysbox runtime when
5		// the daemon lists it, else a privileged container), on its own bridge network	5		// the daemon lists it, else a privileged container), on its own bridge network
6	-	// and with its own scratch volume. Everything Docker-side carries the label	6	+	// and with its own scratch disk. The scratch disk is a sparse ext4 file of
7	-	// sandbox.LabelJob so that Reap finds it after a crash.	7	+	// ERMITE_RUNNER_JOB_DISK, loop-mounted on the host (so it needs root, like the
			8	+	// rest of this mode) and bind-mounted at /scratch: the nested dockerd keeps its
			9	+	// data root there and the job workspace is a directory of it, so a job cannot
			10	+	// write more than its disk size to the host (the container's own writable layer
			11	+	// stays small: nothing big is written outside /scratch and /home/runner/work).
			12	+	// Everything Docker-side carries the label sandbox.LabelJob so that Reap finds
			13	+	// it after a crash.
8		//	14		//
9		// The executor stream is a Unix socket in a per-job directory of the runner's	15		// The executor stream is a Unix socket in a per-job directory of the runner's
10		// state directory, bind-mounted into the container. The runner executable is	16		// state directory, bind-mounted into the container. The runner executable is

@@ -21,6 +27,7 @@ import (
21		"os"	27		"os"
22		"path/filepath"	28		"path/filepath"
23		"slices"	29		"slices"
			30	+	"strings"
24		"sync"	31		"sync"
25		"time"	32		"time"
26			33

@@ -29,6 +36,7 @@ import (
29		"bnema.dev/ermite-runner/internal/cmdrun"	36		"bnema.dev/ermite-runner/internal/cmdrun"
30		"bnema.dev/ermite-runner/internal/netpolicy"	37		"bnema.dev/ermite-runner/internal/netpolicy"
31		"bnema.dev/ermite-runner/internal/sandbox"	38		"bnema.dev/ermite-runner/internal/sandbox"
			39	+	"bnema.dev/ermite-runner/internal/sandbox/disk"
32		"bnema.dev/ermite-runner/internal/sandbox/netsetup"	40		"bnema.dev/ermite-runner/internal/sandbox/netsetup"
33		)	41		)
34			42

@@ -52,6 +60,10 @@ type Options struct {
52		// ConnectTimeout bounds container start + init + executor connection	60		// ConnectTimeout bounds container start + init + executor connection
53		// (default 90 s).	61		// (default 90 s).
54		ConnectTimeout time.Duration	62		ConnectTimeout time.Duration
			63	+	// Mounter overrides the loop mounts (tests).
			64	+	Mounter *disk.Mounter
			65	+	// FreeSpace overrides statfs for the free-space warning (tests).
			66	+	FreeSpace func(string) (int64, error)
55		}	67		}
56			68
57		// Driver implements sandbox.Driver for containers.	69		// Driver implements sandbox.Driver for containers.

@@ -60,6 +72,8 @@ type Driver struct {
60		opt Options	72		opt Options
61		eng engine	73		eng engine
62		net netsetup.Plumbing	74		net netsetup.Plumbing
			75	+	mnt disk.Mounter
			76	+	lock *sandbox.HostLock
63		image string	77		image string
64		runtime string // "" when privileged	78		runtime string // "" when privileged
65		privileged bool	79		privileged bool

@@ -69,8 +83,11 @@ type Driver struct {
69		slots *sandbox.Slots	83		slots *sandbox.Slots
70		}	84		}
71			85
72	-	// New connects to the Docker daemon (DOCKER_HOST honoured) and checks it: not	86	+	// New takes the host-wide lock, connects to the Docker daemon (DOCKER_HOST
73	-	// rootless, the job image present (pulled if missing), the runtime decided.	87	+	// honoured) and checks it: not rootless, the job image present (pulled if
			88	+	// missing), the runtime decided (sysbox when listed; else privileged, unless
			89	+	// ERMITE_RUNNER_DENY_PRIVILEGED=1, which refuses). IPv4 forwarding is enabled
			90	+	// host-wide and stays on (see the README).
74		func New(ctx context.Context, cfg sandbox.Config, opt Options) (*Driver, error) {	91		func New(ctx context.Context, cfg sandbox.Config, opt Options) (*Driver, error) {
75		if opt.Run == nil {	92		if opt.Run == nil {
76		opt.Run = cmdrun.Exec{}	93		opt.Run = cmdrun.Exec{}

@@ -78,25 +95,39 @@ func New(ctx context.Context, cfg sandbox.Config, opt Options) (*Driver, error)
78		if opt.ConnectTimeout == 0 {	95		if opt.ConnectTimeout == 0 {
79		opt.ConnectTimeout = 90 * time.Second	96		opt.ConnectTimeout = 90 * time.Second
80		}	97		}
			98	+	d := &Driver{cfg: cfg, opt: opt, image: cfg.OCIImage(), slots: sandbox.NewSlots(cfg.Capacity)}
			99	+	if opt.Mounter != nil {
			100	+	d.mnt = *opt.Mounter
			101	+	} else {
			102	+	d.mnt = disk.Mounter{Run: opt.Run}
			103	+	}
			104	+	exe, err := cfg.ExecutablePath()
			105	+	if err != nil {
			106	+	return nil, err
			107	+	}
			108	+	if err := sandbox.CheckStatic(exe); err != nil {
			109	+	return nil, err
			110	+	}
			111	+	d.exe = exe
			112	+
			113	+	if d.lock, err = sandbox.AcquireHostLock(cfg.LockFile); err != nil {
			114	+	return nil, err
			115	+	}
81		eng := opt.engine	116		eng := opt.engine
82		if eng == nil {	117		if eng == nil {
83		e, err := newDockerEngine()	118		e, err := newDockerEngine()
84		if err != nil {	119		if err != nil {
			120	+	_ = d.lock.Release()
85		return nil, err	121		return nil, err
86		}	122		}
87		eng = e	123		eng = e
88		}	124		}
89	-	d := &Driver{cfg: cfg, opt: opt, eng: eng, image: cfg.OCIImage(), slots: sandbox.NewSlots(cfg.Capacity)}	125	+	d.eng = eng
90	-	fail := func(err error) (*Driver, error) { _ = eng.Close(); return nil, err }	126	+	fail := func(err error) (*Driver, error) {
91	-		127	+	_ = eng.Close()
92	-	exe, err := cfg.ExecutablePath()	128	+	_ = d.lock.Release()
93	-	if err != nil {	129	+	return nil, err
94	-	return fail(err)
95	-	}
96	-	if err := sandbox.CheckStatic(exe); err != nil {
97	-	return fail(err)
98		}	130		}
99	-	d.exe = exe
100			131
101		info, err := eng.Info(ctx)	132		info, err := eng.Info(ctx)
102		if err != nil {	133		if err != nil {

@@ -113,9 +144,19 @@ func New(ctx context.Context, cfg sandbox.Config, opt Options) (*Driver, error)
113		d.runtime = SysboxRuntime	144		d.runtime = SysboxRuntime
114		log.Info().Str("runtime", SysboxRuntime).Msg("container isolation: using the sysbox runtime")	145		log.Info().Str("runtime", SysboxRuntime).Msg("container isolation: using the sysbox runtime")
115		} else {	146		} else {
			147	+	if cfg.DenyPrivileged {
			148	+	return fail(fmt.Errorf("the sysbox runtime is not available and %s=1 forbids privileged containers (install sysbox, or unset the setting to accept host-root-equivalent jobs)", sandbox.EnvDenyPrivileged))
			149	+	}
116		d.privileged = true	150		d.privileged = true
117	-	log.Warn().Msg("container isolation: sysbox-runc not available, jobs run in PRIVILEGED containers, which is host-root equivalent; their network policy is best effort. Prefer vm isolation or install sysbox")	151	+	log.Warn().Msg(privilegedWarning)
118		}	152		}
			153	+	if err := disk.CheckMkfs(ctx, opt.Run); err != nil {
			154	+	return fail(err)
			155	+	}
			156	+	if err := os.MkdirAll(cfg.StateDir, 0o755); err != nil {
			157	+	return fail(fmt.Errorf("state directory: %w", err))
			158	+	}
			159	+	sandbox.WarnIfLowSpace(ctx, cfg.StateDir, cfg.Capacity, cfg.JobDiskMiB, opt.FreeSpace)
119		if err := eng.EnsureImage(ctx, d.image); err != nil {	160		if err := eng.EnsureImage(ctx, d.image); err != nil {
120		return fail(err)	161		return fail(err)
121		}	162		}

@@ -129,9 +170,24 @@ func New(ctx context.Context, cfg sandbox.Config, opt Options) (*Driver, error)
129		return d, nil	170		return d, nil
130		}	171		}
131			172
			173	+	// maxSockPath is the longest Unix socket path Linux accepts (108 with the NUL).
			174	+	const maxSockPath = 107
			175	+
			176	+	// privilegedWarning is logged when the driver starts and again at every job
			177	+	// start: a privileged container is host-root equivalent.
			178	+	const privilegedWarning = "privileged container: host-root equivalent, network policy best effort"
			179	+
132		// Mode implements sandbox.Driver.	180		// Mode implements sandbox.Driver.
133		func (d *Driver) Mode() sandbox.Mode { return sandbox.Container }	181		func (d *Driver) Mode() sandbox.Mode { return sandbox.Container }
134			182
			183	+	// Isolation implements sandbox.Driver.
			184	+	func (d *Driver) Isolation() sandbox.Isolation {
			185	+	if d.privileged {
			186	+	return sandbox.IsolationContainerPrivileged
			187	+	}
			188	+	return sandbox.IsolationContainerSysbox
			189	+	}
			190	+
135		// Privileged reports whether jobs run privileged (no sysbox).	191		// Privileged reports whether jobs run privileged (no sysbox).
136		func (d *Driver) Privileged() bool { return d.privileged }	192		func (d *Driver) Privileged() bool { return d.privileged }
137			193

@@ -140,18 +196,23 @@ func (d *Driver) Close() error {
140		if d.net != nil {	196		if d.net != nil {
141		d.net.Close()	197		d.net.Close()
142		}	198		}
143	-	return d.eng.Close()	199	+	err := d.eng.Close()
			200	+	return errors.Join(err, d.lock.Release())
144		}	201		}
145			202
146		// New implements sandbox.Driver.	203		// New implements sandbox.Driver.
147		func (d *Driver) New() (sandbox.Sandbox, error) {	204		func (d *Driver) New() (sandbox.Sandbox, error) {
			205	+	id, err := sandbox.NewID()
			206	+	if err != nil {
			207	+	return nil, err
			208	+	}
148		d.mu.Lock()	209		d.mu.Lock()
149		defer d.mu.Unlock()	210		defer d.mu.Unlock()
150		slot, err := d.slots.Acquire()	211		slot, err := d.slots.Acquire()
151		if err != nil {	212		if err != nil {
152		return nil, err	213		return nil, err
153		}	214		}
154	-	return &containerSandbox{d: d, slot: slot}, nil	215	+	return &containerSandbox{d: d, slot: slot, id: id}, nil
155		}	216		}
156			217
157		func (d *Driver) release(slot int) {	218		func (d *Driver) release(slot int) {

@@ -160,19 +221,44 @@ func (d *Driver) release(slot int) {
160		d.mu.Unlock()	221		d.mu.Unlock()
161		}	222		}
162			223
			224	+	func (d *Driver) markLeaked(slot int) {
			225	+	d.mu.Lock()
			226	+	d.slots.MarkLeaked(slot)
			227	+	d.mu.Unlock()
			228	+	}
			229	+
163		func (d *Driver) runDir(id string) string { return filepath.Join(d.cfg.StateDir, "containers", id) }	230		func (d *Driver) runDir(id string) string { return filepath.Join(d.cfg.StateDir, "containers", id) }
164			231
			232	+	// sockDir holds only the executor socket: it is the one directory of the job
			233	+	// that is bind-mounted (as /run/ermite), so the job never sees the scratch disk
			234	+	// file, nor the other host paths of its own sandbox.
			235	+	func (d *Driver) sockDir(id string) string { return filepath.Join(d.runDir(id), "sock") }
			236	+
			237	+	// scratchDir is where the loop-mounted scratch disk of a job is mounted.
			238	+	func (d *Driver) scratchDir(id string) string { return filepath.Join(d.runDir(id), "scratch") }
			239	+
			240	+	// workDir is the job workspace, a directory of the scratch disk.
			241	+	func (d *Driver) workDir(id string) string { return filepath.Join(d.scratchDir(id), "work") }
			242	+
			243	+	// Where the job workspace and its temporary files appear in the container.
			244	+	const (
			245	+	containerWorkspace = "/home/runner/work"
			246	+	containerTmp = "/tmp"
			247	+	)
			248	+
165		type containerSandbox struct {	249		type containerSandbox struct {
166		d *Driver	250		d *Driver
167		slot int	251		slot int
			252	+	id string
			253	+	gate sandbox.BootGate
168			254
169		mu sync.Mutex	255		mu sync.Mutex
170		started bool	256		started bool
171		stopped bool	257		stopped bool
172	-	id string	258	+	leakErr error
173		netID string	259		netID string
174	-	volume string
175		container string	260		container string
			261	+	mounted bool
176		listener net.Listener	262		listener net.Listener
177		conn net.Conn	263		conn net.Conn
178		iface string	264		iface string

@@ -180,8 +266,18 @@ type containerSandbox struct {
180		attached bool	266		attached bool
181		}	267		}
182			268
			269	+	// ID implements sandbox.Sandbox.
			270	+	func (s *containerSandbox) ID() string { return s.id }
			271	+
183		// Start implements sandbox.Sandbox.	272		// Start implements sandbox.Sandbox.
184		func (s *containerSandbox) Start(ctx context.Context, spec sandbox.JobSpec) (sandbox.Conn, error) {	273		func (s *containerSandbox) Start(ctx context.Context, spec sandbox.JobSpec) (sandbox.Conn, error) {
			274	+	// A Stop from another goroutine aborts the boot through the gate.
			275	+	bootCtx, err := s.gate.Begin(ctx)
			276	+	if err != nil {
			277	+	return nil, fmt.Errorf("container sandbox: %w", err)
			278	+	}
			279	+	defer s.gate.End()
			280	+
185		s.mu.Lock()	281		s.mu.Lock()
186		defer s.mu.Unlock()	282		defer s.mu.Unlock()
187		if s.started \|\| s.stopped {	283		if s.started \|\| s.stopped {

@@ -191,14 +287,12 @@ func (s *containerSandbox) Start(ctx context.Context, spec sandbox.JobSpec) (san
191		d := s.d	287		d := s.d
192		log := zerowrap.FromCtx(ctx)	288		log := zerowrap.FromCtx(ctx)
193		spec = d.cfg.Resolve(spec)	289		spec = d.cfg.Resolve(spec)
194	-		290	+	if d.privileged {
195	-	id, err := sandbox.NewID()	291	+	// At every job start, not just at driver start: the job log and the
196	-	if err != nil {	292	+	// operator's logs must say what kind of isolation this job got.
197	-	s.releaseLocked()	293	+	log.Warn().Str("job", spec.JobID).Str("container", s.id).Msg(privilegedWarning)
198	-	return nil, err
199		}	294		}
200	-	s.id = id	295	+	conn, err := s.run(bootCtx, spec)
201	-	conn, err := s.run(ctx, spec)
202		if err != nil {	296		if err != nil {
203		var tail string	297		var tail string
204		if s.container != "" {	298		if s.container != "" {

@@ -214,25 +308,54 @@ func (s *containerSandbox) Start(ctx context.Context, spec sandbox.JobSpec) (san
214		return conn, nil	308		return conn, nil
215		}	309		}
216			310
			311	+	// run runs the boot stages in order; each records what it created for cleanup.
217		func (s *containerSandbox) run(ctx context.Context, spec sandbox.JobSpec) (sandbox.Conn, error) {	312		func (s *containerSandbox) run(ctx context.Context, spec sandbox.JobSpec) (sandbox.Conn, error) {
218	-	d := s.d	313	+	netw, err := netpolicy.Subnet(s.d.cfg.Network, s.slot)
219	-	netw, err := netpolicy.Subnet(d.cfg.Network, s.slot)	314	+	if err != nil {
			315	+	return nil, err
			316	+	}
			317	+	accepted, err := s.runSocket()
220		if err != nil {	318		if err != nil {
221		return nil, err	319		return nil, err
222		}	320		}
			321	+	if err := s.runScratch(ctx, spec); err != nil {
			322	+	return nil, err
			323	+	}
223		labels := map[string]string{sandbox.LabelJob: spec.JobID}	324		labels := map[string]string{sandbox.LabelJob: spec.JobID}
224	-	name := "ermite-" + s.id	325	+	if err := s.runNetwork(ctx, netw, labels); err != nil {
225	-
226	-	// Per-job directory with the executor socket. It is created before anything
227	-	// on the Docker side so that a crash leaves a recognisable directory.
228	-	dir := d.runDir(s.id)
229	-	if err := os.MkdirAll(dir, 0o711); err != nil {
230		return nil, err	326		return nil, err
231		}	327		}
232	-	if err := os.Chmod(dir, 0o711); err != nil {	328	+	exited, err := s.runContainer(ctx, spec, netw, labels)
			329	+	if err != nil {
233		return nil, err	330		return nil, err
234		}	331		}
			332	+	return s.waitExecutor(ctx, accepted, exited)
			333	+	}
			334	+
			335	+	type acceptResult struct {
			336	+	c net.Conn
			337	+	err error
			338	+	}
			339	+
			340	+	// runSocket creates the per-job directory and listens on the executor socket in
			341	+	// it. The directory exists before anything on the Docker side, so a crash
			342	+	// leaves a recognisable one.
			343	+	func (s *containerSandbox) runSocket() (<-chan acceptResult, error) {
			344	+	d := s.d
			345	+	for _, dir := range []string{d.runDir(s.id), d.sockDir(s.id)} {
			346	+	if err := os.MkdirAll(dir, 0o711); err != nil {
			347	+	return nil, err
			348	+	}
			349	+	if err := os.Chmod(dir, 0o711); err != nil {
			350	+	return nil, err
			351	+	}
			352	+	}
			353	+	dir := d.sockDir(s.id)
235		sock := filepath.Join(dir, sandbox.ContainerSocket)	354		sock := filepath.Join(dir, sandbox.ContainerSocket)
			355	+	// sockaddr_un.sun_path holds 108 bytes: say so instead of "invalid argument".
			356	+	if len(sock) > maxSockPath {
			357	+	return nil, fmt.Errorf("executor socket path %q is %d bytes, over the %d-byte limit of a Unix socket: use a shorter %s", sock, len(sock), maxSockPath, sandbox.EnvStateDir)
			358	+	}
236		ln, err := net.Listen("unix", sock)	359		ln, err := net.Listen("unix", sock)
237		if err != nil {	360		if err != nil {
238		return nil, fmt.Errorf("listen executor socket: %w", err)	361		return nil, fmt.Errorf("listen executor socket: %w", err)

@@ -243,55 +366,79 @@ func (s *containerSandbox) run(ctx context.Context, spec sandbox.JobSpec) (sandb
243		if err := os.Chmod(sock, 0o666); err != nil {	366		if err := os.Chmod(sock, 0o666); err != nil {
244		return nil, err	367		return nil, err
245		}	368		}
246	-	accepted := make(chan struct {	369	+	accepted := make(chan acceptResult, 1)
247	-	c net.Conn
248	-	err error
249	-	}, 1)
250		go func() {	370		go func() {
251		c, err := ln.Accept()	371		c, err := ln.Accept()
252	-	accepted <- struct {	372	+	accepted <- acceptResult{c, err}
253	-	c net.Conn
254	-	err error
255	-	}{c, err}
256		}()	373		}()
			374	+	return accepted, nil
			375	+	}
			376	+
			377	+	// runScratch creates the sparse ext4 scratch disk (ERMITE_RUNNER_JOB_DISK) and
			378	+	// loop-mounts it; the workspace directory lives on it.
			379	+	func (s *containerSandbox) runScratch(ctx context.Context, spec sandbox.JobSpec) error {
			380	+	d := s.d
			381	+	file := filepath.Join(d.runDir(s.id), "scratch.ext4")
			382	+	if err := disk.CreateScratch(ctx, d.opt.Run, file, spec.DiskMiB, -1); err != nil {
			383	+	return err
			384	+	}
			385	+	mnt := d.scratchDir(s.id)
			386	+	if err := os.MkdirAll(mnt, 0o755); err != nil {
			387	+	return err
			388	+	}
			389	+	if err := d.mnt.Mount(ctx, file, mnt); err != nil {
			390	+	return err
			391	+	}
			392	+	s.mounted = true
			393	+	// Workspace, temp and the nested dockerd's data root on the scratch disk. The
			394	+	// executor (phase G) sets the job layout and ownership inside the sandbox.
			395	+	for _, sub := range []string{"work", "tmp", "docker"} {
			396	+	if err := os.MkdirAll(filepath.Join(mnt, sub), 0o755); err != nil {
			397	+	return err
			398	+	}
			399	+	}
			400	+	return os.Chmod(filepath.Join(mnt, "tmp"), 0o1777)
			401	+	}
257			402
			403	+	// runNetwork creates the job's bridge and installs the firewall policy.
			404	+	func (s *containerSandbox) runNetwork(ctx context.Context, netw netpolicy.Net, labels map[string]string) error {
			405	+	d := s.d
258		s.iface = fmt.Sprintf("%s%d", bridgePrefix, s.slot)	406		s.iface = fmt.Sprintf("%s%d", bridgePrefix, s.slot)
259		s.gateway = netw.Gateway	407		s.gateway = netw.Gateway
260	-	netID, err := d.eng.CreateNetwork(ctx, networkSpec{Name: name, Bridge: s.iface, Subnet: netw.Prefix, Gateway: netw.Gateway, Labels: labels})	408	+	netID, err := d.eng.CreateNetwork(ctx, networkSpec{Name: "ermite-" + s.id, Bridge: s.iface, Subnet: netw.Prefix, Gateway: netw.Gateway, Labels: labels})
261		if err != nil {	409		if err != nil {
262	-	return nil, err	410	+	return err
263		}	411		}
264		s.netID = netID	412		s.netID = netID
265		s.attached = true	413		s.attached = true
266	-	if err := d.net.Attach(ctx, netpolicy.Guest{Iface: s.iface, Gateway: netw.Gateway, Addr: netw.Guest}); err != nil {	414	+	return d.net.Attach(ctx, netpolicy.Guest{Iface: s.iface, Gateway: netw.Gateway, Addr: netw.Guest})
267	-	return nil, err	415	+	}
268	-	}
269	-
270	-	s.volume = name
271	-	if err := d.eng.CreateVolume(ctx, name, labels); err != nil {
272	-	s.volume = ""
273	-	return nil, fmt.Errorf("create volume: %w", err)
274	-	}
275			416
			417	+	// runContainer creates and starts the container.
			418	+	func (s *containerSandbox) runContainer(ctx context.Context, spec sandbox.JobSpec, netw netpolicy.Net, labels map[string]string) (<-chan exitResult, error) {
			419	+	d := s.d
276		entry := sandbox.GuestExecDir + "/" + sandbox.GuestExecName	420		entry := sandbox.GuestExecDir + "/" + sandbox.GuestExecName
277		cid, err := d.eng.CreateContainer(ctx, containerSpec{	421		cid, err := d.eng.CreateContainer(ctx, containerSpec{
278	-	Name: name,	422	+	Name: "ermite-" + s.id,
279		Image: d.image,	423		Image: d.image,
280		Entrypoint: []string{entry, "init"},	424		Entrypoint: []string{entry, "init"},
281		Env: []string{"ERMITE_CONNECT=unix:" + sandbox.ContainerRunDir + "/" + sandbox.ContainerSocket},	425		Env: []string{"ERMITE_CONNECT=unix:" + sandbox.ContainerRunDir + "/" + sandbox.ContainerSocket},
282		Labels: labels,	426		Labels: labels,
283		Runtime: d.runtime,	427		Runtime: d.runtime,
284		Privileged: d.privileged,	428		Privileged: d.privileged,
285	-	Network: name,	429	+	Network: "ermite-" + s.id,
286		IPv4: netw.Guest,	430		IPv4: netw.Guest,
287		DNS: netw.Gateway,	431		DNS: netw.Gateway,
288		Binds: []bindMount{	432		Binds: []bindMount{
289		{Source: d.exe, Target: entry, ReadOnly: true},	433		{Source: d.exe, Target: entry, ReadOnly: true},
290	-	{Source: dir, Target: sandbox.ContainerRunDir},	434	+	{Source: d.sockDir(s.id), Target: sandbox.ContainerRunDir},
			435	+	{Source: d.scratchDir(s.id), Target: sandbox.GuestScratch},
			436	+	{Source: d.workDir(s.id), Target: containerWorkspace},
			437	+	{Source: filepath.Join(d.scratchDir(s.id), "tmp"), Target: containerTmp},
291		},	438		},
292	-	Volumes: []volumeMount{{Name: name, Target: sandbox.GuestScratch}},
293		MemoryMiB: spec.MemoryMiB,	439		MemoryMiB: spec.MemoryMiB,
294		CPUs: spec.CPUs,	440		CPUs: spec.CPUs,
			441	+	Pids: d.cfg.JobPids,
295		})	442		})
296		if err != nil {	443		if err != nil {
297		return nil, err	444		return nil, err

@@ -301,8 +448,14 @@ func (s *containerSandbox) run(ctx context.Context, spec sandbox.JobSpec) (sandb
301		if err := d.eng.StartContainer(ctx, cid); err != nil {	448		if err := d.eng.StartContainer(ctx, cid); err != nil {
302		return nil, fmt.Errorf("start container: %w", err)	449		return nil, fmt.Errorf("start container: %w", err)
303		}	450		}
			451	+	return exited, nil
			452	+	}
304			453
305	-	timer := time.NewTimer(d.opt.ConnectTimeout)	454	+	// waitExecutor waits for the executor to connect. Only ctx (the caller's
			455	+	// context, or Stop through the boot gate), the container exiting, and the
			456	+	// timeout end the wait.
			457	+	func (s *containerSandbox) waitExecutor(ctx context.Context, accepted <-chan acceptResult, exited <-chan exitResult) (sandbox.Conn, error) {
			458	+	timer := time.NewTimer(s.d.opt.ConnectTimeout)
306		defer timer.Stop()	459		defer timer.Stop()
307		select {	460		select {
308		case r := <-accepted:	461		case r := <-accepted:

@@ -312,20 +465,25 @@ func (s *containerSandbox) run(ctx context.Context, spec sandbox.JobSpec) (sandb
312		s.conn = r.c	465		s.conn = r.c
313		return r.c, nil	466		return r.c, nil
314		case ex := <-exited:	467		case ex := <-exited:
			468	+	if ex.Err != nil {
			469	+	return nil, fmt.Errorf("waiting for the container: %w", ex.Err)
			470	+	}
315		return nil, fmt.Errorf("container exited (code %d) before the executor connected", ex.Code)	471		return nil, fmt.Errorf("container exited (code %d) before the executor connected", ex.Code)
316		case <-timer.C:	472		case <-timer.C:
317	-	return nil, fmt.Errorf("executor did not connect within %s", d.opt.ConnectTimeout)	473	+	return nil, fmt.Errorf("executor did not connect within %s", s.d.opt.ConnectTimeout)
318		case <-ctx.Done():	474		case <-ctx.Done():
319	-	return nil, ctx.Err()	475	+	return nil, context.Cause(ctx)
320		}	476		}
321		}	477		}
322			478
323		// Stop implements sandbox.Sandbox.	479		// Stop implements sandbox.Sandbox.
324		func (s *containerSandbox) Stop(ctx context.Context) error {	480		func (s *containerSandbox) Stop(ctx context.Context) error {
			481	+	// Abort a boot in progress first: Start holds s.mu for its whole duration.
			482	+	s.gate.Abort()
325		s.mu.Lock()	483		s.mu.Lock()
326		defer s.mu.Unlock()	484		defer s.mu.Unlock()
327		if s.stopped {	485		if s.stopped {
328	-	return nil	486	+	return s.leakErr
329		}	487		}
330		if !s.started {	488		if !s.started {
331		s.releaseLocked()	489		s.releaseLocked()

@@ -339,10 +497,11 @@ func (s *containerSandbox) releaseLocked() {
339		s.d.release(s.slot)	497		s.d.release(s.slot)
340		}	498		}
341			499
342	-	// cleanupLocked removes the container, then its network, volume, firewall rules	500	+	// cleanupLocked removes the container first; only when it is confirmed gone are
343	-	// and directory, continuing after errors. The slot is released only when	501	+	// the firewall policy, the network, the scratch mount and the files removed. A
344	-	// everything is gone (so its bridge name and subnet are not reused while the	502	+	// container that may still run keeps its policy and bridge: tearing the policy
345	-	// old ones still exist); leftovers are reaped by label at the next start.	503	+	// down under it would remove the isolation it relies on. Failures mark the
			504	+	// sandbox as leaked (sandbox.ErrLeaked): its slot stays taken until Reap.
346		func (s *containerSandbox) cleanupLocked(ctx context.Context) error {	505		func (s *containerSandbox) cleanupLocked(ctx context.Context) error {
347		d := s.d	506		d := s.d
348		var errs []error	507		var errs []error

@@ -361,45 +520,59 @@ func (s *containerSandbox) cleanupLocked(ctx context.Context) error {
361		s.container = ""	520		s.container = ""
362		}	521		}
363		}	522		}
364	-	if s.attached {	523	+	if s.container == "" {
365	-	if err := d.net.Detach(ctx, s.iface, s.gateway); err != nil {	524	+	if s.attached {
366	-	errs = append(errs, err)	525	+	if err := d.net.Detach(ctx, s.iface, s.gateway); err != nil {
367	-	} else {	526	+	errs = append(errs, err)
368	-	s.attached = false	527	+	} else {
			528	+	s.attached = false
			529	+	}
369		}	530		}
370	-	}	531	+	if s.netID != "" {
371	-	if s.netID != "" && s.container == "" {	532	+	if err := d.eng.RemoveNetwork(ctx, s.netID); err != nil {
372	-	if err := d.eng.RemoveNetwork(ctx, s.netID); err != nil {	533	+	errs = append(errs, fmt.Errorf("remove network: %w", err))
373	-	errs = append(errs, fmt.Errorf("remove network: %w", err))	534	+	} else {
374	-	} else {	535	+	s.netID = ""
375	-	s.netID = ""	536	+	}
376		}	537		}
377	-	}	538	+	if s.mounted {
378	-	if s.volume != "" && s.container == "" {	539	+	if err := d.mnt.Unmount(ctx, d.scratchDir(s.id)); err != nil {
379	-	if err := d.eng.RemoveVolume(ctx, s.volume); err != nil {	540	+	errs = append(errs, err)
380	-	errs = append(errs, fmt.Errorf("remove volume: %w", err))	541	+	} else {
381	-	} else {	542	+	s.mounted = false
382	-	s.volume = ""	543	+	}
			544	+	}
			545	+	// Never delete files below a mount that is still there.
			546	+	if !s.mounted {
			547	+	if err := os.RemoveAll(d.runDir(s.id)); err != nil {
			548	+	errs = append(errs, err)
			549	+	}
383		}	550		}
384	-	}
385	-	if err := os.RemoveAll(d.runDir(s.id)); err != nil {
386	-	errs = append(errs, err)
387		}	551		}
388		if len(errs) == 0 {	552		if len(errs) == 0 {
389		s.releaseLocked()	553		s.releaseLocked()
390		return nil	554		return nil
391		}	555		}
392		s.stopped = true	556		s.stopped = true
393	-	err := errors.Join(errs...)	557	+	d.markLeaked(s.slot)
			558	+	s.leakErr = fmt.Errorf("%w: container sandbox %s: %w", sandbox.ErrLeaked, s.id, errors.Join(errs...))
394		log := zerowrap.FromCtx(ctx)	559		log := zerowrap.FromCtx(ctx)
395	-	log.Error().Err(err).Str("job", s.id).Msg("container cleanup incomplete; the next runner start will retry")	560	+	log.Error().Err(s.leakErr).Str("job", s.id).Msg("container cleanup incomplete; call Reap to finish it")
396	-	return err	561	+	return s.leakErr
397		}	562		}
398			563
399	-	// Reap implements sandbox.Driver: remove every container, network and volume	564	+	// Reap implements sandbox.Driver: remove every container and network carrying
400	-	// carrying the job label, the nftables table, and the per-job directories.	565	+	// the job label, the nftables table, the scratch mounts and the per-job
			566	+	// directories, and free the slots of leaked sandboxes. It returns ErrBusy while
			567	+	// a sandbox is running.
401		func (d *Driver) Reap(ctx context.Context) error {	568		func (d *Driver) Reap(ctx context.Context) error {
402		log := zerowrap.FromCtx(ctx)	569		log := zerowrap.FromCtx(ctx)
			570	+	d.mu.Lock()
			571	+	busy := d.slots.Busy()
			572	+	d.mu.Unlock()
			573	+	if busy > 0 {
			574	+	return fmt.Errorf("%w (%d running)", sandbox.ErrBusy, busy)
			575	+	}
403		var errs []error	576		var errs []error
404		ids, err := d.eng.ListContainers(ctx, sandbox.LabelJob)	577		ids, err := d.eng.ListContainers(ctx, sandbox.LabelJob)
405		if err != nil {	578		if err != nil {

@@ -421,27 +594,44 @@ func (d *Driver) Reap(ctx context.Context) error {
421		errs = append(errs, err)	594		errs = append(errs, err)
422		}	595		}
423		}	596		}
424	-	vols, err := d.eng.ListVolumes(ctx, sandbox.LabelJob)
425	-	if err != nil {
426	-	errs = append(errs, fmt.Errorf("list volumes: %w", err))
427	-	}
428	-	for _, v := range vols {
429	-	log.Warn().Str("volume", v).Msg("removing leftover job volume")
430	-	if err := d.eng.RemoveVolume(ctx, v); err != nil {
431	-	errs = append(errs, err)
432	-	}
433	-	}
434		if err := d.net.Reset(ctx); err != nil {	597		if err := d.net.Reset(ctx); err != nil {
435		errs = append(errs, fmt.Errorf("reset nftables table: %w", err))	598		errs = append(errs, fmt.Errorf("reset nftables table: %w", err))
436		}	599		}
			600	+	// Unmount before deleting: a directory with a live mount below it must not
			601	+	// be removed (and EBUSY would be the best case).
437		root := filepath.Join(d.cfg.StateDir, "containers")	602		root := filepath.Join(d.cfg.StateDir, "containers")
			603	+	mounts, merr := d.mnt.MountPoints()
			604	+	if merr != nil {
			605	+	errs = append(errs, fmt.Errorf("read mounts: %w", merr))
			606	+	}
			607	+	stuck := map[string]bool{}
			608	+	for _, mp := range mounts {
			609	+	rel, ok := strings.CutPrefix(mp, root+"/")
			610	+	if !ok {
			611	+	continue
			612	+	}
			613	+	id, _, _ := strings.Cut(rel, "/")
			614	+	if !sandbox.ValidID(id) {
			615	+	continue
			616	+	}
			617	+	log.Warn().Str("mount", mp).Msg("unmounting leftover scratch disk")
			618	+	if err := d.mnt.Unmount(ctx, mp); err != nil {
			619	+	stuck[id] = true
			620	+	errs = append(errs, err)
			621	+	}
			622	+	}
438		ents, _ := os.ReadDir(root)	623		ents, _ := os.ReadDir(root)
439		for _, e := range ents {	624		for _, e := range ents {
440	-	if sandbox.ValidID(e.Name()) {	625	+	if sandbox.ValidID(e.Name()) && !stuck[e.Name()] {
441		if err := os.RemoveAll(filepath.Join(root, e.Name())); err != nil {	626		if err := os.RemoveAll(filepath.Join(root, e.Name())); err != nil {
442		errs = append(errs, err)	627		errs = append(errs, err)
443		}	628		}
444		}	629		}
445		}	630		}
			631	+	if len(errs) == 0 {
			632	+	d.mu.Lock()
			633	+	d.slots.ReleaseLeaked()
			634	+	d.mu.Unlock()
			635	+	}
446		return errors.Join(errs...)	636		return errors.Join(errs...)
447		}	637		}

M internal/sandbox/container/engine.go +32 -39


@@ -27,9 +27,6 @@ type engine interface {
27	27		CreateNetwork(ctx context.Context, spec networkSpec) (string, error)
28	28		RemoveNetwork(ctx context.Context, id string) error
29	29		ListNetworks(ctx context.Context, labelKey string) ([]string, error)
30		-	CreateVolume(ctx context.Context, name string, labels map[string]string) error
31		-	RemoveVolume(ctx context.Context, name string) error
32		-	ListVolumes(ctx context.Context, labelKey string) ([]string, error)
33	30		CreateContainer(ctx context.Context, spec containerSpec) (string, error)
34	31		StartContainer(ctx context.Context, id string) error
35	32		// WaitExit returns a channel that yields the exit code once the container

@@ -72,9 +69,9 @@ type containerSpec struct {
72	69		IPv4 netip.Addr
73	70		DNS netip.Addr
74	71		Binds []bindMount
75		-	Volumes []volumeMount
76		-	MemoryMiB int
	72	+	MemoryMiB int // also the swap limit: no swap
77	73		CPUs int
	74	+	Pids int // PidsLimit
78	75		}
79	76
80	77		type bindMount struct {

@@ -82,8 +79,6 @@ type bindMount struct {
82	79		ReadOnly bool
83	80		}
84	81
85		-	type volumeMount struct{ Name, Target string }
86		-
87	82		// dockerEngine implements engine with the moby client.
88	83		type dockerEngine struct{ cli *client.Client }
89	84

@@ -171,29 +166,8 @@ func (e *dockerEngine) ListNetworks(ctx context.Context, key string) ([]string,
171	166		return ids, nil
172	167		}
173	168
174		-	func (e *dockerEngine) CreateVolume(ctx context.Context, name string, labels map[string]string) error {
175		-	_, err := e.cli.VolumeCreate(ctx, client.VolumeCreateOptions{Name: name, Labels: labels})
176		-	return err
177		-	}
178		-
179		-	func (e *dockerEngine) RemoveVolume(ctx context.Context, name string) error {
180		-	_, err := e.cli.VolumeRemove(ctx, name, client.VolumeRemoveOptions{Force: true})
181		-	return ignoreNotFound(err)
182		-	}
183		-
184		-	func (e *dockerEngine) ListVolumes(ctx context.Context, key string) ([]string, error) {
185		-	r, err := e.cli.VolumeList(ctx, client.VolumeListOptions{Filters: client.Filters{}.Add("label", key)})
186		-	if err != nil {
187		-	return nil, err
188		-	}
189		-	var names []string
190		-	for _, v := range r.Items {
191		-	names = append(names, v.Name)
192		-	}
193		-	return names, nil
194		-	}
195		-
196		-	func (e *dockerEngine) CreateContainer(ctx context.Context, s containerSpec) (string, error) {
	169	+	// hostConfigFor builds the Docker host configuration of a job container.
	170	+	func hostConfigFor(s containerSpec) *container.HostConfig {
197	171		hc := &container.HostConfig{
198	172		NetworkMode: container.NetworkMode(s.Network),
199	173		Privileged: s.Privileged,

@@ -204,17 +178,24 @@ func (e *dockerEngine) CreateContainer(ctx context.Context, s containerSpec) (st
204	178		}
205	179		if s.MemoryMiB > 0 {
206	180		hc.Memory = int64(s.MemoryMiB) << 20
	181	+	hc.MemorySwap = hc.Memory // swap = memory: the job cannot use swap
207	182		}
208	183		if s.CPUs > 0 {
209	184		hc.NanoCPUs = int64(s.CPUs) * 1_000_000_000
210	185		}
	186	+	if s.Pids > 0 {
	187	+	p := int64(s.Pids)
	188	+	hc.PidsLimit = &p
	189	+	}
211	190		for _, b := range s.Binds {
212	191		hc.Mounts = append(hc.Mounts, mount.Mount{Type: mount.TypeBind, Source: b.Source, Target: b.Target, ReadOnly: b.ReadOnly,
213	192		BindOptions: &mount.BindOptions{CreateMountpoint: true}})
214	193		}
215		-	for _, v := range s.Volumes {
216		-	hc.Mounts = append(hc.Mounts, mount.Mount{Type: mount.TypeVolume, Source: v.Name, Target: v.Target})
217		-	}
	194	+	return hc
	195	+	}
	196	+
	197	+	func (e *dockerEngine) CreateContainer(ctx context.Context, s containerSpec) (string, error) {
	198	+	hc := hostConfigFor(s)
218	199		cfg := &container.Config{
219	200		Image: s.Image,
220	201		Entrypoint: s.Entrypoint,

@@ -239,17 +220,29 @@ func (e *dockerEngine) StartContainer(ctx context.Context, id string) error {
239	220		}
240	221
241	222		func (e *dockerEngine) WaitExit(ctx context.Context, id string) <-chan exitResult {
242		-	out := make(chan exitResult, 1)
243	223		w := e.cli.ContainerWait(ctx, id, client.ContainerWaitOptions{Condition: container.WaitConditionNextExit})
	224	+	return awaitExit(ctx, w.Result, w.Error)
	225	+	}
	226	+
	227	+	// awaitExit turns the two channels of a Docker wait into one result. A context
	228	+	// that ended is reported as such (code -1, ctx.Err()), whichever channel the
	229	+	// select picked: a cancelled wait must never look like a clean exit.
	230	+	func awaitExit(ctx context.Context, result <-chan container.WaitResponse, errc <-chan error) <-chan exitResult {
	231	+	out := make(chan exitResult, 1)
244	232		go func() {
	233	+	var res exitResult
245	234		select {
246		-	case r := <-w.Result:
247		-	out <- exitResult{Code: int(r.StatusCode)}
248		-	case err := <-w.Error:
249		-	out <- exitResult{Code: -1, Err: err}
	235	+	case r := <-result:
	236	+	res = exitResult{Code: int(r.StatusCode)}
	237	+	case err := <-errc:
	238	+	res = exitResult{Code: -1, Err: err}
250	239		case <-ctx.Done():
251		-	out <- exitResult{Code: -1, Err: ctx.Err()}
	240	+	res = exitResult{Code: -1, Err: ctx.Err()}
	241	+	}
	242	+	if err := ctx.Err(); err != nil {
	243	+	res = exitResult{Code: -1, Err: err}
252	244		}
	245	+	out <- res
253	246		}()
254	247		return out
255	248		}


@@ -27,9 +27,6 @@ type engine interface {
27		CreateNetwork(ctx context.Context, spec networkSpec) (string, error)	27		CreateNetwork(ctx context.Context, spec networkSpec) (string, error)
28		RemoveNetwork(ctx context.Context, id string) error	28		RemoveNetwork(ctx context.Context, id string) error
29		ListNetworks(ctx context.Context, labelKey string) ([]string, error)	29		ListNetworks(ctx context.Context, labelKey string) ([]string, error)
30	-	CreateVolume(ctx context.Context, name string, labels map[string]string) error
31	-	RemoveVolume(ctx context.Context, name string) error
32	-	ListVolumes(ctx context.Context, labelKey string) ([]string, error)
33		CreateContainer(ctx context.Context, spec containerSpec) (string, error)	30		CreateContainer(ctx context.Context, spec containerSpec) (string, error)
34		StartContainer(ctx context.Context, id string) error	31		StartContainer(ctx context.Context, id string) error
35		// WaitExit returns a channel that yields the exit code once the container	32		// WaitExit returns a channel that yields the exit code once the container

@@ -72,9 +69,9 @@ type containerSpec struct {
72		IPv4 netip.Addr	69		IPv4 netip.Addr
73		DNS netip.Addr	70		DNS netip.Addr
74		Binds []bindMount	71		Binds []bindMount
75	-	Volumes []volumeMount	72	+	MemoryMiB int // also the swap limit: no swap
76	-	MemoryMiB int
77		CPUs int	73		CPUs int
			74	+	Pids int // PidsLimit
78		}	75		}
79			76
80		type bindMount struct {	77		type bindMount struct {

@@ -82,8 +79,6 @@ type bindMount struct {
82		ReadOnly bool	79		ReadOnly bool
83		}	80		}
84			81
85	-	type volumeMount struct{ Name, Target string }
86	-
87		// dockerEngine implements engine with the moby client.	82		// dockerEngine implements engine with the moby client.
88		type dockerEngine struct{ cli *client.Client }	83		type dockerEngine struct{ cli *client.Client }
89			84

@@ -171,29 +166,8 @@ func (e *dockerEngine) ListNetworks(ctx context.Context, key string) ([]string,
171		return ids, nil	166		return ids, nil
172		}	167		}
173			168
174	-	func (e *dockerEngine) CreateVolume(ctx context.Context, name string, labels map[string]string) error {	169	+	// hostConfigFor builds the Docker host configuration of a job container.
175	-	_, err := e.cli.VolumeCreate(ctx, client.VolumeCreateOptions{Name: name, Labels: labels})	170	+	func hostConfigFor(s containerSpec) *container.HostConfig {
176	-	return err
177	-	}
178	-
179	-	func (e *dockerEngine) RemoveVolume(ctx context.Context, name string) error {
180	-	_, err := e.cli.VolumeRemove(ctx, name, client.VolumeRemoveOptions{Force: true})
181	-	return ignoreNotFound(err)
182	-	}
183	-
184	-	func (e *dockerEngine) ListVolumes(ctx context.Context, key string) ([]string, error) {
185	-	r, err := e.cli.VolumeList(ctx, client.VolumeListOptions{Filters: client.Filters{}.Add("label", key)})
186	-	if err != nil {
187	-	return nil, err
188	-	}
189	-	var names []string
190	-	for _, v := range r.Items {
191	-	names = append(names, v.Name)
192	-	}
193	-	return names, nil
194	-	}
195	-
196	-	func (e *dockerEngine) CreateContainer(ctx context.Context, s containerSpec) (string, error) {
197		hc := &container.HostConfig{	171		hc := &container.HostConfig{
198		NetworkMode: container.NetworkMode(s.Network),	172		NetworkMode: container.NetworkMode(s.Network),
199		Privileged: s.Privileged,	173		Privileged: s.Privileged,

@@ -204,17 +178,24 @@ func (e *dockerEngine) CreateContainer(ctx context.Context, s containerSpec) (st
204		}	178		}
205		if s.MemoryMiB > 0 {	179		if s.MemoryMiB > 0 {
206		hc.Memory = int64(s.MemoryMiB) << 20	180		hc.Memory = int64(s.MemoryMiB) << 20
			181	+	hc.MemorySwap = hc.Memory // swap = memory: the job cannot use swap
207		}	182		}
208		if s.CPUs > 0 {	183		if s.CPUs > 0 {
209		hc.NanoCPUs = int64(s.CPUs) * 1_000_000_000	184		hc.NanoCPUs = int64(s.CPUs) * 1_000_000_000
210		}	185		}
			186	+	if s.Pids > 0 {
			187	+	p := int64(s.Pids)
			188	+	hc.PidsLimit = &p
			189	+	}
211		for _, b := range s.Binds {	190		for _, b := range s.Binds {
212		hc.Mounts = append(hc.Mounts, mount.Mount{Type: mount.TypeBind, Source: b.Source, Target: b.Target, ReadOnly: b.ReadOnly,	191		hc.Mounts = append(hc.Mounts, mount.Mount{Type: mount.TypeBind, Source: b.Source, Target: b.Target, ReadOnly: b.ReadOnly,
213		BindOptions: &mount.BindOptions{CreateMountpoint: true}})	192		BindOptions: &mount.BindOptions{CreateMountpoint: true}})
214		}	193		}
215	-	for _, v := range s.Volumes {	194	+	return hc
216	-	hc.Mounts = append(hc.Mounts, mount.Mount{Type: mount.TypeVolume, Source: v.Name, Target: v.Target})	195	+	}
217	-	}	196	+
			197	+	func (e *dockerEngine) CreateContainer(ctx context.Context, s containerSpec) (string, error) {
			198	+	hc := hostConfigFor(s)
218		cfg := &container.Config{	199		cfg := &container.Config{
219		Image: s.Image,	200		Image: s.Image,
220		Entrypoint: s.Entrypoint,	201		Entrypoint: s.Entrypoint,

@@ -239,17 +220,29 @@ func (e *dockerEngine) StartContainer(ctx context.Context, id string) error {
239		}	220		}
240			221
241		func (e *dockerEngine) WaitExit(ctx context.Context, id string) <-chan exitResult {	222		func (e *dockerEngine) WaitExit(ctx context.Context, id string) <-chan exitResult {
242	-	out := make(chan exitResult, 1)
243		w := e.cli.ContainerWait(ctx, id, client.ContainerWaitOptions{Condition: container.WaitConditionNextExit})	223		w := e.cli.ContainerWait(ctx, id, client.ContainerWaitOptions{Condition: container.WaitConditionNextExit})
			224	+	return awaitExit(ctx, w.Result, w.Error)
			225	+	}
			226	+
			227	+	// awaitExit turns the two channels of a Docker wait into one result. A context
			228	+	// that ended is reported as such (code -1, ctx.Err()), whichever channel the
			229	+	// select picked: a cancelled wait must never look like a clean exit.
			230	+	func awaitExit(ctx context.Context, result <-chan container.WaitResponse, errc <-chan error) <-chan exitResult {
			231	+	out := make(chan exitResult, 1)
244		go func() {	232		go func() {
			233	+	var res exitResult
245		select {	234		select {
246	-	case r := <-w.Result:	235	+	case r := <-result:
247	-	out <- exitResult{Code: int(r.StatusCode)}	236	+	res = exitResult{Code: int(r.StatusCode)}
248	-	case err := <-w.Error:	237	+	case err := <-errc:
249	-	out <- exitResult{Code: -1, Err: err}	238	+	res = exitResult{Code: -1, Err: err}
250		case <-ctx.Done():	239		case <-ctx.Done():
251	-	out <- exitResult{Code: -1, Err: ctx.Err()}	240	+	res = exitResult{Code: -1, Err: ctx.Err()}
			241	+	}
			242	+	if err := ctx.Err(); err != nil {
			243	+	res = exitResult{Code: -1, Err: err}
252		}	244		}
			245	+	out <- res
253		}()	246		}()
254		return out	247		return out
255		}	248		}

A internal/sandbox/disk/disk.go +83 -0

@@ -0,0 +1,83 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	// Package disk creates the per-job scratch disks (sparse ext4 files) and, for
	4	+	// the container mode, loop-mounts them. Both isolation modes size a job's
	5	+	// writable space the same way: a sparse file of ERMITE_RUNNER_JOB_DISK.
	6	+	package disk
	7	+
	8	+	import (
	9	+	"context"
	10	+	"fmt"
	11	+	"os"
	12	+	"regexp"
	13	+	"strconv"
	14	+	"strings"
	15	+	"syscall"
	16	+
	17	+	"bnema.dev/ermite-runner/internal/cmdrun"
	18	+	)
	19	+
	20	+	// CreateScratch creates the sparse scratch disk of sizeMiB at path and formats
	21	+	// it. The file is created exclusively without following symlinks and handed to
	22	+	// owner (fchown on the open descriptor, never by path) when owner >= 0.
	23	+	func CreateScratch(ctx context.Context, run cmdrun.Runner, path string, sizeMiB, owner int) error {
	24	+	f, err := os.OpenFile(path, os.O_WRONLY\|os.O_CREATE\|os.O_EXCL\|syscall.O_NOFOLLOW, 0o600)
	25	+	if err != nil {
	26	+	return err
	27	+	}
	28	+	if owner >= 0 {
	29	+	if err := f.Chown(owner, owner); err != nil {
	30	+	_ = f.Close()
	31	+	return fmt.Errorf("chown scratch disk: %w", err)
	32	+	}
	33	+	}
	34	+	if err := f.Truncate(int64(sizeMiB) << 20); err != nil {
	35	+	_ = f.Close()
	36	+	return err
	37	+	}
	38	+	if err := f.Close(); err != nil {
	39	+	return err
	40	+	}
	41	+	// No journal and no reserved blocks: it is a throw-away disk. Lazy table
	42	+	// initialisation keeps creation instant and the file sparse.
	43	+	if _, err := run.Run(ctx, nil, "mkfs.ext4", "-q", "-F", "-L", "ermite-scratch", "-m", "0",
	44	+	"-O", "^has_journal,^resize_inode", "-E", "lazy_itable_init=1,nodiscard", path); err != nil {
	45	+	return fmt.Errorf("format scratch disk: %w", err)
	46	+	}
	47	+	return nil
	48	+	}
	49	+
	50	+	var mkfsVersionRe = regexp.MustCompile(`(?m)^mke2fs (\d+)\.(\d+)(?:\.(\d+))?`)
	51	+
	52	+	// MinMkfs is the oldest e2fsprogs the runner accepts: mkfs.ext4 -d (populate
	53	+	// from a directory, without root) appeared in 1.43. scripts/build-image.sh needs
	54	+	// 1.47.1 to populate from a tarball.
	55	+	const MinMkfs = "1.43"
	56	+
	57	+	// CheckMkfs fails when mkfs.ext4 is missing or older than MinMkfs.
	58	+	func CheckMkfs(ctx context.Context, run cmdrun.Runner) error {
	59	+	out, err := run.Run(ctx, nil, "mkfs.ext4", "-V")
	60	+	// mke2fs prints its version on stderr and exits 0 or 1 depending on the
	61	+	// release: judge by the output, not by the exit status.
	62	+	maj, min, ok := parseMkfsVersion(string(out))
	63	+	if !ok {
	64	+	if err != nil {
	65	+	return fmt.Errorf("mkfs.ext4 not usable (install e2fsprogs >= %s): %w", MinMkfs, err)
	66	+	}
	67	+	return fmt.Errorf("cannot read the mkfs.ext4 version from %q (need e2fsprogs >= %s)", strings.TrimSpace(string(out)), MinMkfs)
	68	+	}
	69	+	if maj < 1 \|\| (maj == 1 && min < 43) {
	70	+	return fmt.Errorf("mkfs.ext4 is e2fsprogs %d.%d, need >= %s for mkfs.ext4 -d", maj, min, MinMkfs)
	71	+	}
	72	+	return nil
	73	+	}
	74	+
	75	+	func parseMkfsVersion(s string) (maj, min int, ok bool) {
	76	+	m := mkfsVersionRe.FindStringSubmatch(s)
	77	+	if m == nil {
	78	+	return 0, 0, false
	79	+	}
	80	+	maj, _ = strconv.Atoi(m[1])
	81	+	min, _ = strconv.Atoi(m[2])
	82	+	return maj, min, true
	83	+	}

@@ -0,0 +1,83 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	// Package disk creates the per-job scratch disks (sparse ext4 files) and, for
	4	+	// the container mode, loop-mounts them. Both isolation modes size a job's
	5	+	// writable space the same way: a sparse file of ERMITE_RUNNER_JOB_DISK.
	6	+	package disk
	7	+
	8	+	import (
	9	+	"context"
	10	+	"fmt"
	11	+	"os"
	12	+	"regexp"
	13	+	"strconv"
	14	+	"strings"
	15	+	"syscall"
	16	+
	17	+	"bnema.dev/ermite-runner/internal/cmdrun"
	18	+	)
	19	+
	20	+	// CreateScratch creates the sparse scratch disk of sizeMiB at path and formats
	21	+	// it. The file is created exclusively without following symlinks and handed to
	22	+	// owner (fchown on the open descriptor, never by path) when owner >= 0.
	23	+	func CreateScratch(ctx context.Context, run cmdrun.Runner, path string, sizeMiB, owner int) error {
	24	+	f, err := os.OpenFile(path, os.O_WRONLY\|os.O_CREATE\|os.O_EXCL\|syscall.O_NOFOLLOW, 0o600)
	25	+	if err != nil {
	26	+	return err
	27	+	}
	28	+	if owner >= 0 {
	29	+	if err := f.Chown(owner, owner); err != nil {
	30	+	_ = f.Close()
	31	+	return fmt.Errorf("chown scratch disk: %w", err)
	32	+	}
	33	+	}
	34	+	if err := f.Truncate(int64(sizeMiB) << 20); err != nil {
	35	+	_ = f.Close()
	36	+	return err
	37	+	}
	38	+	if err := f.Close(); err != nil {
	39	+	return err
	40	+	}
	41	+	// No journal and no reserved blocks: it is a throw-away disk. Lazy table
	42	+	// initialisation keeps creation instant and the file sparse.
	43	+	if _, err := run.Run(ctx, nil, "mkfs.ext4", "-q", "-F", "-L", "ermite-scratch", "-m", "0",
	44	+	"-O", "^has_journal,^resize_inode", "-E", "lazy_itable_init=1,nodiscard", path); err != nil {
	45	+	return fmt.Errorf("format scratch disk: %w", err)
	46	+	}
	47	+	return nil
	48	+	}
	49	+
	50	+	var mkfsVersionRe = regexp.MustCompile(`(?m)^mke2fs (\d+)\.(\d+)(?:\.(\d+))?`)
	51	+
	52	+	// MinMkfs is the oldest e2fsprogs the runner accepts: mkfs.ext4 -d (populate
	53	+	// from a directory, without root) appeared in 1.43. scripts/build-image.sh needs
	54	+	// 1.47.1 to populate from a tarball.
	55	+	const MinMkfs = "1.43"
	56	+
	57	+	// CheckMkfs fails when mkfs.ext4 is missing or older than MinMkfs.
	58	+	func CheckMkfs(ctx context.Context, run cmdrun.Runner) error {
	59	+	out, err := run.Run(ctx, nil, "mkfs.ext4", "-V")
	60	+	// mke2fs prints its version on stderr and exits 0 or 1 depending on the
	61	+	// release: judge by the output, not by the exit status.
	62	+	maj, min, ok := parseMkfsVersion(string(out))
	63	+	if !ok {
	64	+	if err != nil {
	65	+	return fmt.Errorf("mkfs.ext4 not usable (install e2fsprogs >= %s): %w", MinMkfs, err)
	66	+	}
	67	+	return fmt.Errorf("cannot read the mkfs.ext4 version from %q (need e2fsprogs >= %s)", strings.TrimSpace(string(out)), MinMkfs)
	68	+	}
	69	+	if maj < 1 \|\| (maj == 1 && min < 43) {
	70	+	return fmt.Errorf("mkfs.ext4 is e2fsprogs %d.%d, need >= %s for mkfs.ext4 -d", maj, min, MinMkfs)
	71	+	}
	72	+	return nil
	73	+	}
	74	+
	75	+	func parseMkfsVersion(s string) (maj, min int, ok bool) {
	76	+	m := mkfsVersionRe.FindStringSubmatch(s)
	77	+	if m == nil {
	78	+	return 0, 0, false
	79	+	}
	80	+	maj, _ = strconv.Atoi(m[1])
	81	+	min, _ = strconv.Atoi(m[2])
	82	+	return maj, min, true
	83	+	}

A internal/sandbox/disk/disk_test.go +126 -0

@@ -0,0 +1,126 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package disk
	4	+
	5	+	import (
	6	+	"context"
	7	+	"errors"
	8	+	"os"
	9	+	"path/filepath"
	10	+	"strings"
	11	+	"testing"
	12	+
	13	+	"bnema.dev/ermite-runner/internal/cmdrun"
	14	+	)
	15	+
	16	+	func TestCreateScratchIsSparseExclusiveAndNoFollow(t *testing.T) {
	17	+	dir := t.TempDir()
	18	+	run := &cmdrun.Fake{}
	19	+	p := filepath.Join(dir, "scratch.ext4")
	20	+	if err := CreateScratch(context.Background(), run, p, 64, os.Getuid()); err != nil {
	21	+	t.Fatal(err)
	22	+	}
	23	+	st, _ := os.Stat(p)
	24	+	if st.Size() != 64<<20 \|\| st.Mode().Perm() != 0o600 {
	25	+	t.Fatalf("%v", st)
	26	+	}
	27	+	if got := run.Lines(); len(got) != 1 \|\| !strings.Contains(got[0], "mkfs.ext4") \|\| !strings.Contains(got[0], "^has_journal") {
	28	+	t.Fatalf("mkfs call: %v", got)
	29	+	}
	30	+	if err := CreateScratch(context.Background(), run, p, 64, -1); err == nil {
	31	+	t.Fatal("an existing file must be refused")
	32	+	}
	33	+	victim := filepath.Join(dir, "victim")
	34	+	_ = os.WriteFile(victim, []byte("x"), 0o600)
	35	+	link := filepath.Join(dir, "link")
	36	+	_ = os.Symlink(victim, link)
	37	+	if err := CreateScratch(context.Background(), run, link, 1, -1); err == nil {
	38	+	t.Fatal("a symlink must be refused")
	39	+	}
	40	+	// mkfs failure is reported.
	41	+	bad := &cmdrun.Fake{Handle: func(cmdrun.Call) ([]byte, error) { return nil, errors.New("boom") }}
	42	+	if err := CreateScratch(context.Background(), bad, filepath.Join(dir, "x.ext4"), 1, -1); err == nil \|\| !strings.Contains(err.Error(), "format scratch disk") {
	43	+	t.Fatalf("%v", err)
	44	+	}
	45	+	}
	46	+
	47	+	// C14: the mkfs.ext4 version is checked.
	48	+	func TestCheckMkfs(t *testing.T) {
	49	+	ver := func(out string, err error) cmdrun.Runner {
	50	+	return &cmdrun.Fake{Handle: func(cmdrun.Call) ([]byte, error) { return []byte(out), err }}
	51	+	}
	52	+	for name, tc := range map[string]struct {
	53	+	r cmdrun.Runner
	54	+	ok bool
	55	+	}{
	56	+	"current": {ver("mke2fs 1.47.4 (6-Mar-2025)\n", nil), true},
	57	+	"exit 1 on -V": {ver("mke2fs 1.46.5 (30-Dec-2021)\nUsing EXT2FS Library version 1.46.5\n", errors.New("exit status 1")), true},
	58	+	"minimum": {ver("mke2fs 1.43 (17-May-2016)\n", nil), true},
	59	+	"too old": {ver("mke2fs 1.42.13 (17-May-2015)\n", nil), false},
	60	+	"major 0": {ver("mke2fs 0.9 (1-Jan-2000)\n", nil), false},
	61	+	"garbage": {ver("hello\n", nil), false},
	62	+	"missing": {ver("", errors.New("executable file not found")), false},
	63	+	} {
	64	+	err := CheckMkfs(context.Background(), tc.r)
	65	+	if (err == nil) != tc.ok {
	66	+	t.Errorf("%s: err=%v ok=%v", name, err, tc.ok)
	67	+	}
	68	+	}
	69	+	if err := CheckMkfs(context.Background(), ver("mke2fs 1.42.9 (4-Feb-2014)", nil)); err == nil \|\| !strings.Contains(err.Error(), "1.43") {
	70	+	t.Errorf("the error must name the minimum: %v", err)
	71	+	}
	72	+	}
	73	+
	74	+	const mountinfo = `22 28 0:21 / /sys rw,nosuid shared:7 - sysfs sysfs rw
	75	+	36 28 7:0 / /var/lib/ermite-runner/containers/er00112233/scratch rw,nosuid,nodev shared:9 - ext4 /dev/loop0 rw
	76	+	37 28 7:1 / /srv/with\040space rw - ext4 /dev/loop1 rw
	77	+	`
	78	+
	79	+	func TestMounterIsMountedAndUnmount(t *testing.T) {
	80	+	mi := filepath.Join(t.TempDir(), "mountinfo")
	81	+	_ = os.WriteFile(mi, []byte(mountinfo), 0o644)
	82	+	f := &cmdrun.Fake{}
	83	+	m := Mounter{Run: f, Mountinfo: mi}
	84	+	for dir, want := range map[string]bool{
	85	+	"/var/lib/ermite-runner/containers/er00112233/scratch": true,
	86	+	"/srv/with space": true,
	87	+	"/var/lib/ermite-runner/containers/er00112233": false,
	88	+	"/nowhere": false,
	89	+	} {
	90	+	if got, err := m.IsMounted(dir); err != nil \|\| got != want {
	91	+	t.Errorf("%s: %v %v", dir, got, err)
	92	+	}
	93	+	}
	94	+	if err := m.Unmount(context.Background(), "/nowhere"); err != nil \|\| len(f.Calls) != 0 {
	95	+	t.Fatalf("a directory that is not mounted needs no umount: %v %v", err, f.Lines())
	96	+	}
	97	+	if err := m.Unmount(context.Background(), "/var/lib/ermite-runner/containers/er00112233/scratch"); err != nil {
	98	+	t.Fatal(err)
	99	+	}
	100	+	if got := f.Lines(); len(got) != 1 \|\| got[0] != "umount /var/lib/ermite-runner/containers/er00112233/scratch" {
	101	+	t.Fatalf("%v", got)
	102	+	}
	103	+	if err := m.Mount(context.Background(), "/s/scratch.ext4", "/s/mnt"); err != nil {
	104	+	t.Fatal(err)
	105	+	}
	106	+	if last := f.Lines()[1]; !strings.Contains(last, "loop,noatime") \|\| !strings.HasSuffix(last, "/s/scratch.ext4 /s/mnt") {
	107	+	t.Fatalf("mount call: %s", last)
	108	+	}
	109	+	bad := Mounter{Run: &cmdrun.Fake{Handle: func(cmdrun.Call) ([]byte, error) { return nil, errors.New("busy") }}, Mountinfo: mi}
	110	+	if err := bad.Unmount(context.Background(), "/srv/with space"); err == nil {
	111	+	t.Fatal("umount failure must be reported")
	112	+	}
	113	+	if _, err := (Mounter{Mountinfo: "/nonexistent/mountinfo"}).IsMounted("/x"); err == nil {
	114	+	t.Fatal("an unreadable mountinfo must be an error, not 'not mounted'")
	115	+	}
	116	+	}
	117	+
	118	+	func TestUnescapeMountinfo(t *testing.T) {
	119	+	for in, want := range map[string]string{
	120	+	`/plain`: "/plain", `/a\040b`: "/a b", `/tab\011x`: "/tab\tx", `/bs\134x`: `/bs\x`, `/bad\9zz`: `/bad\9zz`, `/end\04`: `/end\04`,
	121	+	} {
	122	+	if got := unescapeMountinfo(in); got != want {
	123	+	t.Errorf("%q: got %q want %q", in, got, want)
	124	+	}
	125	+	}
	126	+	}

@@ -0,0 +1,126 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package disk
	4	+
	5	+	import (
	6	+	"context"
	7	+	"errors"
	8	+	"os"
	9	+	"path/filepath"
	10	+	"strings"
	11	+	"testing"
	12	+
	13	+	"bnema.dev/ermite-runner/internal/cmdrun"
	14	+	)
	15	+
	16	+	func TestCreateScratchIsSparseExclusiveAndNoFollow(t *testing.T) {
	17	+	dir := t.TempDir()
	18	+	run := &cmdrun.Fake{}
	19	+	p := filepath.Join(dir, "scratch.ext4")
	20	+	if err := CreateScratch(context.Background(), run, p, 64, os.Getuid()); err != nil {
	21	+	t.Fatal(err)
	22	+	}
	23	+	st, _ := os.Stat(p)
	24	+	if st.Size() != 64<<20 \|\| st.Mode().Perm() != 0o600 {
	25	+	t.Fatalf("%v", st)
	26	+	}
	27	+	if got := run.Lines(); len(got) != 1 \|\| !strings.Contains(got[0], "mkfs.ext4") \|\| !strings.Contains(got[0], "^has_journal") {
	28	+	t.Fatalf("mkfs call: %v", got)
	29	+	}
	30	+	if err := CreateScratch(context.Background(), run, p, 64, -1); err == nil {
	31	+	t.Fatal("an existing file must be refused")
	32	+	}
	33	+	victim := filepath.Join(dir, "victim")
	34	+	_ = os.WriteFile(victim, []byte("x"), 0o600)
	35	+	link := filepath.Join(dir, "link")
	36	+	_ = os.Symlink(victim, link)
	37	+	if err := CreateScratch(context.Background(), run, link, 1, -1); err == nil {
	38	+	t.Fatal("a symlink must be refused")
	39	+	}
	40	+	// mkfs failure is reported.
	41	+	bad := &cmdrun.Fake{Handle: func(cmdrun.Call) ([]byte, error) { return nil, errors.New("boom") }}
	42	+	if err := CreateScratch(context.Background(), bad, filepath.Join(dir, "x.ext4"), 1, -1); err == nil \|\| !strings.Contains(err.Error(), "format scratch disk") {
	43	+	t.Fatalf("%v", err)
	44	+	}
	45	+	}
	46	+
	47	+	// C14: the mkfs.ext4 version is checked.
	48	+	func TestCheckMkfs(t *testing.T) {
	49	+	ver := func(out string, err error) cmdrun.Runner {
	50	+	return &cmdrun.Fake{Handle: func(cmdrun.Call) ([]byte, error) { return []byte(out), err }}
	51	+	}
	52	+	for name, tc := range map[string]struct {
	53	+	r cmdrun.Runner
	54	+	ok bool
	55	+	}{
	56	+	"current": {ver("mke2fs 1.47.4 (6-Mar-2025)\n", nil), true},
	57	+	"exit 1 on -V": {ver("mke2fs 1.46.5 (30-Dec-2021)\nUsing EXT2FS Library version 1.46.5\n", errors.New("exit status 1")), true},
	58	+	"minimum": {ver("mke2fs 1.43 (17-May-2016)\n", nil), true},
	59	+	"too old": {ver("mke2fs 1.42.13 (17-May-2015)\n", nil), false},
	60	+	"major 0": {ver("mke2fs 0.9 (1-Jan-2000)\n", nil), false},
	61	+	"garbage": {ver("hello\n", nil), false},
	62	+	"missing": {ver("", errors.New("executable file not found")), false},
	63	+	} {
	64	+	err := CheckMkfs(context.Background(), tc.r)
	65	+	if (err == nil) != tc.ok {
	66	+	t.Errorf("%s: err=%v ok=%v", name, err, tc.ok)
	67	+	}
	68	+	}
	69	+	if err := CheckMkfs(context.Background(), ver("mke2fs 1.42.9 (4-Feb-2014)", nil)); err == nil \|\| !strings.Contains(err.Error(), "1.43") {
	70	+	t.Errorf("the error must name the minimum: %v", err)
	71	+	}
	72	+	}
	73	+
	74	+	const mountinfo = `22 28 0:21 / /sys rw,nosuid shared:7 - sysfs sysfs rw
	75	+	36 28 7:0 / /var/lib/ermite-runner/containers/er00112233/scratch rw,nosuid,nodev shared:9 - ext4 /dev/loop0 rw
	76	+	37 28 7:1 / /srv/with\040space rw - ext4 /dev/loop1 rw
	77	+	`
	78	+
	79	+	func TestMounterIsMountedAndUnmount(t *testing.T) {
	80	+	mi := filepath.Join(t.TempDir(), "mountinfo")
	81	+	_ = os.WriteFile(mi, []byte(mountinfo), 0o644)
	82	+	f := &cmdrun.Fake{}
	83	+	m := Mounter{Run: f, Mountinfo: mi}
	84	+	for dir, want := range map[string]bool{
	85	+	"/var/lib/ermite-runner/containers/er00112233/scratch": true,
	86	+	"/srv/with space": true,
	87	+	"/var/lib/ermite-runner/containers/er00112233": false,
	88	+	"/nowhere": false,
	89	+	} {
	90	+	if got, err := m.IsMounted(dir); err != nil \|\| got != want {
	91	+	t.Errorf("%s: %v %v", dir, got, err)
	92	+	}
	93	+	}
	94	+	if err := m.Unmount(context.Background(), "/nowhere"); err != nil \|\| len(f.Calls) != 0 {
	95	+	t.Fatalf("a directory that is not mounted needs no umount: %v %v", err, f.Lines())
	96	+	}
	97	+	if err := m.Unmount(context.Background(), "/var/lib/ermite-runner/containers/er00112233/scratch"); err != nil {
	98	+	t.Fatal(err)
	99	+	}
	100	+	if got := f.Lines(); len(got) != 1 \|\| got[0] != "umount /var/lib/ermite-runner/containers/er00112233/scratch" {
	101	+	t.Fatalf("%v", got)
	102	+	}
	103	+	if err := m.Mount(context.Background(), "/s/scratch.ext4", "/s/mnt"); err != nil {
	104	+	t.Fatal(err)
	105	+	}
	106	+	if last := f.Lines()[1]; !strings.Contains(last, "loop,noatime") \|\| !strings.HasSuffix(last, "/s/scratch.ext4 /s/mnt") {
	107	+	t.Fatalf("mount call: %s", last)
	108	+	}
	109	+	bad := Mounter{Run: &cmdrun.Fake{Handle: func(cmdrun.Call) ([]byte, error) { return nil, errors.New("busy") }}, Mountinfo: mi}
	110	+	if err := bad.Unmount(context.Background(), "/srv/with space"); err == nil {
	111	+	t.Fatal("umount failure must be reported")
	112	+	}
	113	+	if _, err := (Mounter{Mountinfo: "/nonexistent/mountinfo"}).IsMounted("/x"); err == nil {
	114	+	t.Fatal("an unreadable mountinfo must be an error, not 'not mounted'")
	115	+	}
	116	+	}
	117	+
	118	+	func TestUnescapeMountinfo(t *testing.T) {
	119	+	for in, want := range map[string]string{
	120	+	`/plain`: "/plain", `/a\040b`: "/a b", `/tab\011x`: "/tab\tx", `/bs\134x`: `/bs\x`, `/bad\9zz`: `/bad\9zz`, `/end\04`: `/end\04`,
	121	+	} {
	122	+	if got := unescapeMountinfo(in); got != want {
	123	+	t.Errorf("%q: got %q want %q", in, got, want)
	124	+	}
	125	+	}
	126	+	}

A internal/sandbox/disk/mount.go +127 -0

@@ -0,0 +1,127 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package disk
	4	+
	5	+	import (
	6	+	"bufio"
	7	+	"context"
	8	+	"fmt"
	9	+	"os"
	10	+	"strings"
	11	+
	12	+	"bnema.dev/ermite-runner/internal/cmdrun"
	13	+	)
	14	+
	15	+	// Mounter loop-mounts and unmounts scratch disks. It needs root (loop devices),
	16	+	// like the rest of the container mode.
	17	+	type Mounter struct {
	18	+	Run cmdrun.Runner
	19	+	// Mountinfo is /proc/self/mountinfo; tests point it at a fixture.
	20	+	Mountinfo string
	21	+	}
	22	+
	23	+	// Mount loop-mounts the ext4 file at dir (which must exist). The mount is
	24	+	// deliberately not nosuid/nodev: the nested dockerd keeps its overlay2 layers
	25	+	// here, and overlayfs opens the real files of a layer through the layer's own
	26	+	// mount, so device nodes and setuid binaries of container images would stop
	27	+	// working on a nodev/nosuid backing mount.
	28	+	func (m Mounter) Mount(ctx context.Context, file, dir string) error {
	29	+	if _, err := m.Run.Run(ctx, nil, "mount", "-o", "loop,noatime", "-t", "ext4", file, dir); err != nil {
	30	+	return fmt.Errorf("loop-mount scratch disk: %w", err)
	31	+	}
	32	+	return nil
	33	+	}
	34	+
	35	+	// Unmount unmounts dir; the loop device is released with it (mount -o loop sets
	36	+	// autoclear). A directory that is not a mount point is not an error.
	37	+	func (m Mounter) Unmount(ctx context.Context, dir string) error {
	38	+	mounted, err := m.IsMounted(dir)
	39	+	if err != nil {
	40	+	return err
	41	+	}
	42	+	if !mounted {
	43	+	return nil
	44	+	}
	45	+	if _, err := m.Run.Run(ctx, nil, "umount", dir); err != nil {
	46	+	return fmt.Errorf("unmount scratch disk: %w", err)
	47	+	}
	48	+	return nil
	49	+	}
	50	+
	51	+	// IsMounted reports whether dir is a mount point.
	52	+	func (m Mounter) IsMounted(dir string) (bool, error) {
	53	+	mp, err := m.MountPoints()
	54	+	if err != nil {
	55	+	return false, err
	56	+	}
	57	+	for _, p := range mp {
	58	+	if p == dir {
	59	+	return true, nil
	60	+	}
	61	+	}
	62	+	return false, nil
	63	+	}
	64	+
	65	+	// MountPoints lists every mount point of the mount namespace.
	66	+	func (m Mounter) MountPoints() ([]string, error) {
	67	+	path := m.Mountinfo
	68	+	if path == "" {
	69	+	path = "/proc/self/mountinfo"
	70	+	}
	71	+	f, err := os.Open(path)
	72	+	if err != nil {
	73	+	return nil, err
	74	+	}
	75	+	defer func() { _ = f.Close() }()
	76	+	var out []string
	77	+	sc := bufio.NewScanner(f)
	78	+	sc.Buffer(make([]byte, 64<<10), 1<<20)
	79	+	for sc.Scan() {
	80	+	if mp, ok := mountPointOf(sc.Text()); ok {
	81	+	out = append(out, mp)
	82	+	}
	83	+	}
	84	+	return out, sc.Err()
	85	+	}
	86	+
	87	+	// mountPointOf extracts the mount point (5th field) of a mountinfo line, with
	88	+	// the kernel's octal escapes (\040 for a space) decoded.
	89	+	func mountPointOf(line string) (string, bool) {
	90	+	f := strings.Fields(line)
	91	+	if len(f) < 5 {
	92	+	return "", false
	93	+	}
	94	+	return unescapeMountinfo(f[4]), true
	95	+	}
	96	+
	97	+	func unescapeMountinfo(s string) string {
	98	+	if !strings.Contains(s, `\`) {
	99	+	return s
	100	+	}
	101	+	var b strings.Builder
	102	+	for i := 0; i < len(s); i++ {
	103	+	if s[i] == '\\' && i+3 < len(s) {
	104	+	if v, ok := octal3(s[i+1 : i+4]); ok {
	105	+	b.WriteByte(v)
	106	+	i += 3
	107	+	continue
	108	+	}
	109	+	}
	110	+	b.WriteByte(s[i])
	111	+	}
	112	+	return b.String()
	113	+	}
	114	+
	115	+	func octal3(s string) (byte, bool) {
	116	+	if len(s) != 3 {
	117	+	return 0, false
	118	+	}
	119	+	var v int
	120	+	for _, c := range s {
	121	+	if c < '0' \|\| c > '7' {
	122	+	return 0, false
	123	+	}
	124	+	v = v*8 + int(c-'0')
	125	+	}
	126	+	return byte(v), v < 256
	127	+	}

@@ -0,0 +1,127 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package disk
	4	+
	5	+	import (
	6	+	"bufio"
	7	+	"context"
	8	+	"fmt"
	9	+	"os"
	10	+	"strings"
	11	+
	12	+	"bnema.dev/ermite-runner/internal/cmdrun"
	13	+	)
	14	+
	15	+	// Mounter loop-mounts and unmounts scratch disks. It needs root (loop devices),
	16	+	// like the rest of the container mode.
	17	+	type Mounter struct {
	18	+	Run cmdrun.Runner
	19	+	// Mountinfo is /proc/self/mountinfo; tests point it at a fixture.
	20	+	Mountinfo string
	21	+	}
	22	+
	23	+	// Mount loop-mounts the ext4 file at dir (which must exist). The mount is
	24	+	// deliberately not nosuid/nodev: the nested dockerd keeps its overlay2 layers
	25	+	// here, and overlayfs opens the real files of a layer through the layer's own
	26	+	// mount, so device nodes and setuid binaries of container images would stop
	27	+	// working on a nodev/nosuid backing mount.
	28	+	func (m Mounter) Mount(ctx context.Context, file, dir string) error {
	29	+	if _, err := m.Run.Run(ctx, nil, "mount", "-o", "loop,noatime", "-t", "ext4", file, dir); err != nil {
	30	+	return fmt.Errorf("loop-mount scratch disk: %w", err)
	31	+	}
	32	+	return nil
	33	+	}
	34	+
	35	+	// Unmount unmounts dir; the loop device is released with it (mount -o loop sets
	36	+	// autoclear). A directory that is not a mount point is not an error.
	37	+	func (m Mounter) Unmount(ctx context.Context, dir string) error {
	38	+	mounted, err := m.IsMounted(dir)
	39	+	if err != nil {
	40	+	return err
	41	+	}
	42	+	if !mounted {
	43	+	return nil
	44	+	}
	45	+	if _, err := m.Run.Run(ctx, nil, "umount", dir); err != nil {
	46	+	return fmt.Errorf("unmount scratch disk: %w", err)
	47	+	}
	48	+	return nil
	49	+	}
	50	+
	51	+	// IsMounted reports whether dir is a mount point.
	52	+	func (m Mounter) IsMounted(dir string) (bool, error) {
	53	+	mp, err := m.MountPoints()
	54	+	if err != nil {
	55	+	return false, err
	56	+	}
	57	+	for _, p := range mp {
	58	+	if p == dir {
	59	+	return true, nil
	60	+	}
	61	+	}
	62	+	return false, nil
	63	+	}
	64	+
	65	+	// MountPoints lists every mount point of the mount namespace.
	66	+	func (m Mounter) MountPoints() ([]string, error) {
	67	+	path := m.Mountinfo
	68	+	if path == "" {
	69	+	path = "/proc/self/mountinfo"
	70	+	}
	71	+	f, err := os.Open(path)
	72	+	if err != nil {
	73	+	return nil, err
	74	+	}
	75	+	defer func() { _ = f.Close() }()
	76	+	var out []string
	77	+	sc := bufio.NewScanner(f)
	78	+	sc.Buffer(make([]byte, 64<<10), 1<<20)
	79	+	for sc.Scan() {
	80	+	if mp, ok := mountPointOf(sc.Text()); ok {
	81	+	out = append(out, mp)
	82	+	}
	83	+	}
	84	+	return out, sc.Err()
	85	+	}
	86	+
	87	+	// mountPointOf extracts the mount point (5th field) of a mountinfo line, with
	88	+	// the kernel's octal escapes (\040 for a space) decoded.
	89	+	func mountPointOf(line string) (string, bool) {
	90	+	f := strings.Fields(line)
	91	+	if len(f) < 5 {
	92	+	return "", false
	93	+	}
	94	+	return unescapeMountinfo(f[4]), true
	95	+	}
	96	+
	97	+	func unescapeMountinfo(s string) string {
	98	+	if !strings.Contains(s, `\`) {
	99	+	return s
	100	+	}
	101	+	var b strings.Builder
	102	+	for i := 0; i < len(s); i++ {
	103	+	if s[i] == '\\' && i+3 < len(s) {
	104	+	if v, ok := octal3(s[i+1 : i+4]); ok {
	105	+	b.WriteByte(v)
	106	+	i += 3
	107	+	continue
	108	+	}
	109	+	}
	110	+	b.WriteByte(s[i])
	111	+	}
	112	+	return b.String()
	113	+	}
	114	+
	115	+	func octal3(s string) (byte, bool) {
	116	+	if len(s) != 3 {
	117	+	return 0, false
	118	+	}
	119	+	var v int
	120	+	for _, c := range s {
	121	+	if c < '0' \|\| c > '7' {
	122	+	return 0, false
	123	+	}
	124	+	v = v*8 + int(c-'0')
	125	+	}
	126	+	return byte(v), v < 256
	127	+	}

M internal/sandbox/vm/driver.go +5 -1


@@ -30,6 +30,7 @@ import (
30	30		"bnema.dev/ermite-runner/internal/cmdrun"
31	31		"bnema.dev/ermite-runner/internal/netpolicy"
32	32		"bnema.dev/ermite-runner/internal/sandbox"
	33	+	"bnema.dev/ermite-runner/internal/sandbox/disk"
33	34		"bnema.dev/ermite-runner/internal/sandbox/netsetup"
34	35		)
35	36

@@ -127,6 +128,9 @@ func New(ctx context.Context, cfg sandbox.Config, opt Options) (*Driver, error)
127	128		}
128	129		}
129	130		sandbox.WarnIfLowSpace(ctx, cfg.StateDir, cfg.Capacity, cfg.JobDiskMiB, opt.FreeSpace)
	131	+	if err := disk.CheckMkfs(ctx, opt.Run); err != nil {
	132	+	return fail(err)
	133	+	}
130	134		if d.execDisk, err = ensureExecDisk(ctx, opt.Run, exe, cfg.StateDir); err != nil {
131	135		return fail(err)
132	136		}

@@ -362,7 +366,7 @@ func (s *vmSandbox) bootFiles(ctx context.Context, spec sandbox.JobSpec) (vmFile
362	366		if s.lay.jailed {
363	367		owner = s.lay.uid
364	368		}
365		-	return f, makeScratch(ctx, d.opt.Run, s.lay.host("scratch.ext4"), spec.DiskMiB, owner)
	369	+	return f, disk.CreateScratch(ctx, d.opt.Run, s.lay.host("scratch.ext4"), spec.DiskMiB, owner)
366	370		}
367	371
368	372		// bootVsock listens on the host side of the vsock stream: Firecracker connects

 	"bnema.dev/ermite-runner/internal/cmdrun"
 	"bnema.dev/ermite-runner/internal/netpolicy"
 	"bnema.dev/ermite-runner/internal/sandbox"
+	"bnema.dev/ermite-runner/internal/sandbox/disk"
 	"bnema.dev/ermite-runner/internal/sandbox/netsetup"
 )
 		}
 	}
 	sandbox.WarnIfLowSpace(ctx, cfg.StateDir, cfg.Capacity, cfg.JobDiskMiB, opt.FreeSpace)
+	if err := disk.CheckMkfs(ctx, opt.Run); err != nil {
+		return fail(err)
+	}
 	if d.execDisk, err = ensureExecDisk(ctx, opt.Run, exe, cfg.StateDir); err != nil {
 		return fail(err)
 	}
 	if s.lay.jailed {
 		owner = s.lay.uid
 	}
-	return f, makeScratch(ctx, d.opt.Run, s.lay.host("scratch.ext4"), spec.DiskMiB, owner)
+	return f, disk.CreateScratch(ctx, d.opt.Run, s.lay.host("scratch.ext4"), spec.DiskMiB, owner)
 }
 // bootVsock listens on the host side of the vsock stream: Firecracker connects

M internal/sandbox/vm/images.go +0 -31


@@ -10,7 +10,6 @@ import (
10	10		"io"
11	11		"os"
12	12		"path/filepath"
13		-	"syscall"
14	13
15	14		"bnema.dev/ermite-runner/internal/cmdrun"
16	15		"bnema.dev/ermite-runner/internal/sandbox"

@@ -83,36 +82,6 @@ func ensureExecDisk(ctx context.Context, run cmdrun.Runner, exe, stateDir string
83	82		return dest, nil
84	83		}
85	84
86		-	// makeScratch creates the per-job sparse scratch disk of sizeMiB and formats it.
87		-	// The file is created exclusively without following symlinks and handed to
88		-	// owner (fchown on the open descriptor, never by path) when owner >= 0.
89		-	func makeScratch(ctx context.Context, run cmdrun.Runner, path string, sizeMiB, owner int) error {
90		-	f, err := os.OpenFile(path, os.O_WRONLY\|os.O_CREATE\|os.O_EXCL\|syscall.O_NOFOLLOW, 0o600)
91		-	if err != nil {
92		-	return err
93		-	}
94		-	if owner >= 0 {
95		-	if err := f.Chown(owner, owner); err != nil {
96		-	_ = f.Close()
97		-	return fmt.Errorf("chown scratch disk: %w", err)
98		-	}
99		-	}
100		-	if err := f.Truncate(int64(sizeMiB) << 20); err != nil {
101		-	_ = f.Close()
102		-	return err
103		-	}
104		-	if err := f.Close(); err != nil {
105		-	return err
106		-	}
107		-	// No journal and no reserved blocks: it is a throw-away disk. Lazy table
108		-	// initialisation keeps creation instant and the file sparse.
109		-	if _, err := run.Run(ctx, nil, "mkfs.ext4", "-q", "-F", "-L", "ermite-scratch", "-m", "0",
110		-	"-O", "^has_journal,^resize_inode", "-E", "lazy_itable_init=1,nodiscard", path); err != nil {
111		-	return fmt.Errorf("format scratch disk: %w", err)
112		-	}
113		-	return nil
114		-	}
115		-
116	85		func hashFile(path string) (sum string, size int64, err error) {
117	86		f, err := os.Open(path)
118	87		if err != nil {


@@ -10,7 +10,6 @@ import (
10		"io"	10	"io"
11		"os"	11	"os"
12		"path/filepath"	12	"path/filepath"
13	-	"syscall"
14			13
15		"bnema.dev/ermite-runner/internal/cmdrun"	14	"bnema.dev/ermite-runner/internal/cmdrun"
16		"bnema.dev/ermite-runner/internal/sandbox"	15	"bnema.dev/ermite-runner/internal/sandbox"

@@ -83,36 +82,6 @@ func ensureExecDisk(ctx context.Context, run cmdrun.Runner, exe, stateDir string
83		return dest, nil	82	return dest, nil
84		}	83	}
85			84
86	-	// makeScratch creates the per-job sparse scratch disk of sizeMiB and formats it.
87	-	// The file is created exclusively without following symlinks and handed to
88	-	// owner (fchown on the open descriptor, never by path) when owner >= 0.
89	-	func makeScratch(ctx context.Context, run cmdrun.Runner, path string, sizeMiB, owner int) error {
90	-	f, err := os.OpenFile(path, os.O_WRONLY\|os.O_CREATE\|os.O_EXCL\|syscall.O_NOFOLLOW, 0o600)
91	-	if err != nil {
92	-	return err
93	-	}
94	-	if owner >= 0 {
95	-	if err := f.Chown(owner, owner); err != nil {
96	-	_ = f.Close()
97	-	return fmt.Errorf("chown scratch disk: %w", err)
98	-	}
99	-	}
100	-	if err := f.Truncate(int64(sizeMiB) << 20); err != nil {
101	-	_ = f.Close()
102	-	return err
103	-	}
104	-	if err := f.Close(); err != nil {
105	-	return err
106	-	}
107	-	// No journal and no reserved blocks: it is a throw-away disk. Lazy table
108	-	// initialisation keeps creation instant and the file sparse.
109	-	if _, err := run.Run(ctx, nil, "mkfs.ext4", "-q", "-F", "-L", "ermite-scratch", "-m", "0",
110	-	"-O", "^has_journal,^resize_inode", "-E", "lazy_itable_init=1,nodiscard", path); err != nil {
111	-	return fmt.Errorf("format scratch disk: %w", err)
112	-	}
113	-	return nil
114	-	}
115	-
116		func hashFile(path string) (sum string, size int64, err error) {	85	func hashFile(path string) (sum string, size int64, err error) {
117		f, err := os.Open(path)	86	f, err := os.Open(path)
118		if err != nil {	87	if err != nil {

M internal/sandbox/vm/vm_test.go +7 -4


@@ -27,6 +27,7 @@ import (
27	27		"bnema.dev/ermite-runner/internal/execstream"
28	28		"bnema.dev/ermite-runner/internal/execstream/execpb"
29	29		"bnema.dev/ermite-runner/internal/sandbox"
	30	+	"bnema.dev/ermite-runner/internal/sandbox/disk"
30	31		"bnema.dev/ermite-runner/internal/sandbox/sandboxtest"
31	32		)
32	33

@@ -145,6 +146,8 @@ func newEnvNamed(t testing.T, mode, argv0 string) env {
145	146		e.mu.Lock()
146	147		defer e.mu.Unlock()
147	148		switch {
	149	+	case c.Name == "mkfs.ext4" && len(c.Args) == 1 && c.Args[0] == "-V":
	150	+	return []byte("mke2fs 1.47.4 (6-Mar-2025)\n"), nil
148	151		case c.Name == "ip" && len(c.Args) > 1 && c.Args[0] == "tuntap":
149	152		e.taps[c.Args[3]] = true
150	153		case c.Name == "ip" && len(c.Args) > 2 && c.Args[0] == "link" && c.Args[1] == "del":

@@ -537,7 +540,7 @@ func TestExecDiskIsKeyedByExecutableHash(t *testing.T) {
537	540		t.Fatal("exec disk rebuilt for the same executable")
538	541		}
539	542		for _, c := range e.run.Calls[n:] {
540		-	if c.Name == "mkfs.ext4" {
	543	+	if c.Name == "mkfs.ext4" && !(len(c.Args) == 1 && c.Args[0] == "-V") {
541	544		t.Fatal("mkfs run again for an unchanged executable")
542	545		}
543	546		}

@@ -964,7 +967,7 @@ func TestMakeScratchRefusesSymlinksAndExistingFiles(t *testing.T) {
964	967		t.Fatal(err)
965	968		}
966	969		run := &cmdrun.Fake{}
967		-	if err := makeScratch(context.Background(), run, link, 8, -1); err == nil {
	970	+	if err := disk.CreateScratch(context.Background(), run, link, 8, -1); err == nil {
968	971		t.Fatal("a planted symlink must be refused")
969	972		}
970	973		if b, _ := os.ReadFile(target); string(b) != "precious" {

@@ -974,13 +977,13 @@ func TestMakeScratchRefusesSymlinksAndExistingFiles(t *testing.T) {
974	977		t.Fatalf("mkfs must not run: %v", run.Lines())
975	978		}
976	979		ok := filepath.Join(dir, "fresh.ext4")
977		-	if err := makeScratch(context.Background(), run, ok, 8, os.Getuid()); err != nil {
	980	+	if err := disk.CreateScratch(context.Background(), run, ok, 8, os.Getuid()); err != nil {
978	981		t.Fatal(err)
979	982		}
980	983		if st, err := os.Stat(ok); err != nil \|\| st.Size() != 8<<20 \|\| st.Mode().Perm() != 0o600 {
981	984		t.Fatalf("%v %v", st, err)
982	985		}
983		-	if err := makeScratch(context.Background(), run, ok, 8, -1); err == nil {
	986	+	if err := disk.CreateScratch(context.Background(), run, ok, 8, -1); err == nil {
984	987		t.Fatal("an existing file must be refused (O_EXCL)")
985	988		}
986	989		}

 	"bnema.dev/ermite-runner/internal/execstream"
 	"bnema.dev/ermite-runner/internal/execstream/execpb"
 	"bnema.dev/ermite-runner/internal/sandbox"
+	"bnema.dev/ermite-runner/internal/sandbox/disk"
 	"bnema.dev/ermite-runner/internal/sandbox/sandboxtest"
 )
 		e.mu.Lock()
 		defer e.mu.Unlock()
 		switch {
+		case c.Name == "mkfs.ext4" && len(c.Args) == 1 && c.Args[0] == "-V":
+			return []byte("mke2fs 1.47.4 (6-Mar-2025)\n"), nil
 		case c.Name == "ip" && len(c.Args) > 1 && c.Args[0] == "tuntap":
 			e.taps[c.Args[3]] = true
 		case c.Name == "ip" && len(c.Args) > 2 && c.Args[0] == "link" && c.Args[1] == "del":
 		t.Fatal("exec disk rebuilt for the same executable")
 	}
 	for _, c := range e.run.Calls[n:] {
-		if c.Name == "mkfs.ext4" {
+		if c.Name == "mkfs.ext4" && !(len(c.Args) == 1 && c.Args[0] == "-V") {
 			t.Fatal("mkfs run again for an unchanged executable")
 		}
 	}
 		t.Fatal(err)
 	}
 	run := &cmdrun.Fake{}
-	if err := makeScratch(context.Background(), run, link, 8, -1); err == nil {
+	if err := disk.CreateScratch(context.Background(), run, link, 8, -1); err == nil {
 		t.Fatal("a planted symlink must be refused")
 	}
 	if b, _ := os.ReadFile(target); string(b) != "precious" {
 		t.Fatalf("mkfs must not run: %v", run.Lines())
 	}
 	ok := filepath.Join(dir, "fresh.ext4")
-	if err := makeScratch(context.Background(), run, ok, 8, os.Getuid()); err != nil {
+	if err := disk.CreateScratch(context.Background(), run, ok, 8, os.Getuid()); err != nil {
 		t.Fatal(err)
 	}
 	if st, err := os.Stat(ok); err != nil || st.Size() != 8<<20 || st.Mode().Perm() != 0o600 {
 		t.Fatalf("%v %v", st, err)
 	}
-	if err := makeScratch(context.Background(), run, ok, 8, -1); err == nil {
+	if err := disk.CreateScratch(context.Background(), run, ok, 8, -1); err == nil {
 		t.Fatal("an existing file must be refused (O_EXCL)")
 	}
 }