1// SPDX-License-Identifier: MIT
2
3// Package container is the container sandbox, for hosts without KVM: one
4// container per job with its own Docker daemon inside (the sysbox runtime when
5// the daemon lists it, else a privileged container), on its own bridge network
6// and with its own scratch disk. The scratch disk is a sparse ext4 file of
7// ERMITE_RUNNER_JOB_DISK, loop-mounted on the host (so it needs root, like the
8// rest of this mode) and bind-mounted at /scratch: the nested dockerd keeps its
9// data root there and the job workspace is a directory of it, so a job cannot
10// write more than its disk size to the host (the container's own writable layer
11// stays small: nothing big is written outside /scratch and /home/runner/work).
12// Everything Docker-side carries the label sandbox.LabelJob so that Reap finds
13// it after a crash.
14//
15// The executor stream is a Unix socket in a per-job directory of the runner's
16// state directory, bind-mounted into the container. The runner executable and
17// the host's external tools (Node.js) are bind-mounted read-only. Both paths are host paths given to the Docker daemon:
18// the daemon must run on the same host (and mount namespace) as the runner.
19package container
20
21import (
22 "context"
23 "errors"
24 "fmt"
25 "net"
26 "net/netip"
27 "os"
28 "path/filepath"
29 "slices"
30 "strings"
31 "sync"
32 "time"
33
34 "github.com/bnema/zerowrap"
35
36 "bnema.dev/ermite-runner/internal/cmdrun"
37 "bnema.dev/ermite-runner/internal/netpolicy"
38 "bnema.dev/ermite-runner/internal/sandbox"
39 "bnema.dev/ermite-runner/internal/sandbox/disk"
40 "bnema.dev/ermite-runner/internal/sandbox/netsetup"
41)
42
43// SysboxRuntime is the name of the sysbox runtime in `docker info`.
44const SysboxRuntime = "sysbox-runc"
45
46// bridgePrefix must match the "erbr*" wildcard of the nftables policy.
47const bridgePrefix = "erbr"
48
49// Options are the testable knobs of the driver.
50type Options struct {
51 Run cmdrun.Runner
52 // Net overrides the network plumbing (tests).
53 Net netsetup.Plumbing
54 // engine overrides the Docker client (tests).
55 engine engine
56 // UnsafeAllowRootlessForTests lets tests exercise the Engine API and the executor against
57 // the rootless daemon of a workstation. Never set outside tests: a nested
58 // dockerd cannot work in a rootless daemon, so the jobs would run without
59 // Docker. The factories of package drivers never set it.
60 UnsafeAllowRootlessForTests bool
61 // ConnectTimeout bounds container start + init + executor connection
62 // (default 90 s).
63 ConnectTimeout time.Duration
64 // Mounter overrides the loop mounts (tests).
65 Mounter *disk.Mounter
66 // FreeSpace overrides statfs for the free-space warning (tests).
67 FreeSpace func(string) (int64, error)
68}
69
70// Driver implements sandbox.Driver for containers.
71type Driver struct {
72 cfg sandbox.Config
73 opt Options
74 eng engine
75 net netsetup.Plumbing
76 mnt disk.Mounter
77 lock *sandbox.HostLock
78 image string
79 runtime string // "" when privileged
80 privileged bool
81 exe string
82
83 mu sync.Mutex
84 slots *sandbox.Slots
85}
86
87// New takes the host-wide lock, connects to the Docker daemon (DOCKER_HOST
88// honoured) and checks it: not rootless, the job image present (pulled if
89// missing), the runtime decided (sysbox when listed; else privileged, unless
90// ERMITE_RUNNER_DENY_PRIVILEGED=1, which refuses). IPv4 forwarding is enabled
91// host-wide and stays on (see the README).
92func New(ctx context.Context, cfg sandbox.Config, opt Options) (*Driver, error) {
93 if opt.Run == nil {
94 opt.Run = cmdrun.Exec{}
95 }
96 if opt.ConnectTimeout == 0 {
97 opt.ConnectTimeout = 90 * time.Second
98 }
99 d := &Driver{cfg: cfg, opt: opt, image: cfg.OCIImage(), slots: sandbox.NewSlots(cfg.Capacity)}
100 if opt.Mounter != nil {
101 d.mnt = *opt.Mounter
102 } else {
103 d.mnt = disk.Mounter{Run: opt.Run}
104 }
105 exe, err := cfg.ExecutablePath()
106 if err != nil {
107 return nil, err
108 }
109 if err := sandbox.CheckStatic(exe); err != nil {
110 return nil, err
111 }
112 d.exe = exe
113
114 if d.lock, err = sandbox.AcquireHostLock(cfg.LockFile); err != nil {
115 return nil, err
116 }
117 eng := opt.engine
118 if eng == nil {
119 e, err := newDockerEngine()
120 if err != nil {
121 _ = d.lock.Release()
122 return nil, err
123 }
124 eng = e
125 }
126 d.eng = eng
127 fail := func(err error) (*Driver, error) {
128 _ = eng.Close()
129 _ = d.lock.Release()
130 return nil, err
131 }
132
133 info, err := eng.Info(ctx)
134 if err != nil {
135 return fail(fmt.Errorf("cannot reach the Docker daemon (DOCKER_HOST): %w", err))
136 }
137 if info.Rootless && !opt.UnsafeAllowRootlessForTests {
138 return fail(errRootless)
139 }
140 if info.OSType != "" && info.OSType != "linux" {
141 return fail(fmt.Errorf("the Docker daemon runs %s containers, need linux", info.OSType))
142 }
143 log := zerowrap.FromCtx(ctx)
144 if slices.Contains(info.Runtimes, SysboxRuntime) {
145 d.runtime = SysboxRuntime
146 log.Info().Str("runtime", SysboxRuntime).Msg("container isolation: using the sysbox runtime")
147 } else {
148 if cfg.DenyPrivileged {
149 return fail(fmt.Errorf("the sysbox runtime is not available and %s=1 forbids privileged containers (install sysbox, or unset the setting to accept host-root-equivalent jobs)", sandbox.EnvDenyPrivileged))
150 }
151 d.privileged = true
152 log.Warn().Msg(privilegedWarning)
153 }
154 if err := disk.CheckMkfs(ctx, opt.Run); err != nil {
155 return fail(err)
156 }
157 if err := os.MkdirAll(cfg.StateDir, 0o755); err != nil {
158 return fail(fmt.Errorf("state directory: %w", err))
159 }
160 sandbox.WarnIfLowSpace(ctx, cfg.StateDir, cfg.Capacity, cfg.JobDiskMiB, opt.FreeSpace)
161 if err := eng.EnsureImage(ctx, d.image); err != nil {
162 return fail(err)
163 }
164 d.net = opt.Net
165 if d.net == nil {
166 d.net = newNet(opt.Run, cfg.DNSPort, info.Podman, cfg.GuestForge.DNSName())
167 }
168 if err := d.net.EnableForwarding(); err != nil {
169 return fail(err)
170 }
171 if cfg.GuestForge != nil {
172 log.Warn().Msg(cfg.GuestForge.Warning())
173 }
174 return d, nil
175}
176
177// newNet builds the host network plumbing. Only podman creates the bridge
178// address late (see daemonInfo.Podman), so only there the DNS listener may come
179// before the address; on Docker a missing gateway address stays a startup error.
180func newNet(run cmdrun.Runner, dnsPort int, podman bool, forgeHost string) *netsetup.Net {
181 n := netsetup.New(run, dnsPort, "")
182 n.DNS.FreeBind = podman
183 n.DNS.Override = forgeHost
184 return n
185}
186
187// maxSockPath is the longest Unix socket path Linux accepts (108 with the NUL).
188const maxSockPath = 107
189
190// privilegedWarning is logged when the driver starts and again at every job
191// start: a privileged container is host-root equivalent.
192const privilegedWarning = "privileged container: host-root equivalent, network policy best effort"
193
194// Mode implements sandbox.Driver.
195func (d *Driver) Mode() sandbox.Mode { return sandbox.Container }
196
197// Isolation implements sandbox.Driver.
198func (d *Driver) Isolation() sandbox.Isolation {
199 if d.privileged {
200 return sandbox.IsolationContainerPrivileged
201 }
202 return sandbox.IsolationContainerSysbox
203}
204
205// Privileged reports whether jobs run privileged (no sysbox).
206func (d *Driver) Privileged() bool { return d.privileged }
207
208// Close implements sandbox.Driver.
209func (d *Driver) Close() error {
210 if d.net != nil {
211 d.net.Close()
212 }
213 err := d.eng.Close()
214 return errors.Join(err, d.lock.Release())
215}
216
217// New implements sandbox.Driver.
218func (d *Driver) New() (sandbox.Sandbox, error) {
219 id, err := sandbox.NewID()
220 if err != nil {
221 return nil, err
222 }
223 d.mu.Lock()
224 defer d.mu.Unlock()
225 slot, err := d.slots.Acquire()
226 if err != nil {
227 return nil, err
228 }
229 return &containerSandbox{d: d, slot: slot, id: id}, nil
230}
231
232func (d *Driver) release(slot int) {
233 d.mu.Lock()
234 d.slots.Release(slot)
235 d.mu.Unlock()
236}
237
238func (d *Driver) markLeaked(slot int) {
239 d.mu.Lock()
240 d.slots.MarkLeaked(slot)
241 d.mu.Unlock()
242}
243
244func (d *Driver) runDir(id string) string { return filepath.Join(d.cfg.StateDir, "containers", id) }
245
246// sockDir holds only the executor socket: it is the one directory of the job
247// that is bind-mounted (as /run/ermite), so the job never sees the scratch disk
248// file, nor the other host paths of its own sandbox.
249func (d *Driver) sockDir(id string) string { return filepath.Join(d.runDir(id), "sock") }
250
251// scratchDir is where the loop-mounted scratch disk of a job is mounted.
252func (d *Driver) scratchDir(id string) string { return filepath.Join(d.runDir(id), "scratch") }
253
254// workDir is the job workspace, a directory of the scratch disk.
255func (d *Driver) workDir(id string) string { return filepath.Join(d.scratchDir(id), "work") }
256
257// Where the job workspace and its temporary files appear in the container.
258const (
259 containerWorkspace = "/home/runner/work"
260 containerTmp = "/tmp"
261)
262
263type containerSandbox struct {
264 d *Driver
265 slot int
266 id string
267 gate sandbox.BootGate
268
269 mu sync.Mutex
270 started bool
271 stopped bool
272 leakErr error
273 netID string
274 container string
275 mounted bool
276 listener net.Listener
277 conn net.Conn
278 iface string
279 gateway netip.Addr
280 attached bool
281}
282
283// ID implements sandbox.Sandbox.
284func (s *containerSandbox) ID() string { return s.id }
285
286// Start implements sandbox.Sandbox.
287func (s *containerSandbox) Start(ctx context.Context, spec sandbox.JobSpec) (sandbox.Conn, error) {
288 // A Stop from another goroutine aborts the boot through the gate.
289 bootCtx, err := s.gate.Begin(ctx)
290 if err != nil {
291 return nil, fmt.Errorf("container sandbox: %w", err)
292 }
293 defer s.gate.End()
294
295 s.mu.Lock()
296 defer s.mu.Unlock()
297 if s.started || s.stopped {
298 return nil, errors.New("container sandbox: Start called twice or after Stop")
299 }
300 s.started = true
301 d := s.d
302 log := zerowrap.FromCtx(ctx)
303 spec = d.cfg.Resolve(spec)
304 if d.privileged {
305 // At every job start, not just at driver start: the job log and the
306 // operator's logs must say what kind of isolation this job got.
307 log.Warn().Str("job", spec.JobID).Str("container", s.id).Msg(privilegedWarning)
308 }
309 if d.cfg.GuestForge != nil {
310 log.Warn().Str("job", spec.JobID).Str("container", s.id).Msg(d.cfg.GuestForge.Warning())
311 }
312 conn, err := s.run(bootCtx, spec)
313 if err != nil {
314 var tail string
315 if s.container != "" {
316 tail, _ = d.eng.LogsTail(context.WithoutCancel(ctx), s.container, 40)
317 }
318 cerr := s.cleanupLocked(context.WithoutCancel(ctx))
319 if tail != "" {
320 err = fmt.Errorf("%w\ncontainer log tail:\n%s", err, tail)
321 }
322 return nil, errors.Join(err, cerr)
323 }
324 log.Info().Str("container", s.container).Int("slot", s.slot).Str("job", spec.JobID).Bool("privileged", d.privileged).Msg("container running")
325 return conn, nil
326}
327
328// run runs the boot stages in order; each records what it created for cleanup.
329func (s *containerSandbox) run(ctx context.Context, spec sandbox.JobSpec) (sandbox.Conn, error) {
330 netw, err := netpolicy.Subnet(s.d.cfg.Network, s.slot)
331 if err != nil {
332 return nil, err
333 }
334 accepted, err := s.runSocket()
335 if err != nil {
336 return nil, err
337 }
338 if err := s.runScratch(ctx, spec); err != nil {
339 return nil, err
340 }
341 labels := map[string]string{sandbox.LabelJob: spec.JobID}
342 if err := s.runNetwork(ctx, netw, labels); err != nil {
343 return nil, err
344 }
345 exited, err := s.runContainer(ctx, spec, netw, labels)
346 if err != nil {
347 return nil, err
348 }
349 return s.waitExecutor(ctx, accepted, exited)
350}
351
352type acceptResult struct {
353 c net.Conn
354 err error
355}
356
357// runSocket creates the per-job directory and listens on the executor socket in
358// it. The directory exists before anything on the Docker side, so a crash
359// leaves a recognisable one.
360func (s *containerSandbox) runSocket() (<-chan acceptResult, error) {
361 d := s.d
362 for _, dir := range []string{d.runDir(s.id), d.sockDir(s.id)} {
363 if err := os.MkdirAll(dir, 0o711); err != nil {
364 return nil, err
365 }
366 if err := os.Chmod(dir, 0o711); err != nil {
367 return nil, err
368 }
369 }
370 dir := d.sockDir(s.id)
371 sock := filepath.Join(dir, sandbox.ContainerSocket)
372 // sockaddr_un.sun_path holds 108 bytes: say so instead of "invalid argument".
373 if len(sock) > maxSockPath {
374 return nil, fmt.Errorf("executor socket path %q is %d bytes, over the %d-byte limit of a Unix socket: use a shorter %s", sock, len(sock), maxSockPath, sandbox.EnvStateDir)
375 }
376 ln, err := net.Listen("unix", sock)
377 if err != nil {
378 return nil, fmt.Errorf("listen executor socket: %w", err)
379 }
380 s.listener = ln
381 // The container's root may be a mapped (sysbox) user: it must be able to
382 // connect. The path is private to the job (random id, 0711 directory).
383 if err := os.Chmod(sock, 0o666); err != nil {
384 return nil, err
385 }
386 accepted := make(chan acceptResult, 1)
387 go func() {
388 c, err := ln.Accept()
389 accepted <- acceptResult{c, err}
390 }()
391 return accepted, nil
392}
393
394// runScratch creates the sparse ext4 scratch disk (ERMITE_RUNNER_JOB_DISK) and
395// loop-mounts it; the workspace directory lives on it.
396func (s *containerSandbox) runScratch(ctx context.Context, spec sandbox.JobSpec) error {
397 d := s.d
398 file := filepath.Join(d.runDir(s.id), "scratch.ext4")
399 if err := disk.CreateScratch(ctx, d.opt.Run, file, spec.DiskMiB, -1); err != nil {
400 return err
401 }
402 mnt := d.scratchDir(s.id)
403 if err := os.MkdirAll(mnt, 0o755); err != nil {
404 return err
405 }
406 if err := d.mnt.Mount(ctx, file, mnt); err != nil {
407 return err
408 }
409 s.mounted = true
410 // Workspace, temp and the nested dockerd's data root on the scratch disk. The
411 // executor (phase G) sets the job layout and ownership inside the sandbox.
412 for _, sub := range []string{"work", "tmp", "docker"} {
413 if err := os.MkdirAll(filepath.Join(mnt, sub), 0o755); err != nil {
414 return err
415 }
416 }
417 return os.Chmod(filepath.Join(mnt, "tmp"), sandbox.TmpMode)
418}
419
420// runNetwork creates the job's bridge and installs the firewall policy.
421func (s *containerSandbox) runNetwork(ctx context.Context, netw netpolicy.Net, labels map[string]string) error {
422 d := s.d
423 s.iface = fmt.Sprintf("%s%d", bridgePrefix, s.slot)
424 s.gateway = netw.Gateway
425 netID, err := d.eng.CreateNetwork(ctx, networkSpec{Name: "ermite-" + s.id, Bridge: s.iface, Subnet: netw.Prefix, Gateway: netw.Gateway, Labels: labels})
426 if err != nil {
427 return err
428 }
429 s.netID = netID
430 s.attached = true
431 return d.net.Attach(ctx, netpolicy.Guest{Iface: s.iface, Gateway: netw.Gateway, Addr: netw.Guest, ForgePort: d.cfg.GuestForge.TCPPort()})
432}
433
434// runContainer creates and starts the container.
435func (s *containerSandbox) runContainer(ctx context.Context, spec sandbox.JobSpec, netw netpolicy.Net, labels map[string]string) (<-chan exitResult, error) {
436 d := s.d
437 entry := sandbox.GuestExecDir + "/" + sandbox.GuestExecName
438 binds := []bindMount{
439 {Source: d.exe, Target: entry, ReadOnly: true},
440 {Source: d.sockDir(s.id), Target: sandbox.ContainerRunDir},
441 {Source: d.scratchDir(s.id), Target: sandbox.GuestScratch},
442 {Source: d.workDir(s.id), Target: containerWorkspace},
443 {Source: filepath.Join(d.scratchDir(s.id), "tmp"), Target: containerTmp},
444 }
445 binds = append(binds, externalBinds(d.cfg.ExternalsOf())...)
446 cid, err := d.eng.CreateContainer(ctx, containerSpec{
447 Name: "ermite-" + s.id,
448 Image: d.image,
449 Entrypoint: []string{entry, "init"},
450 Env: []string{"ERMITE_CONNECT=unix:" + sandbox.ContainerRunDir + "/" + sandbox.ContainerSocket},
451 Labels: labels,
452 Runtime: d.runtime,
453 Privileged: d.privileged,
454 Network: "ermite-" + s.id,
455 IPv4: netw.Guest,
456 DNS: netw.Gateway,
457 Binds: binds,
458 MemoryMiB: spec.MemoryMiB,
459 CPUs: spec.CPUs,
460 Pids: d.cfg.JobPids,
461 })
462 if err != nil {
463 return nil, err
464 }
465 s.container = cid
466 exited := d.eng.WaitExit(ctx, cid)
467 if err := d.eng.StartContainer(ctx, cid); err != nil {
468 return nil, fmt.Errorf("start container: %w", err)
469 }
470 return exited, nil
471}
472
473// externalBinds are the read-only bind mounts of the host's external tools
474// (Node.js) at /home/runner/externals/<name>, GitHub's layout. A container needs
475// no more than the host path: it reads the tool directly.
476func externalBinds(set []sandbox.External) []bindMount {
477 var out []bindMount
478 for _, e := range set {
479 out = append(out, bindMount{Source: e.Dir, Target: sandbox.GuestExternalsDir + "/" + e.Name, ReadOnly: true})
480 }
481 return out
482}
483
484// waitExecutor waits for the executor to connect. Only ctx (the caller's
485// context, or Stop through the boot gate), the container exiting, and the
486// timeout end the wait.
487func (s *containerSandbox) waitExecutor(ctx context.Context, accepted <-chan acceptResult, exited <-chan exitResult) (sandbox.Conn, error) {
488 timer := time.NewTimer(s.d.opt.ConnectTimeout)
489 defer timer.Stop()
490 select {
491 case r := <-accepted:
492 if r.err != nil {
493 return nil, fmt.Errorf("accept executor connection: %w", r.err)
494 }
495 s.conn = r.c
496 return r.c, nil
497 case ex := <-exited:
498 if ex.Err != nil {
499 return nil, fmt.Errorf("waiting for the container: %w", ex.Err)
500 }
501 return nil, fmt.Errorf("container exited (code %d) before the executor connected", ex.Code)
502 case <-timer.C:
503 return nil, fmt.Errorf("executor did not connect within %s", s.d.opt.ConnectTimeout)
504 case <-ctx.Done():
505 return nil, context.Cause(ctx)
506 }
507}
508
509// Stop implements sandbox.Sandbox.
510func (s *containerSandbox) Stop(ctx context.Context) error {
511 // Abort a boot in progress first: Start holds s.mu for its whole duration.
512 s.gate.Abort()
513 s.mu.Lock()
514 defer s.mu.Unlock()
515 if s.stopped {
516 return s.leakErr
517 }
518 if !s.started {
519 s.releaseLocked()
520 return nil
521 }
522 return s.cleanupLocked(context.WithoutCancel(ctx))
523}
524
525func (s *containerSandbox) releaseLocked() {
526 s.stopped = true
527 s.d.release(s.slot)
528}
529
530// cleanupLocked removes the container first; only when it is confirmed gone are
531// the firewall policy, the network, the scratch mount and the files removed. A
532// container that may still run keeps its policy and bridge: tearing the policy
533// down under it would remove the isolation it relies on. Failures mark the
534// sandbox as leaked (sandbox.ErrLeaked): its slot stays taken until Reap.
535func (s *containerSandbox) cleanupLocked(ctx context.Context) error {
536 d := s.d
537 var errs []error
538 if s.conn != nil {
539 _ = s.conn.Close()
540 s.conn = nil
541 }
542 if s.listener != nil {
543 _ = s.listener.Close()
544 s.listener = nil
545 }
546 if s.container != "" {
547 if err := d.eng.RemoveContainer(ctx, s.container); err != nil {
548 errs = append(errs, fmt.Errorf("remove container: %w", err))
549 } else {
550 s.container = ""
551 }
552 }
553 if s.container == "" {
554 if s.attached {
555 if err := d.net.Detach(ctx, s.iface, s.gateway); err != nil {
556 errs = append(errs, err)
557 } else {
558 s.attached = false
559 }
560 }
561 if s.netID != "" {
562 if err := d.eng.RemoveNetwork(ctx, s.netID); err != nil {
563 errs = append(errs, fmt.Errorf("remove network: %w", err))
564 } else {
565 s.netID = ""
566 }
567 }
568 if s.mounted {
569 if err := d.mnt.Unmount(ctx, d.scratchDir(s.id)); err != nil {
570 errs = append(errs, err)
571 } else {
572 s.mounted = false
573 }
574 }
575 // Never delete files below a mount that is still there.
576 if !s.mounted {
577 if err := os.RemoveAll(d.runDir(s.id)); err != nil {
578 errs = append(errs, err)
579 }
580 }
581 }
582 if len(errs) == 0 {
583 s.releaseLocked()
584 return nil
585 }
586 s.stopped = true
587 d.markLeaked(s.slot)
588 s.leakErr = fmt.Errorf("%w: container sandbox %s: %w", sandbox.ErrLeaked, s.id, errors.Join(errs...))
589 log := zerowrap.FromCtx(ctx)
590 log.Error().Err(s.leakErr).Str("job", s.id).Msg("container cleanup incomplete; call Reap to finish it")
591 return s.leakErr
592}
593
594// Reap implements sandbox.Driver: remove every container and network carrying
595// the job label, the nftables table, the scratch mounts and the per-job
596// directories, and free the slots of leaked sandboxes. It returns ErrBusy while
597// a sandbox is running.
598func (d *Driver) Reap(ctx context.Context) error {
599 log := zerowrap.FromCtx(ctx)
600 d.mu.Lock()
601 busy := d.slots.Busy()
602 d.mu.Unlock()
603 if busy > 0 {
604 return fmt.Errorf("%w (%d running)", sandbox.ErrBusy, busy)
605 }
606 var errs []error
607 ids, err := d.eng.ListContainers(ctx, sandbox.LabelJob)
608 if err != nil {
609 errs = append(errs, fmt.Errorf("list containers: %w", err))
610 }
611 for _, id := range ids {
612 log.Warn().Str("container", id).Msg("removing leftover job container")
613 if err := d.eng.RemoveContainer(ctx, id); err != nil {
614 errs = append(errs, err)
615 }
616 }
617 nets, err := d.eng.ListNetworks(ctx, sandbox.LabelJob)
618 if err != nil {
619 errs = append(errs, fmt.Errorf("list networks: %w", err))
620 }
621 for _, n := range nets {
622 log.Warn().Str("network", n).Msg("removing leftover job network")
623 if err := d.eng.RemoveNetwork(ctx, n); err != nil {
624 errs = append(errs, err)
625 }
626 }
627 if err := d.net.Reset(ctx); err != nil {
628 errs = append(errs, fmt.Errorf("reset nftables table: %w", err))
629 }
630 // Unmount before deleting: a directory with a live mount below it must not
631 // be removed (and EBUSY would be the best case).
632 root := filepath.Join(d.cfg.StateDir, "containers")
633 mounts, merr := d.mnt.MountPoints()
634 if merr != nil {
635 errs = append(errs, fmt.Errorf("read mounts: %w", merr))
636 }
637 stuck := map[string]bool{}
638 for _, mp := range mounts {
639 rel, ok := strings.CutPrefix(mp, root+"/")
640 if !ok {
641 continue
642 }
643 id, _, _ := strings.Cut(rel, "/")
644 if !sandbox.ValidID(id) {
645 continue
646 }
647 log.Warn().Str("mount", mp).Msg("unmounting leftover scratch disk")
648 if err := d.mnt.Unmount(ctx, mp); err != nil {
649 stuck[id] = true
650 errs = append(errs, err)
651 }
652 }
653 ents, _ := os.ReadDir(root)
654 for _, e := range ents {
655 if sandbox.ValidID(e.Name()) && !stuck[e.Name()] {
656 if err := os.RemoveAll(filepath.Join(root, e.Name())); err != nil {
657 errs = append(errs, err)
658 }
659 }
660 }
661 if len(errs) == 0 {
662 d.mu.Lock()
663 d.slots.ReleaseLeaked()
664 d.mu.Unlock()
665 }
666 return errors.Join(errs...)
667}