1// SPDX-License-Identifier: MIT
2
3package vm
4
5import (
6 "context"
7 "errors"
8 "fmt"
9 "os"
10 "path/filepath"
11 "strconv"
12 "strings"
13 "time"
14
15 "github.com/bnema/zerowrap"
16
17 "bnema.dev/ermite-runner/internal/netpolicy"
18 "bnema.dev/ermite-runner/internal/sandbox"
19)
20
21// Reap implements sandbox.Driver. It removes what a previous run of the runner
22// left: VMM processes (Firecracker and jailer: matched by "--id <our id>" in
23// their command line, never by program name), TAP devices named ertap<N>, the
24// nftables table, VM directories and jailer cgroups. It also finishes the
25// cleanup of sandboxes that reported ErrLeaked and frees their slots. It relies
26// on the host-wide lock taken in New: no other runner can own what it removes.
27// It returns ErrBusy while a sandbox is running.
28func (d *Driver) Reap(ctx context.Context) error {
29 log := zerowrap.FromCtx(ctx)
30 d.mu.Lock()
31 busy := d.slots.Busy()
32 d.mu.Unlock()
33 if busy > 0 {
34 return fmt.Errorf("%w (%d running)", sandbox.ErrBusy, busy)
35 }
36 var errs []error
37
38 recs, err := d.store.list()
39 if err != nil {
40 errs = append(errs, err)
41 }
42 procs := vmmProcs(d.opt.ProcDir, d.scope())
43 killed := map[int]bool{}
44 killProc := func(pid int, id string) {
45 if killed[pid] {
46 return
47 }
48 killed[pid] = true
49 log.Warn().Int("pid", pid).Str("vm", id).Msg("reaping leftover VMM process")
50 if !d.opt.kill(pid, func() bool { return cmdlineID(d.opt.ProcDir, pid, d.scope()) == id }, 5*time.Second) {
51 errs = append(errs, fmt.Errorf("vmm process %d (%s) did not exit", pid, id))
52 }
53 }
54 // 1. Processes. A recorded pid is only trusted when /proc/<pid>/cmdline still
55 // carries this record's id (pids are reused); then every other process of
56 // ours found in /proc, which also covers a lost state directory.
57 for _, r := range recs {
58 if r.PID > 0 && cmdlineID(d.opt.ProcDir, r.PID, d.scope()) == r.ID {
59 killProc(r.PID, r.ID)
60 }
61 }
62 for pid, id := range procs {
63 killProc(pid, id)
64 }
65
66 // 2. Network: the nft table first (it names interfaces), then every TAP.
67 if err := d.net.Reset(ctx); err != nil {
68 errs = append(errs, fmt.Errorf("reset nftables table %s: %w", netpolicy.Table, err))
69 }
70 taps, err := leftoverTaps(ctx, d.opt.Run)
71 if err != nil {
72 errs = append(errs, fmt.Errorf("list interfaces: %w", err))
73 }
74 for _, t := range taps {
75 log.Warn().Str("tap", t).Msg("removing leftover TAP device")
76 if err := deleteTap(ctx, d.opt.Run, t); err != nil {
77 errs = append(errs, err)
78 }
79 }
80
81 // 3. Files and cgroups. The jailer's chroot base has one directory per
82 // exec-file name.
83 parents := []string{filepath.Join(d.cfg.StateDir, "vm")}
84 jails, _ := os.ReadDir(filepath.Join(d.cfg.StateDir, "jail"))
85 for _, j := range jails {
86 if j.IsDir() {
87 parents = append(parents, filepath.Join(d.cfg.StateDir, "jail", j.Name()))
88 }
89 }
90 for _, parent := range parents {
91 ents, _ := os.ReadDir(parent)
92 for _, e := range ents {
93 if sandbox.ValidID(e.Name()) {
94 log.Warn().Str("dir", filepath.Join(parent, e.Name())).Msg("removing leftover VM directory")
95 if err := removeAll(filepath.Join(parent, e.Name())); err != nil {
96 errs = append(errs, err)
97 }
98 }
99 }
100 }
101 cg := filepath.Join(d.opt.CgroupRoot, filepath.Base(d.fcPath))
102 cents, _ := os.ReadDir(cg)
103 for _, e := range cents {
104 if e.IsDir() && sandbox.ValidID(e.Name()) {
105 if err := removeCgroup(d.opt.CgroupRoot, filepath.Base(d.fcPath), e.Name()); err != nil {
106 errs = append(errs, err)
107 }
108 }
109 }
110 if len(errs) == 0 {
111 for _, r := range recs {
112 if err := d.store.remove(r.ID); err != nil {
113 errs = append(errs, err)
114 }
115 }
116 }
117 if len(errs) == 0 {
118 d.mu.Lock()
119 d.slots.ReleaseLeaked()
120 d.mu.Unlock()
121 }
122 return errors.Join(errs...)
123}
124
125// cmdlineID returns the VM id in the command line of pid, or "" (not running,
126// not ours, or recycled for another program).
127func cmdlineID(procDir string, pid int, sc scope) string {
128 dir := filepath.Join(procDir, strconv.Itoa(pid))
129 b, err := os.ReadFile(filepath.Join(dir, "cmdline"))
130 if err != nil || len(b) == 0 {
131 return ""
132 }
133 return sc.match(dir, strings.Split(strings.TrimRight(string(b), "\x00"), "\x00"))
134}