1// SPDX-License-Identifier: MIT
2
3package vm
4
5import (
6 "bytes"
7 "errors"
8 "os"
9 "path/filepath"
10 "slices"
11 "strconv"
12 "strings"
13 "sync"
14 "syscall"
15 "time"
16
17 "golang.org/x/sys/unix"
18
19 "bnema.dev/ermite-runner/internal/sandbox"
20)
21
22// Command-line markers of a VMM started for a VM. A process is one when it has
23// "--id <id>" (an id of the form sandbox.NewID produces) and at least one of
24// them. One set for the driver's scan and for the integration tests' leak check.
25const (
26 // MarkerAPISock: Firecracker started without the jailer.
27 MarkerAPISock = "--api-sock"
28 // MarkerExecFile: the jailer.
29 MarkerExecFile = "--exec-file"
30 // MarkerStartTime: the Firecracker the jailer execs into its chroot; it gets
31 // neither of the first two, only --id and the jailer's --start-time-* flags.
32 MarkerStartTime = "--start-time-us"
33)
34
35// VMMMarkers lists every marker.
36var VMMMarkers = []string{MarkerAPISock, MarkerExecFile, MarkerStartTime}
37
38// vmmArgs is what the scan reads from a command line.
39type vmmArgs struct {
40 id string
41 markers map[string]bool
42 chrootBase string // the jailer's --chroot-base-dir
43 apiSock string // Firecracker's --api-sock value
44}
45
46func parseVMMArgs(args []string) vmmArgs {
47 a := vmmArgs{markers: map[string]bool{}}
48 for i := 1; i < len(args); i++ {
49 switch {
50 case args[i] == "--id":
51 if i+1 < len(args) && sandbox.ValidID(args[i+1]) {
52 a.id = args[i+1]
53 }
54 case args[i] == "--chroot-base-dir":
55 if i+1 < len(args) {
56 a.chrootBase = args[i+1]
57 }
58 case slices.Contains(VMMMarkers, args[i]):
59 a.markers[args[i]] = true
60 if args[i] == MarkerAPISock && i+1 < len(args) {
61 a.apiSock = args[i+1]
62 }
63 }
64 }
65 return a
66}
67
68// IDFromCmdline returns the VM id of a command line that carries one of
69// VMMMarkers, or "". It looks at the arguments only; the driver's scan narrows
70// it to its own state directory (scope.match).
71func IDFromCmdline(args []string) string {
72 a := parseVMMArgs(args)
73 if len(a.markers) == 0 {
74 return ""
75 }
76 return a.id
77}
78
79// scope says which VMM processes are this driver's: the jailed ones must live in
80// (or have been started for) its own jailer chroot base, <state dir>/jail.
81type scope struct {
82 jailBases []string // the base as configured, and with symlinks resolved
83 vmBases []string // <state dir>/vm (unjailed VMs), likewise
84}
85
86func newScope(stateDir string) scope {
87 sc := scope{}
88 for _, b := range []struct {
89 dir string
90 to *[]string
91 }{{jailBase(stateDir), &sc.jailBases}, {filepath.Join(stateDir, "vm"), &sc.vmBases}} {
92 *b.to = []string{b.dir}
93 if r, err := filepath.EvalSymlinks(b.dir); err == nil && r != b.dir {
94 *b.to = append(*b.to, r)
95 }
96 }
97 return sc
98}
99
100// apiSockIsOurs: an unjailed Firecracker's --api-sock must be a path below
101// <state dir>/vm/<id>/ (compared lexically, after cleaning), the directory the
102// driver puts it in. Another Firecracker with an id of our shape and a socket
103// elsewhere is someone else's.
104func (sc scope) apiSockIsOurs(sock, id string) bool {
105 if sock == "" || !filepath.IsAbs(sock) {
106 return false
107 }
108 sock = filepath.Clean(sock)
109 for _, b := range sc.vmBases {
110 if strings.HasPrefix(sock, filepath.Join(b, id)+"/") {
111 return true
112 }
113 }
114 return false
115}
116
117func (sc scope) isBase(p string) bool {
118 return slices.Contains(sc.jailBases, filepath.Clean(p))
119}
120
121func (sc scope) under(p string) bool {
122 for _, b := range sc.jailBases {
123 if strings.HasPrefix(filepath.Clean(p), b+"/") {
124 return true
125 }
126 }
127 return false
128}
129
130// match returns the VM id of the process whose command line is args and whose
131// /proc directory is dir, or "". Besides IDFromCmdline:
132// - a jailer (--exec-file) must have been given our --chroot-base-dir;
133// - a Firecracker that carries only the jailer's --start-time-us (no
134// --api-sock) is the jailed VMM: its root must be the chroot of that id
135// below our jail base (rootIsOurs). Anything else with that flag and an id
136// of our shape belongs to someone else's jailer and is never touched.
137//
138// A Firecracker started with --api-sock (no jailer) is ours when the socket is
139// in <state dir>/vm/<id>/ (apiSockIsOurs).
140func (sc scope) match(dir string, args []string) string {
141 a := parseVMMArgs(args)
142 if a.id == "" || len(a.markers) == 0 {
143 return ""
144 }
145 switch {
146 case a.markers[MarkerAPISock]:
147 if sc.apiSockIsOurs(a.apiSock, a.id) {
148 return a.id
149 }
150 case a.markers[MarkerExecFile]:
151 if sc.isBase(a.chrootBase) {
152 return a.id
153 }
154 case a.markers[MarkerStartTime]:
155 if sc.rootIsOurs(dir, a.id) {
156 return a.id
157 }
158 }
159 return ""
160}
161
162// rootIsOurs reports whether the process's root directory is the chroot the
163// jailer made for id below our jail base. The jailer pivot_roots into a new
164// mount namespace, so readlink(/proc/<pid>/root) says "/" and cannot be used:
165// stat follows the magic link into the VM's root, which is the very directory
166// <base>/<exec name>/<id>/root of the host (same device and inode). A root
167// whose path is lexically below our base is accepted too.
168func (sc scope) rootIsOurs(dir, id string) bool {
169 if root, err := os.Readlink(filepath.Join(dir, "root")); err == nil && sc.under(root) {
170 return true
171 }
172 rootFI, err := os.Stat(filepath.Join(dir, "root"))
173 if err != nil {
174 return false
175 }
176 for _, b := range sc.jailBases {
177 execDirs, _ := os.ReadDir(b)
178 for _, e := range execDirs {
179 if fi, err := os.Stat(filepath.Join(b, e.Name(), id, "root")); err == nil && os.SameFile(fi, rootFI) {
180 return true
181 }
182 }
183 }
184 return false
185}
186
187// vmmProcs scans a /proc tree for processes started by this runner for a VM:
188// the Firecracker process and the jailer. A process is ours when its command
189// line has "--id <id>" with an id of the form sandbox.NewID produces and one of
190// the VMMMarkers, and it passes scope.match. The program name is not looked at:
191// the binary may be renamed or versioned (firecracker-v1.17), and a jailer in a
192// PID namespace leaves a Firecracker whose argv[0] is whatever the jailer
193// chose. Processes of other users of Firecracker (other ids, other jail bases)
194// are never matched. It finds orphans even when the state directory's records
195// were lost.
196func vmmProcs(procDir string, sc scope) map[int]string {
197 out := map[int]string{}
198 ents, err := os.ReadDir(procDir)
199 if err != nil {
200 return out
201 }
202 for _, e := range ents {
203 pid, err := strconv.Atoi(e.Name())
204 if err != nil || pid <= 1 {
205 continue
206 }
207 dir := filepath.Join(procDir, e.Name())
208 b, err := os.ReadFile(filepath.Join(dir, "cmdline"))
209 if err != nil || len(b) == 0 {
210 continue
211 }
212 if id := sc.match(dir, strings.Split(strings.TrimRight(string(b), "\x00"), "\x00")); id != "" {
213 out[pid] = id
214 }
215 }
216 return out
217}
218
219// pidAlive reports whether pid exists and is not a zombie.
220func pidAlive(pid int) bool {
221 if err := syscall.Kill(pid, 0); err != nil {
222 return false
223 }
224 if b, err := os.ReadFile("/proc/" + strconv.Itoa(pid) + "/stat"); err == nil {
225 if i := bytes.LastIndexByte(b, ')'); i >= 0 && i+2 < len(b) && b[i+2] == 'Z' {
226 return false
227 }
228 }
229 return true
230}
231
232// ring keeps the last N bytes written to it: the VMM's stdout/stderr, which
233// carries the guest serial console, for diagnostics when a boot fails.
234type ring struct {
235 mu sync.Mutex
236 max int
237 buf []byte
238}
239
240func newRing(max int) *ring { return &ring{max: max} }
241
242func (r *ring) Write(p []byte) (int, error) {
243 r.mu.Lock()
244 defer r.mu.Unlock()
245 r.buf = append(r.buf, p...)
246 if len(r.buf) > r.max {
247 r.buf = append([]byte(nil), r.buf[len(r.buf)-r.max:]...)
248 }
249 return len(p), nil
250}
251
252// String returns the retained tail, trimmed.
253func (r *ring) String() string {
254 r.mu.Lock()
255 defer r.mu.Unlock()
256 return string(bytes.TrimSpace(r.buf))
257}
258
259// killAndWait kills the process that was identified as pid, through a pidfd, and
260// waits for it to exit. It reports whether that process is gone.
261//
262// A pid read from /proc a moment ago may belong to another process by now (the
263// VMM exited and the number was recycled). The pidfd pins the process; then
264// still() re-runs the identity check (command line and root scope) on the pid,
265// and only if the pidfd is still not readable (the process had not exited at
266// that point, so the pid could not have been recycled while still() read it) the
267// signal goes out through the pidfd, which can only reach that process. A
268// mismatch means the process we meant is gone: nothing is signalled and true is
269// returned. Where pidfds are not available the process is not touched and false
270// is returned (the sandbox is then reported as leaked, never killed blindly).
271func killAndWait(pid int, still func() bool, timeout time.Duration) bool {
272 fd, err := unix.PidfdOpen(pid, 0)
273 if err != nil {
274 return errors.Is(err, unix.ESRCH) // no such process: gone
275 }
276 defer func() { _ = unix.Close(fd) }()
277 ours := still()
278 if exited, _ := pidfdWait(fd, 0); exited {
279 return true // it exited before or while we checked: nothing to kill
280 }
281 if !ours {
282 return true // alive, but not the process we meant
283 }
284 if err := unix.PidfdSendSignal(fd, unix.SIGKILL, nil, 0); err != nil && !errors.Is(err, unix.ESRCH) {
285 return false
286 }
287 exited, _ := pidfdWait(fd, timeout)
288 return exited
289}
290
291// pidfdWait waits up to timeout for the process behind fd to exit (a pidfd is
292// readable once it has, zombie included).
293func pidfdWait(fd int, timeout time.Duration) (bool, error) {
294 deadline := time.Now().Add(timeout)
295 for {
296 left := time.Until(deadline)
297 if left < 0 {
298 left = 0
299 }
300 fds := []unix.PollFd{{Fd: int32(fd), Events: unix.POLLIN}}
301 n, err := unix.Poll(fds, int(left.Milliseconds()))
302 switch {
303 case errors.Is(err, unix.EINTR):
304 continue
305 case err != nil:
306 return false, err
307 case n > 0:
308 return true, nil
309 }
310 return false, nil
311 }
312}