1// SPDX-License-Identifier: MIT
2
3package guest
4
5import (
6 "context"
7 "errors"
8 "fmt"
9 "io"
10 "net"
11 "net/http"
12 "os"
13 "os/signal"
14 "strings"
15 "syscall"
16 "time"
17
18 "github.com/bnema/zerowrap"
19 "golang.org/x/sys/unix"
20
21 "bnema.dev/ermite-runner/internal/sandbox"
22)
23
24// InitConfig configures Init. The zero value reads the real system; tests
25// replace the fields.
26type InitConfig struct {
27 Version string
28 // Cmdline is the kernel command line; empty reads /proc/cmdline.
29 Cmdline string
30 // Getpid reports the process id; Init only acts as a VM's PID 1 when it is 1.
31 Getpid func() int
32 // Signals delivers the termination signals to forward to the executor;
33 // nil means "subscribe to SIGTERM and SIGINT". Tests inject their own.
34 Signals <-chan os.Signal
35 // SkipDocker does not start dockerd (tests, images without Docker).
36 SkipDocker bool
37 // ExecArgs is the executor command line; default
38 // [GuestExecDir/GuestExecName exec].
39 ExecArgs []string
40 // DockerdArgs replaces the dockerd command line, and the check that
41 // /usr/bin/dockerd exists (tests); SkipDocker still wins.
42 DockerdArgs []string
43 // DockerSocket is the Docker API socket dockerd listens on and init waits
44 // for; default dockerSocket.
45 DockerSocket string
46 // DockerReadyTimeout bounds the wait for the Docker API before the executor
47 // is started anyway; default dockerReadyTimeout.
48 DockerReadyTimeout time.Duration
49}
50
51// Init is `ermite-runner init`: the entrypoint of every sandbox.
52//
53// In a micro-VM it is the kernel's init (PID 1, root = the read-only exec disk,
54// /dev/vdb = shared base image, /dev/vdc = per-job scratch disk): it mounts the
55// pseudo filesystems, sets the clock and the DNS server, builds an overlayfs
56// root (lower = base, upper = scratch) and pivots into it, then does what a
57// container does.
58//
59// In a container the root and the pseudo filesystems are ready: it skips all of
60// that and only prepares cgroups. Then, in both: start dockerd with its data
61// root on the scratch space, wait (bounded) for its API to answer, run the executor
62// (`ermite-runner exec`) as a child, reap orphans, stop dockerd, and finish by
63// powering the VM off, or exiting with the executor's code in a container.
64func Init(ctx context.Context, cfg InitConfig) int {
65 log := zerowrap.FromCtx(ctx)
66 // The kernel starts a VM's init with umask 0, which dockerd, the executor
67 // and every step would inherit: files and t.TempDir() directories would be
68 // world-writable. GitHub's runners have 022, and so do the jobs here.
69 syscall.Umask(StepUmask)
70 if cfg.Getpid == nil {
71 cfg.Getpid = os.Getpid
72 }
73 if cfg.ExecArgs == nil {
74 cfg.ExecArgs = []string{sandbox.GuestExecDir + "/" + sandbox.GuestExecName, "exec"}
75 }
76 cmdline := cfg.Cmdline
77 if cmdline == "" {
78 b, err := os.ReadFile("/proc/cmdline")
79 if err != nil && cfg.Getpid() == 1 {
80 // A VM's init starts with nothing mounted: /proc is needed to read
81 // the kernel command line that says what we are. setupVM reuses it.
82 if merr := unix.Mount("proc", "/proc", "proc", 0, ""); merr == nil {
83 b, err = os.ReadFile("/proc/cmdline")
84 }
85 }
86 if err == nil {
87 cmdline = string(b)
88 }
89 }
90 args := ParseCmdline(cmdline)
91 inVM := args[sandbox.CmdlineSandbox] == string(sandbox.VM) && cfg.Getpid() == 1
92
93 mode := string(sandbox.Container)
94 if inVM {
95 mode = string(sandbox.VM)
96 if err := setupVM(log, args); err != nil {
97 log.Error().Err(err).Msg("init: vm setup failed")
98 powerOff()
99 return 1
100 }
101 }
102 code := runPayload(ctx, cfg, mode, args)
103 if inVM {
104 powerOff()
105 }
106 return code
107}
108
109// StepUmask is the umask of everything the init starts, steps included: the
110// one of GitHub's hosted runners.
111const StepUmask = 0o022
112
113// defaultPath is the PATH of a sandbox whose init was started by the kernel
114// with an empty environment.
115const defaultPath = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
116
117func runPayload(ctx context.Context, cfg InitConfig, mode string, args map[string]string) int {
118 log := zerowrap.FromCtx(ctx)
119 if os.Getenv("PATH") == "" {
120 _ = os.Setenv("PATH", defaultPath)
121 }
122 if os.Getenv("HOME") == "" {
123 _ = os.Setenv("HOME", "/root")
124 }
125 if mode == string(sandbox.Container) {
126 if err := nestCgroups(); err != nil {
127 log.Warn().Err(err).Msg("init: cgroup setup (docker inside may not start)")
128 }
129 }
130 target := args[sandbox.CmdlineConnect]
131 if target == "" {
132 target = os.Getenv("ERMITE_CONNECT")
133 }
134 if _, err := ParseTarget(target); err != nil {
135 log.Error().Err(err).Msg("init: no executor target")
136 return 1
137 }
138
139 var dockerPid, containerdPid int
140 _, statErr := os.Stat("/usr/bin/dockerd")
141 if !cfg.SkipDocker && (statErr == nil || cfg.DockerdArgs != nil) {
142 _ = os.MkdirAll(sandbox.GuestScratch+"/docker", 0o711)
143 argv := cfg.DockerdArgs
144 if argv == nil {
145 argv = []string{"/usr/bin/dockerd", "--data-root", sandbox.GuestScratch + "/docker",
146 "--storage-driver", "overlay2", "--host", "unix:///var/run/docker.sock"}
147 // dockerd sets its umask to 0, and a containerd it starts itself
148 // inherits it, then every `docker exec` (the steps of a job
149 // container): their files would be world-writable. Start containerd
150 // here with our umask instead, as systemd does on GitHub's runners.
151 if pid, ok := startContainerd(ctx, log); ok {
152 containerdPid = pid
153 argv = append(argv, "--containerd", containerdSocket)
154 }
155 }
156 pid, err := spawn(argv, nil)
157 if err != nil {
158 log.Error().Err(err).Msg("init: start dockerd")
159 } else {
160 dockerPid = pid
161 }
162 }
163
164 // SIGTERM/SIGINT (docker stop, or the host stopping the VM) go to the
165 // executor, which cancels its script with the usual escalation and reports
166 // its Exit; init then finishes normally instead of dying with children.
167 sigs := cfg.Signals
168 if sigs == nil {
169 ch := make(chan os.Signal, 4)
170 signal.Notify(ch, syscall.SIGTERM, syscall.SIGINT)
171 defer signal.Stop(ch)
172 sigs = ch
173 }
174
175 // The first job-container step needs the Docker API, so the executor must
176 // not start before dockerd answers. If it never does (or dies), start the
177 // executor anyway: jobs that do not use Docker still run. A shutdown signal
178 // during the wait aborts the boot instead: see abortBootError.
179 if dockerPid != 0 {
180 sock := cfg.DockerSocket
181 if sock == "" {
182 sock = dockerSocket
183 }
184 timeout := cfg.DockerReadyTimeout
185 if timeout <= 0 {
186 timeout = dockerReadyTimeout
187 }
188 err := waitDockerReady(ctx, log, sock, dockerPid, timeout, sigs)
189 var exited *dockerdExitError
190 var abort *abortBootError
191 switch {
192 case errors.As(err, &abort):
193 stopDockerd(dockerPid)
194 stopDockerd(containerdPid)
195 return abort.code
196 case errors.As(err, &exited):
197 dockerPid = 0 // already reaped
198 }
199 }
200
201 execEnv := append(os.Environ(), "ERMITE_CONNECT="+target, "ERMITE_SANDBOX="+mode, "ERMITE_VERSION="+cfg.Version)
202 execPid, err := spawn(cfg.ExecArgs, execEnv)
203 if err != nil {
204 log.Error().Err(err).Msg("init: start executor")
205 stopDockerd(dockerPid)
206 stopDockerd(containerdPid)
207 return 1
208 }
209 forward := func(sig os.Signal) {
210 if s, ok := sig.(syscall.Signal); ok {
211 log.Info().Str("signal", s.String()).Msg("init: forwarding the signal to the executor")
212 _ = syscall.Kill(-execPid, s) // the executor leads its own process group
213 _ = syscall.Kill(execPid, s)
214 }
215 }
216 stopFwd := make(chan struct{})
217 defer close(stopFwd)
218 go func() {
219 for {
220 select {
221 case sig := <-sigs:
222 forward(sig)
223 case <-stopFwd:
224 return
225 }
226 }
227 }()
228 code := reapUntil(execPid)
229 stopDockerd(dockerPid)
230 stopDockerd(containerdPid) // after dockerd, which needs it to stop its containers
231 return code
232}
233
234// startContainerd starts /usr/bin/containerd (when the image has it) and waits,
235// bounded, for its socket. false: no containerd, or it did not come up; dockerd
236// then starts its own.
237func startContainerd(ctx context.Context, log zerowrap.Logger) (int, bool) {
238 if _, err := os.Stat(containerdBinary); err != nil {
239 return 0, false
240 }
241 // Its data next to dockerd's, on the scratch space: in a container the
242 // root is an overlay, which the overlayfs snapshotter cannot sit on.
243 root := sandbox.GuestScratch + "/containerd"
244 _ = os.MkdirAll(root, 0o711)
245 pid, err := spawn([]string{containerdBinary, "--address", containerdSocket, "--root", root}, nil)
246 if err != nil {
247 log.Warn().Err(err).Msg("init: start containerd (dockerd starts its own)")
248 return 0, false
249 }
250 deadline := time.Now().Add(containerdReadyTimeout)
251 for time.Now().Before(deadline) && ctx.Err() == nil {
252 if fi, err := os.Stat(containerdSocket); err == nil && fi.Mode()&os.ModeSocket != 0 {
253 return pid, true
254 }
255 if got, _ := wait4NoHang(pid); got == pid {
256 log.Warn().Msg("init: containerd exited at start (dockerd starts its own)")
257 return 0, false
258 }
259 time.Sleep(dockerPollInterval)
260 }
261 log.Warn().Msg("init: containerd socket not ready (dockerd starts its own)")
262 stopDockerd(pid)
263 return 0, false
264}
265
266// stopDockerd stops dockerd or containerd (SIGTERM, then SIGKILL after 5 s) and reaps it (at
267// most 5 s more after the SIGKILL); a zero pid is a no-op.
268func stopDockerd(pid int) {
269 if pid == 0 {
270 return
271 }
272 _ = syscall.Kill(pid, syscall.SIGTERM)
273 deadline := time.Now().Add(5 * time.Second)
274 for time.Now().Before(deadline) {
275 if got, _ := wait4NoHang(pid); got == pid {
276 return
277 }
278 time.Sleep(100 * time.Millisecond)
279 }
280 _ = syscall.Kill(pid, syscall.SIGKILL)
281 // SIGKILL cannot be ignored, so the exit is prompt; poll rather than block
282 // for ever on a process stuck in the kernel (uninterruptible sleep).
283 for end := time.Now().Add(5 * time.Second); time.Now().Before(end); time.Sleep(10 * time.Millisecond) {
284 got, err := wait4NoHang(pid)
285 if got == pid || (err != nil && err != syscall.EINTR) { // ECHILD: already reaped
286 return
287 }
288 }
289}
290
291const (
292 // containerdBinary and containerdSocket are the containerd init starts for
293 // dockerd; containerdReadyTimeout bounds the wait for its socket.
294 containerdBinary = "/usr/bin/containerd"
295 containerdSocket = "/run/containerd/containerd.sock"
296 containerdReadyTimeout = 20 * time.Second
297 // dockerSocket is the Docker API socket of the nested dockerd.
298 dockerSocket = "/var/run/docker.sock"
299 // dockerReadyTimeout bounds the wait for the Docker API at boot.
300 dockerReadyTimeout = 60 * time.Second
301 dockerPollInterval = 100 * time.Millisecond
302 dockerPingTimeout = 2 * time.Second
303)
304
305// dockerdExitError reports that dockerd exited (and was reaped) before its API
306// answered.
307type dockerdExitError struct{ status syscall.WaitStatus }
308
309func (e *dockerdExitError) Error() string {
310 if e.status.Signaled() {
311 return "dockerd killed by signal " + e.status.Signal().String()
312 }
313 return fmt.Sprintf("dockerd exited with code %d", e.status.ExitStatus())
314}
315
316// abortBootError means a shutdown signal (or the cancelled context) arrived
317// while waiting for the Docker API: init must not start the executor, stop
318// dockerd and exit with code (128+signal, as for a killed executor).
319//
320// Not starting the executor is safe for the host: the sandbox ends before the
321// executor connected ("container exited ... before the executor connected" /
322// "firecracker exited before the executor connected"), and the host reports the
323// job cancelled (not failed) when a cancel or shutdown was requested
324// (jobRun.execute: startSandbox error with cancelRequested). A stop that the
325// host did not ask for is reported as "the sandbox did not start".
326type abortBootError struct{ code int }
327
328func (e *abortBootError) Error() string {
329 return fmt.Sprintf("boot aborted by a shutdown signal (exit %d)", e.code)
330}
331
332// reapChildren reaps every child that has exited (orphans adopted by init
333// included) without blocking, and reports whether pid was among them, with its
334// wait status.
335func reapChildren(pid int) (bool, syscall.WaitStatus) {
336 var found bool
337 var status syscall.WaitStatus
338 for {
339 var ws syscall.WaitStatus
340 got, err := syscall.Wait4(-1, &ws, syscall.WNOHANG, nil)
341 if err == syscall.EINTR {
342 continue
343 }
344 if err != nil || got <= 0 {
345 return found, status
346 }
347 if got == pid {
348 found, status = true, ws
349 }
350 }
351}
352
353// waitDockerReady blocks until the Docker API on sock answers GET /_ping with
354// "OK", for at most timeout. It returns nil when ready, and otherwise:
355// - *dockerdExitError: dockerd (pid, a child of init) exited and was reaped;
356// - a plain error: the deadline passed;
357// - *abortBootError: a signal arrived on sigs or ctx was cancelled.
358//
359// The orphans that init adopts are reaped on every tick, and a ping in flight
360// never delays the deadline, the signals or the exit check. Every outcome is
361// logged; for the first two the caller starts the executor anyway.
362func waitDockerReady(ctx context.Context, log zerowrap.Logger, sock string, pid int, timeout time.Duration, sigs <-chan os.Signal) error {
363 start := time.Now()
364 log.Info().Str("socket", sock).Dur("timeout", timeout).Msg("init: waiting for the Docker API")
365 wctx, cancel := context.WithTimeout(ctx, timeout)
366 defer cancel() // also aborts a ping in flight
367 client := &http.Client{Transport: &http.Transport{
368 DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
369 var d net.Dialer
370 return d.DialContext(ctx, "unix", sock)
371 },
372 DisableKeepAlives: true,
373 }}
374 defer client.CloseIdleConnections()
375 ticker := time.NewTicker(dockerPollInterval)
376 defer ticker.Stop()
377
378 var pinging <-chan bool // result of the ping in flight, nil if none
379 startPing := func() {
380 ch := make(chan bool, 1)
381 pinging = ch
382 go func() { ch <- dockerPing(wctx, client) }()
383 }
384 checkExit := func() error {
385 if ok, ws := reapChildren(pid); ok {
386 err := &dockerdExitError{status: ws}
387 log.Warn().Err(err).Msg("init: dockerd died before its API was ready; starting the executor anyway (jobs using Docker will fail)")
388 return err
389 }
390 return nil
391 }
392 if err := checkExit(); err != nil {
393 return err
394 }
395 startPing()
396 for {
397 select {
398 case ok := <-pinging:
399 pinging = nil
400 if ok {
401 log.Info().Dur("after", time.Since(start).Round(time.Millisecond)).Msg("init: the Docker API is ready")
402 return nil
403 }
404 case <-ticker.C:
405 if err := checkExit(); err != nil {
406 return err
407 }
408 if pinging == nil {
409 startPing()
410 }
411 case sig := <-sigs:
412 return abortBoot(log, sig)
413 case <-wctx.Done():
414 if ctx.Err() != nil {
415 return abortBoot(log, syscall.SIGTERM)
416 }
417 log.Warn().Dur("timeout", timeout).Msg("init: the Docker API did not answer in time; starting the executor anyway (jobs using Docker will fail)")
418 return fmt.Errorf("docker API not ready after %s", timeout)
419 }
420 }
421}
422
423func abortBoot(log zerowrap.Logger, sig os.Signal) error {
424 code := 128 + int(syscall.SIGTERM)
425 if s, ok := sig.(syscall.Signal); ok {
426 code = 128 + int(s)
427 }
428 log.Warn().Str("signal", fmt.Sprint(sig)).Msg("init: shutdown signal while waiting for the Docker API; not starting the executor, stopping dockerd")
429 return &abortBootError{code: code}
430}
431
432// dockerPing reports whether GET /_ping answers 200 "OK".
433func dockerPing(ctx context.Context, client *http.Client) bool {
434 // One ping is bounded on its own so that a daemon that accepts and stalls
435 // does not hold the others back; ctx (the readiness deadline) bounds it too.
436 ctx, cancel := context.WithTimeout(ctx, dockerPingTimeout)
437 defer cancel()
438 req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://docker/_ping", nil)
439 if err != nil {
440 return false
441 }
442 resp, err := client.Do(req)
443 if err != nil {
444 return false
445 }
446 defer func() { _ = resp.Body.Close() }()
447 body, _ := io.ReadAll(io.LimitReader(resp.Body, 16))
448 return resp.StatusCode == http.StatusOK && strings.TrimSpace(string(body)) == "OK"
449}
450
451// spawn starts argv[0] as a child (own process group) without os/exec, so that
452// reapUntil is the only waiter and orphans can be reaped safely.
453func spawn(argv, env []string) (int, error) {
454 if env == nil {
455 env = os.Environ()
456 }
457 return syscall.ForkExec(argv[0], argv, &syscall.ProcAttr{
458 Env: env,
459 Files: []uintptr{0, 1, 2},
460 Sys: &syscall.SysProcAttr{Setpgid: true},
461 })
462}
463
464// reapUntil waits for every child (orphans included) and returns when pid
465// exits, with its exit code (128+signal when killed).
466func reapUntil(pid int) int {
467 for {
468 var ws syscall.WaitStatus
469 got, err := syscall.Wait4(-1, &ws, 0, nil)
470 if err == syscall.EINTR {
471 continue
472 }
473 if err != nil { // ECHILD: no child left, the executor included
474 return 1
475 }
476 if got == pid {
477 switch {
478 case ws.Exited():
479 return ws.ExitStatus()
480 case ws.Signaled():
481 return 128 + int(ws.Signal())
482 }
483 return 1
484 }
485 }
486}
487
488func wait4NoHang(pid int) (int, error) {
489 var ws syscall.WaitStatus
490 return syscall.Wait4(pid, &ws, syscall.WNOHANG, nil)
491}
492
493func powerOff() {
494 unix.Sync()
495 // On x86 Firecracker has no ACPI power off: a reboot makes the VMM exit
496 // (kernel cmdline reboot=k). On arm64 PSCI handles power off.
497 _ = unix.Reboot(unix.LINUX_REBOOT_CMD_RESTART)
498 _ = unix.Reboot(unix.LINUX_REBOOT_CMD_POWER_OFF)
499}
500
501// nestCgroups prepares cgroup v2 for a nested dockerd in a container: a cgroup
502// with member processes cannot delegate controllers to children, so move every
503// process to an "init" child and enable the controllers (like the upstream
504// docker-in-docker entrypoint). No-op without cgroup v2.
505func nestCgroups() error {
506 const root = "/sys/fs/cgroup"
507 ctl, err := os.ReadFile(root + "/cgroup.controllers")
508 if err != nil {
509 return nil
510 }
511 if err := os.MkdirAll(root+"/init", 0o755); err != nil {
512 return err
513 }
514 procs, err := os.ReadFile(root + "/cgroup.procs")
515 if err != nil {
516 return err
517 }
518 for _, p := range strings.Fields(string(procs)) {
519 // Processes may exit meanwhile: ignore per-process errors.
520 _ = os.WriteFile(root+"/init/cgroup.procs", []byte(p), 0o644)
521 }
522 var enable []string
523 for _, c := range strings.Fields(string(ctl)) {
524 enable = append(enable, "+"+c)
525 }
526 if len(enable) == 0 {
527 return nil
528 }
529 if err := os.WriteFile(root+"/cgroup.subtree_control", []byte(strings.Join(enable, " ")), 0o644); err != nil {
530 return fmt.Errorf("enable controllers: %w", err)
531 }
532 return nil
533}