1// SPDX-License-Identifier: MIT
2
3package guest
4
5import (
6 "context"
7 "errors"
8 "fmt"
9 "io"
10 "net"
11 "net/http"
12 "os"
13 "os/signal"
14 "strconv"
15 "strings"
16 "syscall"
17 "time"
18
19 "github.com/bnema/zerowrap"
20 "golang.org/x/sys/unix"
21
22 "bnema.dev/ermite-runner/internal/sandbox"
23)
24
25// InitConfig configures Init. The zero value reads the real system; tests
26// replace the fields.
27type InitConfig struct {
28 Version string
29 // Cmdline is the kernel command line; empty reads /proc/cmdline.
30 Cmdline string
31 // Getpid reports the process id; Init only acts as a VM's PID 1 when it is 1.
32 Getpid func() int
33 // Signals delivers the termination signals to forward to the executor;
34 // nil means "subscribe to SIGTERM and SIGINT". Tests inject their own.
35 Signals <-chan os.Signal
36 // SkipDocker does not start dockerd (tests, images without Docker).
37 SkipDocker bool
38 // ExecArgs is the executor command line; default
39 // [GuestExecDir/GuestExecName exec].
40 ExecArgs []string
41 // DockerdArgs replaces the dockerd command line, and the check that
42 // /usr/bin/dockerd exists (tests); SkipDocker still wins.
43 DockerdArgs []string
44 // DockerSocket is the Docker API socket dockerd listens on and init waits
45 // for; default dockerSocket.
46 DockerSocket string
47 // DockerReadyTimeout bounds the wait for the Docker API before the executor
48 // is started anyway; default dockerReadyTimeout.
49 DockerReadyTimeout time.Duration
50}
51
52// Init is `ermite-runner init`: the entrypoint of every sandbox.
53//
54// In a micro-VM it is the kernel's init (PID 1, root = the read-only exec disk,
55// /dev/vdb = shared base image, /dev/vdc = per-job scratch disk): it mounts the
56// pseudo filesystems, sets the clock and the DNS server, builds an overlayfs
57// root (lower = base, upper = scratch) and pivots into it, then does what a
58// container does.
59//
60// In a container the root and the pseudo filesystems are ready: it skips all of
61// that and only prepares cgroups. Then, in both: start dockerd with its data
62// root on the scratch space, wait (bounded) for its API to answer, run the executor
63// (`ermite-runner exec`) as a child, reap orphans, stop dockerd, and finish by
64// powering the VM off, or exiting with the executor's code in a container.
65func Init(ctx context.Context, cfg InitConfig) int {
66 entry := time.Now()
67 // bootMs is the boot phase timing handed to the executor (ERMITE_BOOT_MS),
68 // which reports it in its Hello.
69 bootMs := make(map[string]int64)
70 log := zerowrap.FromCtx(ctx)
71 // The kernel starts a VM's init with umask 0, which dockerd, the executor
72 // and every step would inherit: files and t.TempDir() directories would be
73 // world-writable. GitHub's runners have 022, and so do the jobs here.
74 syscall.Umask(StepUmask)
75 if cfg.Getpid == nil {
76 cfg.Getpid = os.Getpid
77 }
78 if cfg.ExecArgs == nil {
79 cfg.ExecArgs = []string{sandbox.GuestExecDir + "/" + sandbox.GuestExecName, "exec"}
80 }
81 cmdline := cfg.Cmdline
82 if cmdline == "" {
83 b, err := os.ReadFile("/proc/cmdline")
84 if err != nil && cfg.Getpid() == 1 {
85 // A VM's init starts with nothing mounted: /proc is needed to read
86 // the kernel command line that says what we are. setupVM reuses it.
87 if merr := unix.Mount("proc", "/proc", "proc", 0, ""); merr == nil {
88 b, err = os.ReadFile("/proc/cmdline")
89 }
90 }
91 if err == nil {
92 cmdline = string(b)
93 }
94 }
95 args := ParseCmdline(cmdline)
96 inVM := args[sandbox.CmdlineSandbox] == string(sandbox.VM) && cfg.Getpid() == 1
97
98 mode := string(sandbox.Container)
99 if inVM {
100 mode = string(sandbox.VM)
101 if b, err := os.ReadFile("/proc/uptime"); err == nil {
102 if fields := strings.Fields(string(b)); len(fields) > 0 {
103 if seconds, err := strconv.ParseFloat(fields[0], 64); err == nil && seconds >= 0 {
104 bootMs["kernel"] = max(0, int64(seconds*1000)-time.Since(entry).Milliseconds())
105 }
106 }
107 }
108 start := time.Now()
109 if err := setupVM(log, args); err != nil {
110 log.Error().Err(err).Msg("init: vm setup failed")
111 powerOff()
112 return 1
113 }
114 bootMs["setup"] = time.Since(start).Milliseconds()
115 }
116 code := runPayload(ctx, cfg, mode, args, bootMs)
117 if inVM {
118 powerOff()
119 }
120 return code
121}
122
123// StepUmask is the umask of everything the init starts, steps included: the
124// one of GitHub's hosted runners.
125const StepUmask = 0o022
126
127// defaultPath is the PATH of a sandbox whose init was started by the kernel
128// with an empty environment.
129const defaultPath = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
130
131// runPayload starts dockerd and the executor and waits for them. It adds its
132// boot phases to bootMs, which the executor gets.
133func runPayload(ctx context.Context, cfg InitConfig, mode string, args map[string]string, bootMs map[string]int64) int {
134 log := zerowrap.FromCtx(ctx)
135 if os.Getenv("PATH") == "" {
136 _ = os.Setenv("PATH", defaultPath)
137 }
138 if os.Getenv("HOME") == "" {
139 _ = os.Setenv("HOME", "/root")
140 }
141 if mode == string(sandbox.Container) {
142 if err := nestCgroups(); err != nil {
143 log.Warn().Err(err).Msg("init: cgroup setup (docker inside may not start)")
144 }
145 }
146 target := args[sandbox.CmdlineConnect]
147 if target == "" {
148 target = os.Getenv("ERMITE_CONNECT")
149 }
150 if _, err := ParseTarget(target); err != nil {
151 log.Error().Err(err).Msg("init: no executor target")
152 return 1
153 }
154
155 var dockerStart time.Time
156 var dockerPid, containerdPid int
157 _, statErr := os.Stat("/usr/bin/dockerd")
158 if !cfg.SkipDocker && (statErr == nil || cfg.DockerdArgs != nil) {
159 _ = os.MkdirAll(sandbox.GuestScratch+"/docker", 0o711)
160 argv := cfg.DockerdArgs
161 if argv == nil {
162 argv = defaultDockerdArgs()
163 // dockerd sets its umask to 0, and a containerd it starts itself
164 // inherits it, then every `docker exec` (the steps of a job
165 // container): their files would be world-writable. Start containerd
166 // here with our umask instead, as systemd does on GitHub's runners.
167 start := time.Now()
168 pid, ok := startContainerd(ctx, log)
169 // A containerd that never came up still cost its wait: keep it in the
170 // breakdown. A missing binary returns at once (0 ms) and is omitted.
171 if ms := time.Since(start).Milliseconds(); ok || ms > 0 {
172 bootMs["containerd"] = ms
173 }
174 if ok {
175 containerdPid = pid
176 argv = append(argv, "--containerd", containerdSocket)
177 }
178 }
179 dockerStart = time.Now()
180 pid, err := spawn(argv, nil)
181 if err != nil {
182 log.Error().Err(err).Msg("init: start dockerd")
183 } else {
184 dockerPid = pid
185 }
186 }
187
188 // SIGTERM/SIGINT (docker stop, or the host stopping the VM) go to the
189 // executor, which cancels its script with the usual escalation and reports
190 // its Exit; init then finishes normally instead of dying with children.
191 sigs := cfg.Signals
192 if sigs == nil {
193 ch := make(chan os.Signal, 4)
194 signal.Notify(ch, syscall.SIGTERM, syscall.SIGINT)
195 defer signal.Stop(ch)
196 sigs = ch
197 }
198
199 // The first job-container step needs the Docker API, so the executor must
200 // not start before dockerd answers. If it never does (or dies), start the
201 // executor anyway: jobs that do not use Docker still run. A shutdown signal
202 // during the wait aborts the boot instead: see abortBootError.
203 if dockerPid != 0 {
204 sock := cfg.DockerSocket
205 if sock == "" {
206 sock = dockerSocket
207 }
208 timeout := cfg.DockerReadyTimeout
209 if timeout <= 0 {
210 timeout = dockerReadyTimeout
211 }
212 err := waitDockerReady(ctx, log, sock, dockerPid, timeout, sigs)
213 bootMs["dockerd"] = time.Since(dockerStart).Milliseconds()
214 var exited *dockerdExitError
215 var abort *abortBootError
216 switch {
217 case errors.As(err, &abort):
218 stopDockerd(dockerPid)
219 stopDockerd(containerdPid)
220 return abort.code
221 case errors.As(err, &exited):
222 dockerPid = 0 // already reaped
223 }
224 }
225
226 execEnv := append(os.Environ(), "ERMITE_CONNECT="+target, "ERMITE_SANDBOX="+mode, "ERMITE_VERSION="+cfg.Version, "ERMITE_BOOT_MS="+formatBootMs(bootMs))
227 execPid, err := spawn(cfg.ExecArgs, execEnv)
228 if err != nil {
229 log.Error().Err(err).Msg("init: start executor")
230 stopDockerd(dockerPid)
231 stopDockerd(containerdPid)
232 return 1
233 }
234 forward := func(sig os.Signal) {
235 if s, ok := sig.(syscall.Signal); ok {
236 log.Info().Str("signal", s.String()).Msg("init: forwarding the signal to the executor")
237 _ = syscall.Kill(-execPid, s) // the executor leads its own process group
238 _ = syscall.Kill(execPid, s)
239 }
240 }
241 stopFwd := make(chan struct{})
242 defer close(stopFwd)
243 go func() {
244 for {
245 select {
246 case sig := <-sigs:
247 forward(sig)
248 case <-stopFwd:
249 return
250 }
251 }
252 }()
253 code := reapUntil(execPid)
254 stopDockerd(dockerPid)
255 stopDockerd(containerdPid) // after dockerd, which needs it to stop its containers
256 return code
257}
258
259// startContainerd starts /usr/bin/containerd (when the image has it) and waits,
260// bounded, for its socket. false: no containerd, or it did not come up; dockerd
261// then starts its own.
262func startContainerd(ctx context.Context, log zerowrap.Logger) (int, bool) {
263 if _, err := os.Stat(containerdBinary); err != nil {
264 return 0, false
265 }
266 // Its data next to dockerd's, on the scratch space: in a container the
267 // root is an overlay, which the overlayfs snapshotter cannot sit on.
268 root := sandbox.GuestScratch + "/containerd"
269 _ = os.MkdirAll(root, 0o711)
270 pid, err := spawn([]string{containerdBinary, "--address", containerdSocket, "--root", root}, nil)
271 if err != nil {
272 log.Warn().Err(err).Msg("init: start containerd (dockerd starts its own)")
273 return 0, false
274 }
275 deadline := time.Now().Add(containerdReadyTimeout)
276 for time.Now().Before(deadline) && ctx.Err() == nil {
277 if fi, err := os.Stat(containerdSocket); err == nil && fi.Mode()&os.ModeSocket != 0 {
278 return pid, true
279 }
280 if got, _ := wait4NoHang(pid); got == pid {
281 log.Warn().Msg("init: containerd exited at start (dockerd starts its own)")
282 return 0, false
283 }
284 time.Sleep(dockerPollInterval)
285 }
286 log.Warn().Msg("init: containerd socket not ready (dockerd starts its own)")
287 stopDockerd(pid)
288 return 0, false
289}
290
291// stopDockerd stops dockerd or containerd (SIGTERM, then SIGKILL after 5 s) and reaps it (at
292// most 5 s more after the SIGKILL); a zero pid is a no-op.
293func stopDockerd(pid int) {
294 if pid == 0 {
295 return
296 }
297 _ = syscall.Kill(pid, syscall.SIGTERM)
298 deadline := time.Now().Add(5 * time.Second)
299 for time.Now().Before(deadline) {
300 if got, _ := wait4NoHang(pid); got == pid {
301 return
302 }
303 time.Sleep(100 * time.Millisecond)
304 }
305 _ = syscall.Kill(pid, syscall.SIGKILL)
306 // SIGKILL cannot be ignored, so the exit is prompt; poll rather than block
307 // for ever on a process stuck in the kernel (uninterruptible sleep).
308 for end := time.Now().Add(5 * time.Second); time.Now().Before(end); time.Sleep(10 * time.Millisecond) {
309 got, err := wait4NoHang(pid)
310 if got == pid || (err != nil && err != syscall.EINTR) { // ECHILD: already reaped
311 return
312 }
313 }
314}
315
316const (
317 // containerdBinary and containerdSocket are the containerd init starts for
318 // dockerd; containerdReadyTimeout bounds the wait for its socket.
319 containerdBinary = "/usr/bin/containerd"
320 containerdSocket = "/run/containerd/containerd.sock"
321 containerdReadyTimeout = 20 * time.Second
322 // dockerSocket is the Docker API socket of the nested dockerd.
323 dockerSocket = "/var/run/docker.sock"
324 // dockerReadyTimeout bounds the wait for the Docker API at boot.
325 dockerReadyTimeout = 60 * time.Second
326 dockerPollInterval = 100 * time.Millisecond
327 dockerPingTimeout = 2 * time.Second
328)
329
330// dockerdExitError reports that dockerd exited (and was reaped) before its API
331// answered.
332type dockerdExitError struct{ status syscall.WaitStatus }
333
334func (e *dockerdExitError) Error() string {
335 if e.status.Signaled() {
336 return "dockerd killed by signal " + e.status.Signal().String()
337 }
338 return fmt.Sprintf("dockerd exited with code %d", e.status.ExitStatus())
339}
340
341// abortBootError means a shutdown signal (or the cancelled context) arrived
342// while waiting for the Docker API: init must not start the executor, stop
343// dockerd and exit with code (128+signal, as for a killed executor).
344//
345// Not starting the executor is safe for the host: the sandbox ends before the
346// executor connected ("container exited ... before the executor connected" /
347// "firecracker exited before the executor connected"), and the host reports the
348// job cancelled (not failed) when a cancel or shutdown was requested
349// (jobRun.execute: startSandbox error with cancelRequested). A stop that the
350// host did not ask for is reported as "the sandbox did not start".
351type abortBootError struct{ code int }
352
353func (e *abortBootError) Error() string {
354 return fmt.Sprintf("boot aborted by a shutdown signal (exit %d)", e.code)
355}
356
357// reapChildren reaps every child that has exited (orphans adopted by init
358// included) without blocking, and reports whether pid was among them, with its
359// wait status.
360func reapChildren(pid int) (bool, syscall.WaitStatus) {
361 var found bool
362 var status syscall.WaitStatus
363 for {
364 var ws syscall.WaitStatus
365 got, err := syscall.Wait4(-1, &ws, syscall.WNOHANG, nil)
366 if err == syscall.EINTR {
367 continue
368 }
369 if err != nil || got <= 0 {
370 return found, status
371 }
372 if got == pid {
373 found, status = true, ws
374 }
375 }
376}
377
378// waitDockerReady blocks until the Docker API on sock answers GET /_ping with
379// "OK", for at most timeout. It returns nil when ready, and otherwise:
380// - *dockerdExitError: dockerd (pid, a child of init) exited and was reaped;
381// - a plain error: the deadline passed;
382// - *abortBootError: a signal arrived on sigs or ctx was cancelled.
383//
384// The orphans that init adopts are reaped on every tick, and a ping in flight
385// never delays the deadline, the signals or the exit check. Every outcome is
386// logged; for the first two the caller starts the executor anyway.
387func waitDockerReady(ctx context.Context, log zerowrap.Logger, sock string, pid int, timeout time.Duration, sigs <-chan os.Signal) error {
388 start := time.Now()
389 log.Info().Str("socket", sock).Dur("timeout", timeout).Msg("init: waiting for the Docker API")
390 wctx, cancel := context.WithTimeout(ctx, timeout)
391 defer cancel() // also aborts a ping in flight
392 client := &http.Client{Transport: &http.Transport{
393 DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
394 var d net.Dialer
395 return d.DialContext(ctx, "unix", sock)
396 },
397 DisableKeepAlives: true,
398 }}
399 defer client.CloseIdleConnections()
400 ticker := time.NewTicker(dockerPollInterval)
401 defer ticker.Stop()
402
403 var pinging <-chan bool // result of the ping in flight, nil if none
404 startPing := func() {
405 ch := make(chan bool, 1)
406 pinging = ch
407 go func() { ch <- dockerPing(wctx, client) }()
408 }
409 checkExit := func() error {
410 if ok, ws := reapChildren(pid); ok {
411 err := &dockerdExitError{status: ws}
412 log.Warn().Err(err).Msg("init: dockerd died before its API was ready; starting the executor anyway (jobs using Docker will fail)")
413 return err
414 }
415 return nil
416 }
417 if err := checkExit(); err != nil {
418 return err
419 }
420 startPing()
421 for {
422 select {
423 case ok := <-pinging:
424 pinging = nil
425 if ok {
426 log.Info().Dur("after", time.Since(start).Round(time.Millisecond)).Msg("init: the Docker API is ready")
427 return nil
428 }
429 case <-ticker.C:
430 if err := checkExit(); err != nil {
431 return err
432 }
433 if pinging == nil {
434 startPing()
435 }
436 case sig := <-sigs:
437 return abortBoot(log, sig)
438 case <-wctx.Done():
439 if ctx.Err() != nil {
440 return abortBoot(log, syscall.SIGTERM)
441 }
442 log.Warn().Dur("timeout", timeout).Msg("init: the Docker API did not answer in time; starting the executor anyway (jobs using Docker will fail)")
443 return fmt.Errorf("docker API not ready after %s", timeout)
444 }
445 }
446}
447
448func abortBoot(log zerowrap.Logger, sig os.Signal) error {
449 code := 128 + int(syscall.SIGTERM)
450 if s, ok := sig.(syscall.Signal); ok {
451 code = 128 + int(s)
452 }
453 log.Warn().Str("signal", fmt.Sprint(sig)).Msg("init: shutdown signal while waiting for the Docker API; not starting the executor, stopping dockerd")
454 return &abortBootError{code: code}
455}
456
457// dockerPing reports whether GET /_ping answers 200 "OK".
458func dockerPing(ctx context.Context, client *http.Client) bool {
459 // One ping is bounded on its own so that a daemon that accepts and stalls
460 // does not hold the others back; ctx (the readiness deadline) bounds it too.
461 ctx, cancel := context.WithTimeout(ctx, dockerPingTimeout)
462 defer cancel()
463 req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://docker/_ping", nil)
464 if err != nil {
465 return false
466 }
467 resp, err := client.Do(req)
468 if err != nil {
469 return false
470 }
471 defer func() { _ = resp.Body.Close() }()
472 body, _ := io.ReadAll(io.LimitReader(resp.Body, 16))
473 return resp.StatusCode == http.StatusOK && strings.TrimSpace(string(body)) == "OK"
474}
475
476// spawn starts argv[0] as a child (own process group) without os/exec, so that
477// reapUntil is the only waiter and orphans can be reaped safely.
478func spawn(argv, env []string) (int, error) {
479 if env == nil {
480 env = os.Environ()
481 }
482 return syscall.ForkExec(argv[0], argv, &syscall.ProcAttr{
483 Env: env,
484 Files: []uintptr{0, 1, 2},
485 Sys: &syscall.SysProcAttr{Setpgid: true},
486 })
487}
488
489// reapUntil waits for every child (orphans included) and returns when pid
490// exits, with its exit code (128+signal when killed).
491func reapUntil(pid int) int {
492 for {
493 var ws syscall.WaitStatus
494 got, err := syscall.Wait4(-1, &ws, 0, nil)
495 if err == syscall.EINTR {
496 continue
497 }
498 if err != nil { // ECHILD: no child left, the executor included
499 return 1
500 }
501 if got == pid {
502 switch {
503 case ws.Exited():
504 return ws.ExitStatus()
505 case ws.Signaled():
506 return 128 + int(ws.Signal())
507 }
508 return 1
509 }
510 }
511}
512
513func wait4NoHang(pid int) (int, error) {
514 var ws syscall.WaitStatus
515 return syscall.Wait4(pid, &ws, syscall.WNOHANG, nil)
516}
517
518func powerOff() {
519 unix.Sync()
520 // On x86 Firecracker has no ACPI power off: a reboot makes the VMM exit
521 // (kernel cmdline reboot=k). On arm64 PSCI handles power off.
522 _ = unix.Reboot(unix.LINUX_REBOOT_CMD_RESTART)
523 _ = unix.Reboot(unix.LINUX_REBOOT_CMD_POWER_OFF)
524}
525
526// nestCgroups prepares cgroup v2 for a nested dockerd in a container: a cgroup
527// with member processes cannot delegate controllers to children, so move every
528// process to an "init" child and enable the controllers (like the upstream
529// docker-in-docker entrypoint). No-op without cgroup v2.
530func nestCgroups() error {
531 const root = "/sys/fs/cgroup"
532 ctl, err := os.ReadFile(root + "/cgroup.controllers")
533 if err != nil {
534 return nil
535 }
536 if err := os.MkdirAll(root+"/init", 0o755); err != nil {
537 return err
538 }
539 procs, err := os.ReadFile(root + "/cgroup.procs")
540 if err != nil {
541 return err
542 }
543 for _, p := range strings.Fields(string(procs)) {
544 // Processes may exit meanwhile: ignore per-process errors.
545 _ = os.WriteFile(root+"/init/cgroup.procs", []byte(p), 0o644)
546 }
547 var enable []string
548 for _, c := range strings.Fields(string(ctl)) {
549 enable = append(enable, "+"+c)
550 }
551 if len(enable) == 0 {
552 return nil
553 }
554 if err := os.WriteFile(root+"/cgroup.subtree_control", []byte(strings.Join(enable, " ")), 0o644); err != nil {
555 return fmt.Errorf("enable controllers: %w", err)
556 }
557 return nil
558}