1// SPDX-License-Identifier: MIT
2
3package guest
4
5import (
6 "context"
7 "errors"
8 "fmt"
9 "io"
10 "net"
11 "net/http"
12 "os"
13 "os/signal"
14 "strings"
15 "syscall"
16 "time"
17
18 "github.com/bnema/zerowrap"
19 "golang.org/x/sys/unix"
20
21 "bnema.dev/ermite-runner/internal/sandbox"
22)
23
24// InitConfig configures Init. The zero value reads the real system; tests
25// replace the fields.
26type InitConfig struct {
27 Version string
28 // Cmdline is the kernel command line; empty reads /proc/cmdline.
29 Cmdline string
30 // Getpid reports the process id; Init only acts as a VM's PID 1 when it is 1.
31 Getpid func() int
32 // Signals delivers the termination signals to forward to the executor;
33 // nil means "subscribe to SIGTERM and SIGINT". Tests inject their own.
34 Signals <-chan os.Signal
35 // SkipDocker does not start dockerd (tests, images without Docker).
36 SkipDocker bool
37 // ExecArgs is the executor command line; default
38 // [GuestExecDir/GuestExecName exec].
39 ExecArgs []string
40 // DockerdArgs replaces the dockerd command line, and the check that
41 // /usr/bin/dockerd exists (tests); SkipDocker still wins.
42 DockerdArgs []string
43 // DockerSocket is the Docker API socket dockerd listens on and init waits
44 // for; default dockerSocket.
45 DockerSocket string
46 // DockerReadyTimeout bounds the wait for the Docker API before the executor
47 // is started anyway; default dockerReadyTimeout.
48 DockerReadyTimeout time.Duration
49}
50
51// Init is `ermite-runner init`: the entrypoint of every sandbox.
52//
53// In a micro-VM it is the kernel's init (PID 1, root = the read-only exec disk,
54// /dev/vdb = shared base image, /dev/vdc = per-job scratch disk): it mounts the
55// pseudo filesystems, sets the clock and the DNS server, builds an overlayfs
56// root (lower = base, upper = scratch) and pivots into it, then does what a
57// container does.
58//
59// In a container the root and the pseudo filesystems are ready: it skips all of
60// that and only prepares cgroups. Then, in both: start dockerd with its data
61// root on the scratch space, wait (bounded) for its API to answer, run the executor
62// (`ermite-runner exec`) as a child, reap orphans, stop dockerd, and finish by
63// powering the VM off, or exiting with the executor's code in a container.
64func Init(ctx context.Context, cfg InitConfig) int {
65 log := zerowrap.FromCtx(ctx)
66 // The kernel starts a VM's init with umask 0, which dockerd, the executor
67 // and every step would inherit: files and t.TempDir() directories would be
68 // world-writable. GitHub's runners have 022, and so do the jobs here.
69 syscall.Umask(StepUmask)
70 if cfg.Getpid == nil {
71 cfg.Getpid = os.Getpid
72 }
73 if cfg.ExecArgs == nil {
74 cfg.ExecArgs = []string{sandbox.GuestExecDir + "/" + sandbox.GuestExecName, "exec"}
75 }
76 cmdline := cfg.Cmdline
77 if cmdline == "" {
78 b, err := os.ReadFile("/proc/cmdline")
79 if err != nil && cfg.Getpid() == 1 {
80 // A VM's init starts with nothing mounted: /proc is needed to read
81 // the kernel command line that says what we are. setupVM reuses it.
82 if merr := unix.Mount("proc", "/proc", "proc", 0, ""); merr == nil {
83 b, err = os.ReadFile("/proc/cmdline")
84 }
85 }
86 if err == nil {
87 cmdline = string(b)
88 }
89 }
90 args := ParseCmdline(cmdline)
91 inVM := args[sandbox.CmdlineSandbox] == string(sandbox.VM) && cfg.Getpid() == 1
92
93 mode := string(sandbox.Container)
94 if inVM {
95 mode = string(sandbox.VM)
96 if err := setupVM(log, args); err != nil {
97 log.Error().Err(err).Msg("init: vm setup failed")
98 powerOff()
99 return 1
100 }
101 }
102 code := runPayload(ctx, cfg, mode, args)
103 if inVM {
104 powerOff()
105 }
106 return code
107}
108
109// StepUmask is the umask of everything the init starts, steps included: the
110// one of GitHub's hosted runners.
111const StepUmask = 0o022
112
113// defaultPath is the PATH of a sandbox whose init was started by the kernel
114// with an empty environment.
115const defaultPath = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
116
117func runPayload(ctx context.Context, cfg InitConfig, mode string, args map[string]string) int {
118 log := zerowrap.FromCtx(ctx)
119 if os.Getenv("PATH") == "" {
120 _ = os.Setenv("PATH", defaultPath)
121 }
122 if os.Getenv("HOME") == "" {
123 _ = os.Setenv("HOME", "/root")
124 }
125 if mode == string(sandbox.Container) {
126 if err := nestCgroups(); err != nil {
127 log.Warn().Err(err).Msg("init: cgroup setup (docker inside may not start)")
128 }
129 }
130 target := args[sandbox.CmdlineConnect]
131 if target == "" {
132 target = os.Getenv("ERMITE_CONNECT")
133 }
134 if _, err := ParseTarget(target); err != nil {
135 log.Error().Err(err).Msg("init: no executor target")
136 return 1
137 }
138
139 var dockerPid int
140 _, statErr := os.Stat("/usr/bin/dockerd")
141 if !cfg.SkipDocker && (statErr == nil || cfg.DockerdArgs != nil) {
142 _ = os.MkdirAll(sandbox.GuestScratch+"/docker", 0o711)
143 argv := cfg.DockerdArgs
144 if argv == nil {
145 argv = []string{"/usr/bin/dockerd", "--data-root", sandbox.GuestScratch + "/docker",
146 "--storage-driver", "overlay2", "--host", "unix:///var/run/docker.sock"}
147 }
148 pid, err := spawn(argv, nil)
149 if err != nil {
150 log.Error().Err(err).Msg("init: start dockerd")
151 } else {
152 dockerPid = pid
153 }
154 }
155
156 // SIGTERM/SIGINT (docker stop, or the host stopping the VM) go to the
157 // executor, which cancels its script with the usual escalation and reports
158 // its Exit; init then finishes normally instead of dying with children.
159 sigs := cfg.Signals
160 if sigs == nil {
161 ch := make(chan os.Signal, 4)
162 signal.Notify(ch, syscall.SIGTERM, syscall.SIGINT)
163 defer signal.Stop(ch)
164 sigs = ch
165 }
166
167 // The first job-container step needs the Docker API, so the executor must
168 // not start before dockerd answers. If it never does (or dies), start the
169 // executor anyway: jobs that do not use Docker still run. A shutdown signal
170 // during the wait aborts the boot instead: see abortBootError.
171 if dockerPid != 0 {
172 sock := cfg.DockerSocket
173 if sock == "" {
174 sock = dockerSocket
175 }
176 timeout := cfg.DockerReadyTimeout
177 if timeout <= 0 {
178 timeout = dockerReadyTimeout
179 }
180 err := waitDockerReady(ctx, log, sock, dockerPid, timeout, sigs)
181 var exited *dockerdExitError
182 var abort *abortBootError
183 switch {
184 case errors.As(err, &abort):
185 stopDockerd(dockerPid)
186 return abort.code
187 case errors.As(err, &exited):
188 dockerPid = 0 // already reaped
189 }
190 }
191
192 execEnv := append(os.Environ(), "ERMITE_CONNECT="+target, "ERMITE_SANDBOX="+mode, "ERMITE_VERSION="+cfg.Version)
193 execPid, err := spawn(cfg.ExecArgs, execEnv)
194 if err != nil {
195 log.Error().Err(err).Msg("init: start executor")
196 stopDockerd(dockerPid)
197 return 1
198 }
199 forward := func(sig os.Signal) {
200 if s, ok := sig.(syscall.Signal); ok {
201 log.Info().Str("signal", s.String()).Msg("init: forwarding the signal to the executor")
202 _ = syscall.Kill(-execPid, s) // the executor leads its own process group
203 _ = syscall.Kill(execPid, s)
204 }
205 }
206 stopFwd := make(chan struct{})
207 defer close(stopFwd)
208 go func() {
209 for {
210 select {
211 case sig := <-sigs:
212 forward(sig)
213 case <-stopFwd:
214 return
215 }
216 }
217 }()
218 code := reapUntil(execPid)
219 stopDockerd(dockerPid)
220 return code
221}
222
223// stopDockerd stops dockerd (SIGTERM, then SIGKILL after 5 s) and reaps it (at
224// most 5 s more after the SIGKILL); a zero pid is a no-op.
225func stopDockerd(pid int) {
226 if pid == 0 {
227 return
228 }
229 _ = syscall.Kill(pid, syscall.SIGTERM)
230 deadline := time.Now().Add(5 * time.Second)
231 for time.Now().Before(deadline) {
232 if got, _ := wait4NoHang(pid); got == pid {
233 return
234 }
235 time.Sleep(100 * time.Millisecond)
236 }
237 _ = syscall.Kill(pid, syscall.SIGKILL)
238 // SIGKILL cannot be ignored, so the exit is prompt; poll rather than block
239 // for ever on a process stuck in the kernel (uninterruptible sleep).
240 for end := time.Now().Add(5 * time.Second); time.Now().Before(end); time.Sleep(10 * time.Millisecond) {
241 got, err := wait4NoHang(pid)
242 if got == pid || (err != nil && err != syscall.EINTR) { // ECHILD: already reaped
243 return
244 }
245 }
246}
247
248const (
249 // dockerSocket is the Docker API socket of the nested dockerd.
250 dockerSocket = "/var/run/docker.sock"
251 // dockerReadyTimeout bounds the wait for the Docker API at boot.
252 dockerReadyTimeout = 60 * time.Second
253 dockerPollInterval = 100 * time.Millisecond
254 dockerPingTimeout = 2 * time.Second
255)
256
257// dockerdExitError reports that dockerd exited (and was reaped) before its API
258// answered.
259type dockerdExitError struct{ status syscall.WaitStatus }
260
261func (e *dockerdExitError) Error() string {
262 if e.status.Signaled() {
263 return "dockerd killed by signal " + e.status.Signal().String()
264 }
265 return fmt.Sprintf("dockerd exited with code %d", e.status.ExitStatus())
266}
267
268// abortBootError means a shutdown signal (or the cancelled context) arrived
269// while waiting for the Docker API: init must not start the executor, stop
270// dockerd and exit with code (128+signal, as for a killed executor).
271//
272// Not starting the executor is safe for the host: the sandbox ends before the
273// executor connected ("container exited ... before the executor connected" /
274// "firecracker exited before the executor connected"), and the host reports the
275// job cancelled (not failed) when a cancel or shutdown was requested
276// (jobRun.execute: startSandbox error with cancelRequested). A stop that the
277// host did not ask for is reported as "the sandbox did not start".
278type abortBootError struct{ code int }
279
280func (e *abortBootError) Error() string {
281 return fmt.Sprintf("boot aborted by a shutdown signal (exit %d)", e.code)
282}
283
284// reapChildren reaps every child that has exited (orphans adopted by init
285// included) without blocking, and reports whether pid was among them, with its
286// wait status.
287func reapChildren(pid int) (bool, syscall.WaitStatus) {
288 var found bool
289 var status syscall.WaitStatus
290 for {
291 var ws syscall.WaitStatus
292 got, err := syscall.Wait4(-1, &ws, syscall.WNOHANG, nil)
293 if err == syscall.EINTR {
294 continue
295 }
296 if err != nil || got <= 0 {
297 return found, status
298 }
299 if got == pid {
300 found, status = true, ws
301 }
302 }
303}
304
305// waitDockerReady blocks until the Docker API on sock answers GET /_ping with
306// "OK", for at most timeout. It returns nil when ready, and otherwise:
307// - *dockerdExitError: dockerd (pid, a child of init) exited and was reaped;
308// - a plain error: the deadline passed;
309// - *abortBootError: a signal arrived on sigs or ctx was cancelled.
310//
311// The orphans that init adopts are reaped on every tick, and a ping in flight
312// never delays the deadline, the signals or the exit check. Every outcome is
313// logged; for the first two the caller starts the executor anyway.
314func waitDockerReady(ctx context.Context, log zerowrap.Logger, sock string, pid int, timeout time.Duration, sigs <-chan os.Signal) error {
315 start := time.Now()
316 log.Info().Str("socket", sock).Dur("timeout", timeout).Msg("init: waiting for the Docker API")
317 wctx, cancel := context.WithTimeout(ctx, timeout)
318 defer cancel() // also aborts a ping in flight
319 client := &http.Client{Transport: &http.Transport{
320 DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
321 var d net.Dialer
322 return d.DialContext(ctx, "unix", sock)
323 },
324 DisableKeepAlives: true,
325 }}
326 defer client.CloseIdleConnections()
327 ticker := time.NewTicker(dockerPollInterval)
328 defer ticker.Stop()
329
330 var pinging <-chan bool // result of the ping in flight, nil if none
331 startPing := func() {
332 ch := make(chan bool, 1)
333 pinging = ch
334 go func() { ch <- dockerPing(wctx, client) }()
335 }
336 checkExit := func() error {
337 if ok, ws := reapChildren(pid); ok {
338 err := &dockerdExitError{status: ws}
339 log.Warn().Err(err).Msg("init: dockerd died before its API was ready; starting the executor anyway (jobs using Docker will fail)")
340 return err
341 }
342 return nil
343 }
344 if err := checkExit(); err != nil {
345 return err
346 }
347 startPing()
348 for {
349 select {
350 case ok := <-pinging:
351 pinging = nil
352 if ok {
353 log.Info().Dur("after", time.Since(start).Round(time.Millisecond)).Msg("init: the Docker API is ready")
354 return nil
355 }
356 case <-ticker.C:
357 if err := checkExit(); err != nil {
358 return err
359 }
360 if pinging == nil {
361 startPing()
362 }
363 case sig := <-sigs:
364 return abortBoot(log, sig)
365 case <-wctx.Done():
366 if ctx.Err() != nil {
367 return abortBoot(log, syscall.SIGTERM)
368 }
369 log.Warn().Dur("timeout", timeout).Msg("init: the Docker API did not answer in time; starting the executor anyway (jobs using Docker will fail)")
370 return fmt.Errorf("docker API not ready after %s", timeout)
371 }
372 }
373}
374
375func abortBoot(log zerowrap.Logger, sig os.Signal) error {
376 code := 128 + int(syscall.SIGTERM)
377 if s, ok := sig.(syscall.Signal); ok {
378 code = 128 + int(s)
379 }
380 log.Warn().Str("signal", fmt.Sprint(sig)).Msg("init: shutdown signal while waiting for the Docker API; not starting the executor, stopping dockerd")
381 return &abortBootError{code: code}
382}
383
384// dockerPing reports whether GET /_ping answers 200 "OK".
385func dockerPing(ctx context.Context, client *http.Client) bool {
386 // One ping is bounded on its own so that a daemon that accepts and stalls
387 // does not hold the others back; ctx (the readiness deadline) bounds it too.
388 ctx, cancel := context.WithTimeout(ctx, dockerPingTimeout)
389 defer cancel()
390 req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://docker/_ping", nil)
391 if err != nil {
392 return false
393 }
394 resp, err := client.Do(req)
395 if err != nil {
396 return false
397 }
398 defer func() { _ = resp.Body.Close() }()
399 body, _ := io.ReadAll(io.LimitReader(resp.Body, 16))
400 return resp.StatusCode == http.StatusOK && strings.TrimSpace(string(body)) == "OK"
401}
402
403// spawn starts argv[0] as a child (own process group) without os/exec, so that
404// reapUntil is the only waiter and orphans can be reaped safely.
405func spawn(argv, env []string) (int, error) {
406 if env == nil {
407 env = os.Environ()
408 }
409 return syscall.ForkExec(argv[0], argv, &syscall.ProcAttr{
410 Env: env,
411 Files: []uintptr{0, 1, 2},
412 Sys: &syscall.SysProcAttr{Setpgid: true},
413 })
414}
415
416// reapUntil waits for every child (orphans included) and returns when pid
417// exits, with its exit code (128+signal when killed).
418func reapUntil(pid int) int {
419 for {
420 var ws syscall.WaitStatus
421 got, err := syscall.Wait4(-1, &ws, 0, nil)
422 if err == syscall.EINTR {
423 continue
424 }
425 if err != nil { // ECHILD: no child left, the executor included
426 return 1
427 }
428 if got == pid {
429 switch {
430 case ws.Exited():
431 return ws.ExitStatus()
432 case ws.Signaled():
433 return 128 + int(ws.Signal())
434 }
435 return 1
436 }
437 }
438}
439
440func wait4NoHang(pid int) (int, error) {
441 var ws syscall.WaitStatus
442 return syscall.Wait4(pid, &ws, syscall.WNOHANG, nil)
443}
444
445func powerOff() {
446 unix.Sync()
447 // On x86 Firecracker has no ACPI power off: a reboot makes the VMM exit
448 // (kernel cmdline reboot=k). On arm64 PSCI handles power off.
449 _ = unix.Reboot(unix.LINUX_REBOOT_CMD_RESTART)
450 _ = unix.Reboot(unix.LINUX_REBOOT_CMD_POWER_OFF)
451}
452
453// nestCgroups prepares cgroup v2 for a nested dockerd in a container: a cgroup
454// with member processes cannot delegate controllers to children, so move every
455// process to an "init" child and enable the controllers (like the upstream
456// docker-in-docker entrypoint). No-op without cgroup v2.
457func nestCgroups() error {
458 const root = "/sys/fs/cgroup"
459 ctl, err := os.ReadFile(root + "/cgroup.controllers")
460 if err != nil {
461 return nil
462 }
463 if err := os.MkdirAll(root+"/init", 0o755); err != nil {
464 return err
465 }
466 procs, err := os.ReadFile(root + "/cgroup.procs")
467 if err != nil {
468 return err
469 }
470 for _, p := range strings.Fields(string(procs)) {
471 // Processes may exit meanwhile: ignore per-process errors.
472 _ = os.WriteFile(root+"/init/cgroup.procs", []byte(p), 0o644)
473 }
474 var enable []string
475 for _, c := range strings.Fields(string(ctl)) {
476 enable = append(enable, "+"+c)
477 }
478 if len(enable) == 0 {
479 return nil
480 }
481 if err := os.WriteFile(root+"/cgroup.subtree_control", []byte(strings.Join(enable, " ")), 0o644); err != nil {
482 return fmt.Errorf("enable controllers: %w", err)
483 }
484 return nil
485}