1// SPDX-License-Identifier: MIT
2
3package guest
4
5import (
6 "context"
7 "fmt"
8 "os"
9 "os/signal"
10 "strings"
11 "syscall"
12 "time"
13
14 "github.com/bnema/zerowrap"
15 "golang.org/x/sys/unix"
16
17 "bnema.dev/ermite-runner/internal/sandbox"
18)
19
20// InitConfig configures Init. The zero value reads the real system; tests
21// replace the fields.
22type InitConfig struct {
23 Version string
24 // Cmdline is the kernel command line; empty reads /proc/cmdline.
25 Cmdline string
26 // Getpid reports the process id; Init only acts as a VM's PID 1 when it is 1.
27 Getpid func() int
28 // Signals delivers the termination signals to forward to the executor;
29 // nil means "subscribe to SIGTERM and SIGINT". Tests inject their own.
30 Signals <-chan os.Signal
31 // SkipDocker does not start dockerd (tests, images without Docker).
32 SkipDocker bool
33 // ExecArgs is the executor command line; default
34 // [GuestExecDir/GuestExecName exec].
35 ExecArgs []string
36}
37
38// Init is `ermite-runner init`: the entrypoint of every sandbox.
39//
40// In a micro-VM it is the kernel's init (PID 1, root = the read-only exec disk,
41// /dev/vdb = shared base image, /dev/vdc = per-job scratch disk): it mounts the
42// pseudo filesystems, sets the clock and the DNS server, builds an overlayfs
43// root (lower = base, upper = scratch) and pivots into it, then does what a
44// container does.
45//
46// In a container the root and the pseudo filesystems are ready: it skips all of
47// that and only prepares cgroups. Then, in both: start dockerd with its data
48// root on the scratch space, wait for its socket, run the executor
49// (`ermite-runner exec`) as a child, reap orphans, stop dockerd, and finish by
50// powering the VM off, or exiting with the executor's code in a container.
51func Init(ctx context.Context, cfg InitConfig) int {
52 log := zerowrap.FromCtx(ctx)
53 if cfg.Getpid == nil {
54 cfg.Getpid = os.Getpid
55 }
56 if cfg.ExecArgs == nil {
57 cfg.ExecArgs = []string{sandbox.GuestExecDir + "/" + sandbox.GuestExecName, "exec"}
58 }
59 cmdline := cfg.Cmdline
60 if cmdline == "" {
61 b, err := os.ReadFile("/proc/cmdline")
62 if err != nil && cfg.Getpid() == 1 {
63 // A VM's init starts with nothing mounted: /proc is needed to read
64 // the kernel command line that says what we are. setupVM reuses it.
65 if merr := unix.Mount("proc", "/proc", "proc", 0, ""); merr == nil {
66 b, err = os.ReadFile("/proc/cmdline")
67 }
68 }
69 if err == nil {
70 cmdline = string(b)
71 }
72 }
73 args := ParseCmdline(cmdline)
74 inVM := args[sandbox.CmdlineSandbox] == string(sandbox.VM) && cfg.Getpid() == 1
75
76 mode := string(sandbox.Container)
77 if inVM {
78 mode = string(sandbox.VM)
79 if err := setupVM(log, args); err != nil {
80 log.Error().Err(err).Msg("init: vm setup failed")
81 powerOff()
82 return 1
83 }
84 }
85 code := runPayload(ctx, cfg, mode, args)
86 if inVM {
87 powerOff()
88 }
89 return code
90}
91
92// defaultPath is the PATH of a sandbox whose init was started by the kernel
93// with an empty environment.
94const defaultPath = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
95
96func runPayload(ctx context.Context, cfg InitConfig, mode string, args map[string]string) int {
97 log := zerowrap.FromCtx(ctx)
98 if os.Getenv("PATH") == "" {
99 _ = os.Setenv("PATH", defaultPath)
100 }
101 if os.Getenv("HOME") == "" {
102 _ = os.Setenv("HOME", "/root")
103 }
104 if mode == string(sandbox.Container) {
105 if err := nestCgroups(); err != nil {
106 log.Warn().Err(err).Msg("init: cgroup setup (docker inside may not start)")
107 }
108 }
109 target := args[sandbox.CmdlineConnect]
110 if target == "" {
111 target = os.Getenv("ERMITE_CONNECT")
112 }
113 if _, err := ParseTarget(target); err != nil {
114 log.Error().Err(err).Msg("init: no executor target")
115 return 1
116 }
117
118 var dockerPid int
119 if _, err := os.Stat("/usr/bin/dockerd"); err == nil && !cfg.SkipDocker {
120 _ = os.MkdirAll(sandbox.GuestScratch+"/docker", 0o711)
121 pid, err := spawn([]string{"/usr/bin/dockerd", "--data-root", sandbox.GuestScratch + "/docker",
122 "--storage-driver", "overlay2", "--host", "unix:///var/run/docker.sock"}, nil)
123 if err != nil {
124 log.Error().Err(err).Msg("init: start dockerd")
125 } else {
126 dockerPid = pid
127 }
128 }
129
130 execPid, err := spawn(cfg.ExecArgs, append(os.Environ(),
131 "ERMITE_CONNECT="+target, "ERMITE_SANDBOX="+mode, "ERMITE_VERSION="+cfg.Version))
132 if err != nil {
133 log.Error().Err(err).Msg("init: start executor")
134 return 1
135 }
136 // SIGTERM/SIGINT (docker stop, or the host stopping the VM) go to the
137 // executor, which cancels its script with the usual escalation and reports
138 // its Exit; init then finishes normally instead of dying with children.
139 sigs := cfg.Signals
140 if sigs == nil {
141 ch := make(chan os.Signal, 4)
142 signal.Notify(ch, syscall.SIGTERM, syscall.SIGINT)
143 defer signal.Stop(ch)
144 sigs = ch
145 }
146 stopFwd := make(chan struct{})
147 defer close(stopFwd)
148 go func() {
149 for {
150 select {
151 case sig := <-sigs:
152 if s, ok := sig.(syscall.Signal); ok {
153 log.Info().Str("signal", s.String()).Msg("init: forwarding the signal to the executor")
154 _ = syscall.Kill(-execPid, s) // the executor leads its own process group
155 _ = syscall.Kill(execPid, s)
156 }
157 case <-stopFwd:
158 return
159 }
160 }
161 }()
162 code := reapUntil(execPid)
163 if dockerPid != 0 {
164 _ = syscall.Kill(dockerPid, syscall.SIGTERM)
165 deadline := time.Now().Add(5 * time.Second)
166 for time.Now().Before(deadline) {
167 if pid, _ := wait4NoHang(dockerPid); pid == dockerPid {
168 break
169 }
170 time.Sleep(100 * time.Millisecond)
171 }
172 _ = syscall.Kill(dockerPid, syscall.SIGKILL)
173 }
174 return code
175}
176
177// spawn starts argv[0] as a child (own process group) without os/exec, so that
178// reapUntil is the only waiter and orphans can be reaped safely.
179func spawn(argv, env []string) (int, error) {
180 if env == nil {
181 env = os.Environ()
182 }
183 return syscall.ForkExec(argv[0], argv, &syscall.ProcAttr{
184 Env: env,
185 Files: []uintptr{0, 1, 2},
186 Sys: &syscall.SysProcAttr{Setpgid: true},
187 })
188}
189
190// reapUntil waits for every child (orphans included) and returns when pid
191// exits, with its exit code (128+signal when killed).
192func reapUntil(pid int) int {
193 for {
194 var ws syscall.WaitStatus
195 got, err := syscall.Wait4(-1, &ws, 0, nil)
196 if err == syscall.EINTR {
197 continue
198 }
199 if err != nil { // ECHILD: no child left, the executor included
200 return 1
201 }
202 if got == pid {
203 switch {
204 case ws.Exited():
205 return ws.ExitStatus()
206 case ws.Signaled():
207 return 128 + int(ws.Signal())
208 }
209 return 1
210 }
211 }
212}
213
214func wait4NoHang(pid int) (int, error) {
215 var ws syscall.WaitStatus
216 return syscall.Wait4(pid, &ws, syscall.WNOHANG, nil)
217}
218
219func powerOff() {
220 unix.Sync()
221 // On x86 Firecracker has no ACPI power off: a reboot makes the VMM exit
222 // (kernel cmdline reboot=k). On arm64 PSCI handles power off.
223 _ = unix.Reboot(unix.LINUX_REBOOT_CMD_RESTART)
224 _ = unix.Reboot(unix.LINUX_REBOOT_CMD_POWER_OFF)
225}
226
227// nestCgroups prepares cgroup v2 for a nested dockerd in a container: a cgroup
228// with member processes cannot delegate controllers to children, so move every
229// process to an "init" child and enable the controllers (like the upstream
230// docker-in-docker entrypoint). No-op without cgroup v2.
231func nestCgroups() error {
232 const root = "/sys/fs/cgroup"
233 ctl, err := os.ReadFile(root + "/cgroup.controllers")
234 if err != nil {
235 return nil
236 }
237 if err := os.MkdirAll(root+"/init", 0o755); err != nil {
238 return err
239 }
240 procs, err := os.ReadFile(root + "/cgroup.procs")
241 if err != nil {
242 return err
243 }
244 for _, p := range strings.Fields(string(procs)) {
245 // Processes may exit meanwhile: ignore per-process errors.
246 _ = os.WriteFile(root+"/init/cgroup.procs", []byte(p), 0o644)
247 }
248 var enable []string
249 for _, c := range strings.Fields(string(ctl)) {
250 enable = append(enable, "+"+c)
251 }
252 if len(enable) == 0 {
253 return nil
254 }
255 if err := os.WriteFile(root+"/cgroup.subtree_control", []byte(strings.Join(enable, " ")), 0o644); err != nil {
256 return fmt.Errorf("enable controllers: %w", err)
257 }
258 return nil
259}