1// SPDX-License-Identifier: MIT
2
3package guest
4
5import (
6 "fmt"
7 "os"
8 "strconv"
9 "strings"
10
11 "github.com/bnema/zerowrap"
12 "golang.org/x/sys/unix"
13
14 "bnema.dev/ermite-runner/internal/sandbox"
15)
16
17// Devices of the micro-VM, in the order the host attaches them (virtio-blk).
18const (
19 devBase = "/dev/vdb" // shared read-only base image (ext4)
20 devScratch = "/dev/vdc" // per-job scratch disk (ext4)
21 // devExternals is the host's external tools (Node.js), attached last and only
22 // when the command line says ermite.externals=1.
23 devExternals = "/dev/vdd"
24)
25
26// KernelArg names of the clock and DNS settings (not part of the contract
27// constants because only init reads them).
28const (
29 cmdlineTime = "ermite.time" // host unix time at boot, seconds
30)
31
32// setupVM turns the minimal boot root (the exec disk) into the job's root:
33// overlayfs of base + scratch, with the exec disk bound read-only at
34// /opt/ermite, then pivot_root into it and mount the pseudo filesystems.
35func setupVM(log zerowrap.Logger, args map[string]string) error {
36 // Pseudo filesystems of the boot root. Their mountpoints exist on the exec
37 // disk (the root is read-only).
38 for _, m := range []struct{ src, dst, typ string }{
39 {"proc", "/proc", "proc"},
40 {"sysfs", "/sys", "sysfs"},
41 {"devtmpfs", "/dev", "devtmpfs"},
42 } {
43 // proc is mounted by Init (to read the command line) and devtmpfs by the
44 // kernel (DEVTMPFS_MOUNT): mount only what is missing.
45 if isMounted(m.dst) {
46 continue
47 }
48 if err := mount(m.src, m.dst, m.typ, 0, ""); err != nil {
49 return err
50 }
51 }
52 // Firecracker has no RTC on x86: the guest clock starts at the epoch, which
53 // breaks TLS. The host passes its time on the command line.
54 if v, ok := args[cmdlineTime]; ok {
55 if sec, err := strconv.ParseInt(v, 10, 64); err == nil && sec > 1_600_000_000 {
56 if err := unix.ClockSettime(unix.CLOCK_REALTIME, &unix.Timespec{Sec: sec}); err != nil {
57 log.Warn().Err(err).Msg("init: set clock")
58 }
59 }
60 }
61
62 if err := mount(devBase, "/mnt/base", "ext4", unix.MS_RDONLY, ""); err != nil {
63 return err
64 }
65 if err := mount(devScratch, "/mnt/scratch", "ext4", unix.MS_NOATIME, ""); err != nil {
66 return err
67 }
68 for _, d := range []string{"/mnt/scratch/upper", "/mnt/scratch/work", "/mnt/scratch/docker"} {
69 if err := os.MkdirAll(d, 0o755); err != nil {
70 return err
71 }
72 }
73 opts := "lowerdir=/mnt/base,upperdir=/mnt/scratch/upper,workdir=/mnt/scratch/work"
74 if err := mount("overlay", "/mnt/newroot", "overlay", 0, opts); err != nil {
75 return err
76 }
77 // The new root needs these mountpoints; they live in the (writable) upper.
78 for _, d := range []string{"/mnt/newroot/proc", "/mnt/newroot/sys", "/mnt/newroot/dev",
79 "/mnt/newroot/run", "/mnt/newroot/tmp", "/mnt/newroot/scratch", "/mnt/newroot/.oldroot",
80 "/mnt/newroot" + sandbox.GuestExecDir} {
81 if err := os.MkdirAll(d, 0o755); err != nil {
82 return err
83 }
84 }
85 // The host's external tools, read-only, in GitHub's layout. A failure is not
86 // fatal: only the jobs that need Node.js are affected, and they say so.
87 if args[sandbox.CmdlineExternals] == "1" {
88 dst := "/mnt/newroot" + sandbox.GuestExternalsDir
89 if err := os.MkdirAll(dst, 0o755); err != nil {
90 log.Warn().Err(err).Msg("init: externals mountpoint")
91 } else if err := mount(devExternals, dst, "ext4", unix.MS_RDONLY|unix.MS_NOSUID|unix.MS_NODEV, ""); err != nil {
92 log.Warn().Err(err).Msg("init: mount the externals disk")
93 }
94 }
95 // Scratch and the exec disk stay reachable after the pivot.
96 if err := mount("/mnt/scratch", "/mnt/newroot"+sandbox.GuestScratch, "", unix.MS_BIND, ""); err != nil {
97 return err
98 }
99 if err := mount("/", "/mnt/newroot"+sandbox.GuestExecDir, "", unix.MS_BIND, ""); err != nil {
100 return err
101 }
102 if err := mount("", "/mnt/newroot"+sandbox.GuestExecDir, "", unix.MS_BIND|unix.MS_REMOUNT|unix.MS_RDONLY, ""); err != nil {
103 return err
104 }
105 for _, d := range []string{"/proc", "/sys", "/dev"} {
106 if err := mount(d, "/mnt/newroot"+d, "", unix.MS_MOVE, ""); err != nil {
107 return err
108 }
109 }
110 if err := unix.Chdir("/mnt/newroot"); err != nil {
111 return err
112 }
113 if err := unix.PivotRoot(".", ".oldroot"); err != nil {
114 return fmt.Errorf("pivot_root: %w", err)
115 }
116 if err := unix.Chroot("."); err != nil {
117 return err
118 }
119 if err := unix.Chdir("/"); err != nil {
120 return err
121 }
122 _ = unix.Unmount("/.oldroot", unix.MNT_DETACH)
123
124 for _, m := range []struct {
125 src, dst, typ string
126 flags uintptr
127 data string
128 }{
129 {"devpts", "/dev/pts", "devpts", 0, "gid=5,mode=620,ptmxmode=666"},
130 {"tmpfs", "/dev/shm", "tmpfs", unix.MS_NOSUID | unix.MS_NODEV, ""},
131 {"tmpfs", "/run", "tmpfs", unix.MS_NOSUID | unix.MS_NODEV, "mode=755"},
132 {"tmpfs", "/tmp", "tmpfs", unix.MS_NOSUID | unix.MS_NODEV, "mode=1777"},
133 {"cgroup2", "/sys/fs/cgroup", "cgroup2", unix.MS_NOSUID | unix.MS_NODEV | unix.MS_NOEXEC, ""},
134 } {
135 if err := os.MkdirAll(m.dst, 0o755); err != nil {
136 return err
137 }
138 if err := mount(m.src, m.dst, m.typ, m.flags, m.data); err != nil {
139 // devpts and shm failures are not fatal for the stub.
140 log.Warn().Err(err).Str("target", m.dst).Msg("init: mount")
141 }
142 }
143 // Mounted at boot on GitHub's image (systemd's
144 // proc-sys-fs-binfmt_misc.automount): jobs read the handlers that
145 // setup-qemu-action registers there. A kernel without BINFMT_MISC has no
146 // such directory; the jobs run without it.
147 if err := mount("binfmt_misc", "/proc/sys/fs/binfmt_misc", "binfmt_misc", unix.MS_NOSUID|unix.MS_NODEV|unix.MS_NOEXEC, ""); err != nil {
148 log.Warn().Err(err).Msg("init: mount binfmt_misc")
149 }
150 _ = os.Chmod("/tmp", sandbox.TmpMode)
151
152 gw := args[sandbox.CmdlineGateway]
153 if err := writeResolvConf(gw); err != nil {
154 return err
155 }
156 _ = os.WriteFile("/etc/hostname", []byte("ermite-job\n"), 0o644)
157 _ = os.WriteFile("/etc/hosts", []byte("127.0.0.1 localhost ermite-job\n"), 0o644)
158 _ = unix.Sethostname([]byte("ermite-job"))
159 if err := loUp(); err != nil {
160 log.Warn().Err(err).Msg("init: loopback")
161 }
162 // IPv6 is disabled on the kernel command line (ipv6.disable=1); this is a
163 // second line of defence when the module is built in but not disabled.
164 _ = os.WriteFile("/proc/sys/net/ipv6/conf/all/disable_ipv6", []byte("1\n"), 0o644)
165 return nil
166}
167
168// writeResolvConf points the guest at the gateway, where the runner's DNS
169// forwarder listens. /etc/resolv.conf may be a dangling symlink in the image
170// (systemd-resolved stub): replace it.
171func writeResolvConf(gw string) error {
172 if gw == "" {
173 return fmt.Errorf("kernel command line has no %s", sandbox.CmdlineGateway)
174 }
175 _ = os.Remove("/etc/resolv.conf")
176 return os.WriteFile("/etc/resolv.conf", []byte("nameserver "+gw+"\noptions timeout:2 attempts:2\n"), 0o644)
177}
178
179func mount(src, dst, typ string, flags uintptr, data string) error {
180 if err := os.MkdirAll(dst, 0o755); err != nil && !os.IsExist(err) {
181 // Read-only boot root: the mountpoint must already exist.
182 if _, serr := os.Stat(dst); serr != nil {
183 return fmt.Errorf("mountpoint %s: %w", dst, err)
184 }
185 }
186 if err := unix.Mount(src, dst, typ, flags, data); err != nil {
187 return fmt.Errorf("mount %s on %s (%s): %w", src, dst, strings.TrimSpace(typ), err)
188 }
189 return nil
190}
191
192// loUp brings the loopback interface up (the kernel leaves it down).
193func loUp() error {
194 fd, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM|unix.SOCK_CLOEXEC, 0)
195 if err != nil {
196 return err
197 }
198 defer func() { _ = unix.Close(fd) }()
199 ifr, err := unix.NewIfreq("lo")
200 if err != nil {
201 return err
202 }
203 if err := unix.IoctlIfreq(fd, unix.SIOCGIFFLAGS, ifr); err != nil {
204 return err
205 }
206 ifr.SetUint16(ifr.Uint16() | unix.IFF_UP)
207 return unix.IoctlIfreq(fd, unix.SIOCSIFFLAGS, ifr)
208}
209
210// isMounted reports whether path is a mount point listed in /proc/self/mountinfo.
211func isMounted(path string) bool {
212 b, err := os.ReadFile("/proc/self/mountinfo")
213 if err != nil {
214 return false
215 }
216 for _, line := range strings.Split(string(b), "\n") {
217 f := strings.Fields(line)
218 if len(f) > 4 && f[4] == path {
219 return true
220 }
221 }
222 return false
223}