1// SPDX-License-Identifier: MIT
2
3package guest
4
5import (
6 "fmt"
7 "os"
8 "strconv"
9 "strings"
10
11 "github.com/bnema/zerowrap"
12 "golang.org/x/sys/unix"
13
14 "bnema.dev/ermite-runner/internal/sandbox"
15)
16
17// Devices of the micro-VM, in the order the host attaches them (virtio-blk).
18const (
19 devBase = "/dev/vdb" // shared read-only base image (ext4)
20 devScratch = "/dev/vdc" // per-job scratch disk (ext4)
21)
22
23// KernelArg names of the clock and DNS settings (not part of the contract
24// constants because only init reads them).
25const (
26 cmdlineTime = "ermite.time" // host unix time at boot, seconds
27)
28
29// setupVM turns the minimal boot root (the exec disk) into the job's root:
30// overlayfs of base + scratch, with the exec disk bound read-only at
31// /opt/ermite, then pivot_root into it and mount the pseudo filesystems.
32func setupVM(log zerowrap.Logger, args map[string]string) error {
33 // Pseudo filesystems of the boot root. Their mountpoints exist on the exec
34 // disk (the root is read-only).
35 for _, m := range []struct{ src, dst, typ string }{
36 {"proc", "/proc", "proc"},
37 {"sysfs", "/sys", "sysfs"},
38 {"devtmpfs", "/dev", "devtmpfs"},
39 } {
40 // proc is mounted by Init (to read the command line) and devtmpfs by the
41 // kernel (DEVTMPFS_MOUNT): mount only what is missing.
42 if isMounted(m.dst) {
43 continue
44 }
45 if err := mount(m.src, m.dst, m.typ, 0, ""); err != nil {
46 return err
47 }
48 }
49 // Firecracker has no RTC on x86: the guest clock starts at the epoch, which
50 // breaks TLS. The host passes its time on the command line.
51 if v, ok := args[cmdlineTime]; ok {
52 if sec, err := strconv.ParseInt(v, 10, 64); err == nil && sec > 1_600_000_000 {
53 if err := unix.ClockSettime(unix.CLOCK_REALTIME, &unix.Timespec{Sec: sec}); err != nil {
54 log.Warn().Err(err).Msg("init: set clock")
55 }
56 }
57 }
58
59 if err := mount(devBase, "/mnt/base", "ext4", unix.MS_RDONLY, ""); err != nil {
60 return err
61 }
62 if err := mount(devScratch, "/mnt/scratch", "ext4", unix.MS_NOATIME, ""); err != nil {
63 return err
64 }
65 for _, d := range []string{"/mnt/scratch/upper", "/mnt/scratch/work", "/mnt/scratch/docker"} {
66 if err := os.MkdirAll(d, 0o755); err != nil {
67 return err
68 }
69 }
70 opts := "lowerdir=/mnt/base,upperdir=/mnt/scratch/upper,workdir=/mnt/scratch/work"
71 if err := mount("overlay", "/mnt/newroot", "overlay", 0, opts); err != nil {
72 return err
73 }
74 // The new root needs these mountpoints; they live in the (writable) upper.
75 for _, d := range []string{"/mnt/newroot/proc", "/mnt/newroot/sys", "/mnt/newroot/dev",
76 "/mnt/newroot/run", "/mnt/newroot/tmp", "/mnt/newroot/scratch", "/mnt/newroot/.oldroot",
77 "/mnt/newroot" + sandbox.GuestExecDir} {
78 if err := os.MkdirAll(d, 0o755); err != nil {
79 return err
80 }
81 }
82 // Scratch and the exec disk stay reachable after the pivot.
83 if err := mount("/mnt/scratch", "/mnt/newroot"+sandbox.GuestScratch, "", unix.MS_BIND, ""); err != nil {
84 return err
85 }
86 if err := mount("/", "/mnt/newroot"+sandbox.GuestExecDir, "", unix.MS_BIND, ""); err != nil {
87 return err
88 }
89 if err := mount("", "/mnt/newroot"+sandbox.GuestExecDir, "", unix.MS_BIND|unix.MS_REMOUNT|unix.MS_RDONLY, ""); err != nil {
90 return err
91 }
92 for _, d := range []string{"/proc", "/sys", "/dev"} {
93 if err := mount(d, "/mnt/newroot"+d, "", unix.MS_MOVE, ""); err != nil {
94 return err
95 }
96 }
97 if err := unix.Chdir("/mnt/newroot"); err != nil {
98 return err
99 }
100 if err := unix.PivotRoot(".", ".oldroot"); err != nil {
101 return fmt.Errorf("pivot_root: %w", err)
102 }
103 if err := unix.Chroot("."); err != nil {
104 return err
105 }
106 if err := unix.Chdir("/"); err != nil {
107 return err
108 }
109 _ = unix.Unmount("/.oldroot", unix.MNT_DETACH)
110
111 for _, m := range []struct {
112 src, dst, typ string
113 flags uintptr
114 data string
115 }{
116 {"devpts", "/dev/pts", "devpts", 0, "gid=5,mode=620,ptmxmode=666"},
117 {"tmpfs", "/dev/shm", "tmpfs", unix.MS_NOSUID | unix.MS_NODEV, ""},
118 {"tmpfs", "/run", "tmpfs", unix.MS_NOSUID | unix.MS_NODEV, "mode=755"},
119 {"tmpfs", "/tmp", "tmpfs", unix.MS_NOSUID | unix.MS_NODEV, "mode=1777"},
120 {"cgroup2", "/sys/fs/cgroup", "cgroup2", unix.MS_NOSUID | unix.MS_NODEV | unix.MS_NOEXEC, ""},
121 } {
122 if err := os.MkdirAll(m.dst, 0o755); err != nil {
123 return err
124 }
125 if err := mount(m.src, m.dst, m.typ, m.flags, m.data); err != nil {
126 // devpts and shm failures are not fatal for the stub.
127 log.Warn().Err(err).Str("target", m.dst).Msg("init: mount")
128 }
129 }
130 _ = os.Chmod("/tmp", 0o1777)
131
132 gw := args[sandbox.CmdlineGateway]
133 if err := writeResolvConf(gw); err != nil {
134 return err
135 }
136 _ = os.WriteFile("/etc/hostname", []byte("ermite-job\n"), 0o644)
137 _ = os.WriteFile("/etc/hosts", []byte("127.0.0.1 localhost ermite-job\n"), 0o644)
138 _ = unix.Sethostname([]byte("ermite-job"))
139 if err := loUp(); err != nil {
140 log.Warn().Err(err).Msg("init: loopback")
141 }
142 // IPv6 is disabled on the kernel command line (ipv6.disable=1); this is a
143 // second line of defence when the module is built in but not disabled.
144 _ = os.WriteFile("/proc/sys/net/ipv6/conf/all/disable_ipv6", []byte("1\n"), 0o644)
145 return nil
146}
147
148// writeResolvConf points the guest at the gateway, where the runner's DNS
149// forwarder listens. /etc/resolv.conf may be a dangling symlink in the image
150// (systemd-resolved stub): replace it.
151func writeResolvConf(gw string) error {
152 if gw == "" {
153 return fmt.Errorf("kernel command line has no %s", sandbox.CmdlineGateway)
154 }
155 _ = os.Remove("/etc/resolv.conf")
156 return os.WriteFile("/etc/resolv.conf", []byte("nameserver "+gw+"\noptions timeout:2 attempts:2\n"), 0o644)
157}
158
159func mount(src, dst, typ string, flags uintptr, data string) error {
160 if err := os.MkdirAll(dst, 0o755); err != nil && !os.IsExist(err) {
161 // Read-only boot root: the mountpoint must already exist.
162 if _, serr := os.Stat(dst); serr != nil {
163 return fmt.Errorf("mountpoint %s: %w", dst, err)
164 }
165 }
166 if err := unix.Mount(src, dst, typ, flags, data); err != nil {
167 return fmt.Errorf("mount %s on %s (%s): %w", src, dst, strings.TrimSpace(typ), err)
168 }
169 return nil
170}
171
172// loUp brings the loopback interface up (the kernel leaves it down).
173func loUp() error {
174 fd, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM|unix.SOCK_CLOEXEC, 0)
175 if err != nil {
176 return err
177 }
178 defer func() { _ = unix.Close(fd) }()
179 ifr, err := unix.NewIfreq("lo")
180 if err != nil {
181 return err
182 }
183 if err := unix.IoctlIfreq(fd, unix.SIOCGIFFLAGS, ifr); err != nil {
184 return err
185 }
186 ifr.SetUint16(ifr.Uint16() | unix.IFF_UP)
187 return unix.IoctlIfreq(fd, unix.SIOCSIFFLAGS, ifr)
188}
189
190// isMounted reports whether path is a mount point listed in /proc/self/mountinfo.
191func isMounted(path string) bool {
192 b, err := os.ReadFile("/proc/self/mountinfo")
193 if err != nil {
194 return false
195 }
196 for _, line := range strings.Split(string(b), "\n") {
197 f := strings.Fields(line)
198 if len(f) > 4 && f[4] == path {
199 return true
200 }
201 }
202 return false
203}