1// SPDX-License-Identifier: MIT
2
3package vm
4
5import (
6 "context"
7 "crypto/sha256"
8 "encoding/hex"
9 "fmt"
10 "io"
11 "io/fs"
12 "os"
13 "path/filepath"
14 "strconv"
15 "time"
16
17 "bnema.dev/ermite-runner/internal/cmdrun"
18 "bnema.dev/ermite-runner/internal/sandbox"
19)
20
21// ensureExecDisk builds (or reuses) the small read-only ext4 disk that carries
22// the runner's own executable: the guest root during boot, and /opt/ermite
23// afterwards. It is named after the executable's hash, so a runner upgrade
24// builds a new one and no image rebuild is needed; older ones are removed.
25// mkfs.ext4 -d needs no root.
26func ensureExecDisk(ctx context.Context, run cmdrun.Runner, exe, stateDir string) (string, error) {
27 sum, size, err := hashFile(exe)
28 if err != nil {
29 return "", fmt.Errorf("hash executable: %w", err)
30 }
31 cache := filepath.Join(stateDir, "cache")
32 if err := os.MkdirAll(cache, 0o755); err != nil {
33 return "", err
34 }
35 name := "execdisk-" + sum[:16] + ".ext4"
36 dest := filepath.Join(cache, name)
37 if _, err := os.Stat(dest); err == nil {
38 return dest, nil
39 }
40 // Drop disks of other executable versions.
41 if olds, _ := filepath.Glob(filepath.Join(cache, "execdisk-*")); len(olds) > 0 {
42 for _, o := range olds {
43 _ = os.Remove(o)
44 }
45 }
46
47 tmp, err := os.MkdirTemp(cache, "execroot-")
48 if err != nil {
49 return "", err
50 }
51 defer func() { _ = os.RemoveAll(tmp) }()
52 for _, d := range []string{"proc", "sys", "dev", "mnt/base", "mnt/scratch", "mnt/newroot"} {
53 if err := os.MkdirAll(filepath.Join(tmp, d), 0o755); err != nil {
54 return "", err
55 }
56 }
57 if err := copyFile(exe, filepath.Join(tmp, sandbox.GuestExecName)); err != nil {
58 return "", fmt.Errorf("copy executable: %w", err)
59 }
60 if err := os.Chmod(filepath.Join(tmp, sandbox.GuestExecName), 0o755); err != nil {
61 return "", err
62 }
63
64 part := dest + ".part"
65 _ = os.Remove(part)
66 f, err := os.OpenFile(part, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
67 if err != nil {
68 return "", err
69 }
70 // The executable, ext4 overhead and some room; the disk is read-only.
71 mib := size/(1<<20)*115/100 + 16
72 if err := f.Truncate(mib << 20); err != nil {
73 _ = f.Close()
74 return "", err
75 }
76 _ = f.Close()
77 if _, err := run.Run(ctx, nil, "mkfs.ext4", "-q", "-F", "-L", "ermite-exec",
78 "-O", "^has_journal", "-E", "root_owner=0:0", "-d", tmp, part); err != nil {
79 _ = os.Remove(part)
80 return "", fmt.Errorf("build exec disk: %w", err)
81 }
82 if err := os.Rename(part, dest); err != nil {
83 return "", err
84 }
85 return dest, nil
86}
87
88func hashFile(path string) (sum string, size int64, err error) {
89 f, err := os.Open(path)
90 if err != nil {
91 return "", 0, err
92 }
93 defer func() { _ = f.Close() }()
94 h := sha256.New()
95 n, err := io.Copy(h, f)
96 if err != nil {
97 return "", 0, err
98 }
99 return hex.EncodeToString(h.Sum(nil)), n, nil
100}
101
102// ensureExternalsDisk builds (or reuses) the read-only ext4 disk that carries
103// the host's external tools (Node.js): one directory per tool at its root, so
104// that mounted at /home/runner/externals it gives the GitHub layout
105// externals/node24/bin/node. It is named after the content of the set
106// (sandbox.ExternalsKey), so a new Node.js version builds a new disk and leaves
107// the exec disk and the base image alone. It returns "" for an empty set. The
108// caller holds the driver's externals lock, so that two jobs do not build the
109// same disk.
110func ensureExternalsDisk(ctx context.Context, run cmdrun.Runner, set []sandbox.External, stateDir string) (string, error) {
111 if len(set) == 0 {
112 return "", nil
113 }
114 cache := filepath.Join(stateDir, "cache")
115 if err := os.MkdirAll(cache, 0o755); err != nil {
116 return "", err
117 }
118 name := "extdisk-" + sandbox.ExternalsKey(set) + ".ext4"
119 dest := filepath.Join(cache, name)
120 if _, err := os.Stat(dest); err == nil {
121 return dest, nil
122 }
123 // Drop disks of other contents, once they are an hour old: without the
124 // jailer a VM that is about to start may still be about to open the previous
125 // one in place (with the jailer it holds a hard link).
126 olds, _ := filepath.Glob(filepath.Join(cache, "extdisk-*"))
127 for _, o := range olds {
128 if fi, err := os.Stat(o); err == nil && time.Since(fi.ModTime()) > time.Hour {
129 _ = os.Remove(o)
130 }
131 }
132
133 tmp, err := os.MkdirTemp(cache, "extroot-")
134 if err != nil {
135 return "", err
136 }
137 defer func() { _ = os.RemoveAll(tmp) }()
138 var size, files int64
139 for _, e := range set {
140 sz, n, err := cloneTree(e.Dir, filepath.Join(tmp, e.Name))
141 if err != nil {
142 return "", fmt.Errorf("stage %s: %w", e.Name, err)
143 }
144 size, files = size+sz, files+n
145 }
146
147 part := dest + ".part"
148 _ = os.Remove(part)
149 f, err := os.OpenFile(part, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
150 if err != nil {
151 return "", err
152 }
153 // The files, ext4 overhead (inodes, directories, block rounding of many
154 // small files) and some room; the disk is read-only and sparse.
155 mib := size/(1<<20)*125/100 + files*4096/(1<<20) + 16
156 if err := f.Truncate(mib << 20); err != nil {
157 _ = f.Close()
158 return "", err
159 }
160 _ = f.Close()
161 if _, err := run.Run(ctx, nil, "mkfs.ext4", "-q", "-F", "-L", "ermite-ext", "-m", "0",
162 "-O", "^has_journal", "-N", strconv.FormatInt(files*13/10+256, 10),
163 "-E", "root_owner=0:0", "-d", tmp, part); err != nil {
164 _ = os.Remove(part)
165 return "", fmt.Errorf("build externals disk: %w", err)
166 }
167 if err := os.Rename(part, dest); err != nil {
168 return "", err
169 }
170 return dest, nil
171}
172
173// cloneTree recreates the tree src at dst (directories, regular files, symbolic
174// links) for mkfs.ext4 -d, hard-linking files when the filesystem allows and
175// copying them otherwise. It returns the bytes of file content and the number
176// of entries.
177func cloneTree(src, dst string) (size, entries int64, err error) {
178 err = filepath.WalkDir(src, func(p string, d fs.DirEntry, werr error) error {
179 if werr != nil {
180 return werr
181 }
182 rel, err := filepath.Rel(src, p)
183 if err != nil {
184 return err
185 }
186 to := filepath.Join(dst, rel)
187 entries++
188 switch {
189 case d.IsDir():
190 if err := os.MkdirAll(to, 0o755); err != nil {
191 return err
192 }
193 return os.Chmod(to, 0o755) // whatever the umask
194 case d.Type()&fs.ModeSymlink != 0:
195 target, err := os.Readlink(p)
196 if err != nil {
197 return err
198 }
199 return os.Symlink(target, to)
200 case d.Type().IsRegular():
201 fi, err := d.Info()
202 if err != nil {
203 return err
204 }
205 size += fi.Size()
206 if err := os.Link(p, to); err == nil {
207 return nil
208 }
209 if err := copyFile(p, to); err != nil {
210 return err
211 }
212 return os.Chmod(to, fi.Mode().Perm())
213 }
214 return fmt.Errorf("%s: unsupported file type %v", p, d.Type())
215 })
216 return size, entries, err
217}