1// SPDX-License-Identifier: MIT
2
3package integration
4
5import (
6 "context"
7 "net/http"
8 "os"
9 "path/filepath"
10 "strings"
11 "syscall"
12 "testing"
13 "time"
14
15 "bnema.dev/ermite-runner/internal/execstream"
16 "bnema.dev/ermite-runner/internal/execstream/execpb"
17 "bnema.dev/ermite-runner/internal/externals"
18 "bnema.dev/ermite-runner/internal/sandbox"
19 "bnema.dev/ermite-runner/internal/sandbox/drivers"
20)
21
22func TestVMStubExecutor(t *testing.T) {
23 if os.Getenv("ERMITE_RUNNER_IT_VM") != "1" {
24 t.Skip("set ERMITE_RUNNER_IT_VM=1 (needs KVM, firecracker, a kernel and a base image; see README)")
25 }
26 need(t, "mkfs.ext4")
27 needNetAdmin(t)
28 ctx := ctxWithLog(t)
29 cfg := loadConfig(t, sandbox.VM)
30 if why := sandbox.VMUnavailable(cfg, sandbox.Prober{}); len(why) > 0 {
31 t.Fatalf("vm isolation is not available here: %s", strings.Join(why, "; "))
32 }
33
34 scenario(t, ctx, cfg, sandbox.VM, func(t *testing.T, sb sandbox.Sandbox) {
35 // Kill every VMM process of ours (found like Reap finds them: by id).
36 pids := vmmPIDs()
37 if len(pids) == 0 {
38 t.Fatal("no VMM process to kill")
39 }
40 for _, pid := range pids {
41 _ = syscall.Kill(pid, syscall.SIGKILL)
42 }
43 })
44}
45
46// The guest must reach DNS on its gateway and nothing else on the host.
47func TestVMNetworkPolicy(t *testing.T) {
48 if os.Getenv("ERMITE_RUNNER_IT_VM") != "1" {
49 t.Skip("set ERMITE_RUNNER_IT_VM=1")
50 }
51 need(t, "mkfs.ext4")
52 needNetAdmin(t)
53 ctx := ctxWithLog(t)
54 cfg := loadConfig(t, sandbox.VM)
55 drv, err := sandbox.Select(ctx, cfg, drivers.Factories(), sandbox.Prober{})
56 if err != nil {
57 t.Fatal(err)
58 }
59 defer func() { _ = drv.Close() }()
60 _ = drv.Reap(ctx)
61 sb, err := drv.New()
62 if err != nil {
63 t.Fatal(err)
64 }
65 defer func() { _ = sb.Stop(context.Background()) }()
66 conn, err := sb.Start(ctx, sandbox.JobSpec{JobID: "netpolicy"})
67 if err != nil {
68 t.Fatal(err)
69 }
70 // 10.200.0.1 is the gateway of slot 0.
71 script := `
72probe() { timeout 4 bash -c "echo > /dev/tcp/$1/$2" >/dev/null 2>&1; echo "$1:$2 rc=$?"; }
73probe 10.200.0.1 22 # host SSH-like port: must be DROPPED (timeout, rc=124), not refused
74probe 10.200.0.1 8080
75probe 169.254.169.254 80 # cloud metadata
76probe 100.64.0.1 80 # CGNAT / VPN overlay
77probe 192.168.0.1 80 # RFC 1918
78probe 172.16.0.1 80
79probe 10.0.0.1 80
80python3 - <<'PY'
81import socket
82q=bytes.fromhex("abcd01000001000000000000076578616d706c6503636f6d0000010001")
83s=socket.socket(socket.AF_INET,socket.SOCK_DGRAM); s.settimeout(8); s.sendto(q,("10.200.0.1",53))
84try:
85 r=s.recv(512); print("dns answered", len(r))
86except Exception as e: print("dns silent", e)
87PY
88ip -6 addr show scope global | grep -c inet6 || true
89`
90 r, err := runJob(ctx, conn, script)
91 if err != nil {
92 t.Fatal(err)
93 }
94 t.Log("\n" + r.out)
95 for _, line := range strings.Split(r.out, "\n") {
96 if !strings.Contains(line, " rc=") {
97 continue
98 }
99 if strings.HasSuffix(line, "rc=0") {
100 t.Errorf("a guest reached a forbidden destination: %s", line)
101 }
102 }
103 if !strings.Contains(r.out, "10.200.0.1:22 rc=124") {
104 t.Errorf("host port must be dropped (timeout), got:\n%s", r.out)
105 }
106 if !strings.Contains(r.out, "dns answered") {
107 t.Errorf("the DNS forwarder on the gateway must answer:\n%s", r.out)
108 }
109}
110
111// TestVMRunsAJob drives a job (not the stub script) through the executor in a
112// micro-VM: the runner user, the work layout, file commands, outputs, a failing
113// step and cancellation by SIGINT. The VM is rootful-free like the other VM
114// tests (user + network namespace, jailer off).
115func TestVMRunsAJob(t *testing.T) {
116 if os.Getenv("ERMITE_RUNNER_IT_VM") != "1" {
117 t.Skip("set ERMITE_RUNNER_IT_VM=1 (needs KVM, firecracker, a kernel and a base image; see README)")
118 }
119 need(t, "mkfs.ext4")
120 needNetAdmin(t)
121 ctx := ctxWithLog(t)
122 cfg := loadConfig(t, sandbox.VM)
123 if why := sandbox.VMUnavailable(cfg, sandbox.Prober{}); len(why) > 0 {
124 t.Fatalf("vm isolation is not available here: %s", strings.Join(why, "; "))
125 }
126 drv, err := sandbox.Select(ctx, cfg, drivers.Factories(), sandbox.Prober{})
127 if err != nil {
128 t.Fatal(err)
129 }
130 t.Cleanup(func() {
131 _ = drv.Close()
132 if d2, err := sandbox.Select(context.Background(), cfg, drivers.Factories(), sandbox.Prober{}); err == nil {
133 _ = d2.Reap(context.Background())
134 _ = d2.Close()
135 }
136 })
137 if err := drv.Reap(ctx); err != nil {
138 t.Fatal(err)
139 }
140
141 run := func(yaml string, cancelAfter time.Duration) (steps []string, logs []string, res *execpb.JobResult, exit *execpb.Exit) {
142 t.Helper()
143 sb, err := drv.New()
144 if err != nil {
145 t.Fatal(err)
146 }
147 defer func() {
148 if err := sb.Stop(ctx); err != nil {
149 t.Errorf("stop: %v", err)
150 }
151 }()
152 conn, err := sb.Start(ctx, sandbox.JobSpec{JobID: "vm-job"})
153 if err != nil {
154 t.Fatal(err)
155 }
156 var cancel chan *execpb.Cancel
157 if cancelAfter > 0 {
158 cancel = make(chan *execpb.Cancel, 1)
159 time.AfterFunc(cancelAfter, func() { cancel <- &execpb.Cancel{Reason: "test"} })
160 }
161 _, exit, err = execstream.DriveJob(conn, &execpb.Run{Job: &execpb.JobSpec{
162 WorkflowPath: ".github/workflows/ci.yml", WorkflowYaml: yaml, JobKey: "j", Repo: "hello", RunnerName: "it", RunnerVersion: "test",
163 RunnerEnvironment: "self-hosted", Matrix: "{}",
164 Contexts: `{"github":{"repository":"octo/hello","event_name":"push","job":"j","ref":"refs/heads/main","sha":"0123456789abcdef0123456789abcdef01234567"}}`,
165 }}, cancel, execstream.JobHandler{
166 StepStarted: func(s *execpb.StepStarted) { steps = append(steps, s.Name) },
167 Log: func(l *execpb.Log) { logs = append(logs, l.Text) },
168 JobResult: func(r *execpb.JobResult) { res = r },
169 })
170 if err != nil {
171 t.Fatalf("driving the job: %v", err)
172 }
173 return
174 }
175
176 _, logs, res, exit := run(`on: push
177jobs:
178 j:
179 runs-on: x
180 outputs:
181 who: ${{ steps.a.outputs.who }}
182 steps:
183 - id: a
184 run: |
185 echo "who=$(id -un):$(id -u)" >> "$GITHUB_OUTPUT"
186 echo "pwd=$PWD home=$HOME"
187 echo "FOO=bar" >> "$GITHUB_ENV"
188 - run: echo "FOO=$FOO CI=$CI"
189 - run: exit 3
190 - if: always()
191 run: echo still-here
192`, 0)
193 all := strings.Join(logs, "\n")
194 if res == nil || res.Conclusion != execpb.StepConclusion_STEP_CONCLUSION_FAILURE || exit.Code == 0 {
195 t.Fatalf("result %v exit %v\n%s", res, exit, all)
196 }
197 if res.Outputs["who"] != "runner:1001" {
198 t.Errorf("steps ran as %q\n%s", res.Outputs["who"], all)
199 }
200 for _, w := range []string{"pwd=/home/runner/work/hello/hello home=/home/runner", "FOO=bar CI=true", "##[error]Process completed with exit code 3.", "still-here"} {
201 if !strings.Contains(all, w) {
202 t.Errorf("log lacks %q\n%s", w, all)
203 }
204 }
205
206 start := time.Now()
207 _, logs, res, _ = run("on: push\njobs:\n j:\n runs-on: x\n steps:\n - run: |\n trap 'echo got-int; exit 130' INT\n echo ready\n while :; do sleep 0.2; done\n", 8*time.Second)
208 all = strings.Join(logs, "\n")
209 if res == nil || res.Conclusion != execpb.StepConclusion_STEP_CONCLUSION_CANCELLED || !strings.Contains(all, "got-int") {
210 t.Fatalf("cancel: %v\n%s", res, all)
211 }
212 t.Logf("cancelled job ended %v after the start", time.Since(start))
213}
214
215// nodeCache serves the pinned Node.js archives from a local directory
216// (ERMITE_RUNNER_IT_NODE_DIR holding node-v24.21.0-linux-x64.tar.gz, say): the
217// sandbox of the VM tests has no internet, and this way the real archive is
218// still checked against the pinned SHA-256 and unpacked by the real code.
219type nodeCache string
220
221func (d nodeCache) RoundTrip(r *http.Request) (*http.Response, error) {
222 f, err := os.Open(filepath.Join(string(d), filepath.Base(r.URL.Path)))
223 if err != nil {
224 return &http.Response{StatusCode: http.StatusNotFound, Status: "404 Not Found", Body: http.NoBody, Request: r}, nil
225 }
226 return &http.Response{StatusCode: http.StatusOK, Status: "200 OK", Body: f, Request: r}, nil
227}
228
229// TestVMNodeExternals runs the pinned Node.js 24, provided by the host outside
230// the image, inside a job's micro-VM: the externals disk is the fourth drive,
231// mounted read-only at /home/runner/externals, in GitHub's layout.
232//
233// Needs the archive of the pinned node24 for this architecture, either in
234// ERMITE_RUNNER_IT_NODE_DIR (offline) or downloadable from nodejs.org.
235func TestVMNodeExternals(t *testing.T) {
236 if os.Getenv("ERMITE_RUNNER_IT_VM") != "1" {
237 t.Skip("set ERMITE_RUNNER_IT_VM=1 (needs KVM, firecracker, a kernel and a base image; see README)")
238 }
239 need(t, "mkfs.ext4")
240 needNetAdmin(t)
241 ctx := ctxWithLog(t)
242 cfg := loadConfig(t, sandbox.VM)
243 if why := sandbox.VMUnavailable(cfg, sandbox.Prober{}); len(why) > 0 {
244 t.Fatalf("vm isolation is not available here: %s", strings.Join(why, "; "))
245 }
246
247 m := &externals.Manager{Dir: filepath.Join(cfg.StateDir, "externals"), Arch: externals.HostArch()}
248 for _, a := range externals.ForArch(externals.Pinned, m.Arch) {
249 if a.Name == "node24" {
250 m.Table = append(m.Table, a)
251 }
252 }
253 if len(m.Table) != 1 {
254 t.Skipf("no pinned node24 for %q", m.Arch)
255 }
256 if dir := os.Getenv("ERMITE_RUNNER_IT_NODE_DIR"); dir != "" {
257 m.Client = &http.Client{Transport: nodeCache(dir)}
258 }
259 if err := m.Sync(ctx); err != nil {
260 t.Skipf("cannot get the pinned Node.js (set ERMITE_RUNNER_IT_NODE_DIR to a directory with %s): %v", filepath.Base(m.Table[0].URL), err)
261 }
262 cfg.Externals = m
263
264 drv, err := sandbox.Select(ctx, cfg, drivers.Factories(), sandbox.Prober{})
265 if err != nil {
266 t.Fatal(err)
267 }
268 t.Cleanup(func() {
269 _ = drv.Close()
270 if d2, err := sandbox.Select(context.Background(), cfg, drivers.Factories(), sandbox.Prober{}); err == nil {
271 _ = d2.Reap(context.Background())
272 _ = d2.Close()
273 }
274 })
275 if err := drv.Reap(ctx); err != nil {
276 t.Fatal(err)
277 }
278 sb, err := drv.New()
279 if err != nil {
280 t.Fatal(err)
281 }
282 defer func() { _ = sb.Stop(context.Background()) }()
283 conn, err := sb.Start(ctx, sandbox.JobSpec{JobID: "node-externals"})
284 if err != nil {
285 t.Fatal(err)
286 }
287 r, err := runJob(ctx, conn, `
288N=/home/runner/externals/node24/bin/node
289$N -e 'console.log(process.version)'
290echo "has-node24: $(test -x /home/runner/externals/node24/bin/node && echo yes)"
291touch /home/runner/externals/node24/probe 2>/dev/null && echo "WRITABLE" || echo "read-only"
292grep ' /home/runner/externals ' /proc/mounts
293`)
294 if err != nil {
295 t.Fatal(err)
296 }
297 t.Log("\n" + r.out + r.errOut)
298 want := m.Table[0].Version
299 for _, w := range []string{"v" + want + "\n", "has-node24: yes\n", "read-only\n", "/dev/vdd /home/runner/externals ext4 ro"} {
300 if !strings.Contains(r.out, w) {
301 t.Errorf("output lacks %q", w)
302 }
303 }
304 if strings.Contains(r.out, "WRITABLE") {
305 t.Error("the externals are writable from the guest")
306 }
307}
308
309// TestVMBinfmtMisc checks what docker/setup-qemu-action needs from the guest:
310// binfmt_misc mounted at boot, as on GitHub's image, and a handler registered
311// with the F (fix binary) flag that tonistiigi/binfmt uses. The handler here is
312// cat on an extension, so the test needs neither QEMU nor the network.
313func TestVMBinfmtMisc(t *testing.T) {
314 if os.Getenv("ERMITE_RUNNER_IT_VM") != "1" {
315 t.Skip("set ERMITE_RUNNER_IT_VM=1")
316 }
317 need(t, "mkfs.ext4")
318 needNetAdmin(t)
319 ctx := ctxWithLog(t)
320 cfg := loadConfig(t, sandbox.VM)
321 drv, err := sandbox.Select(ctx, cfg, drivers.Factories(), sandbox.Prober{})
322 if err != nil {
323 t.Fatal(err)
324 }
325 defer func() { _ = drv.Close() }()
326 _ = drv.Reap(ctx)
327 sb, err := drv.New()
328 if err != nil {
329 t.Fatal(err)
330 }
331 defer func() { _ = sb.Stop(context.Background()) }()
332 conn, err := sb.Start(ctx, sandbox.JobSpec{JobID: "binfmt"})
333 if err != nil {
334 t.Fatal(err)
335 }
336 r, err := runJob(ctx, conn, `
337S=; [ "$(id -u)" = 0 ] || S="sudo -n"
338grep ' /proc/sys/fs/binfmt_misc ' /proc/mounts
339echo "status: $(cat /proc/sys/fs/binfmt_misc/status)"
340echo ':ermitetest:E::ermitetest::/usr/bin/cat:F' | $S tee /proc/sys/fs/binfmt_misc/register >/dev/null
341grep -x 'flags: F' /proc/sys/fs/binfmt_misc/ermitetest
342printf 'ran through binfmt_misc\n' > /tmp/probe.ermitetest
343chmod +x /tmp/probe.ermitetest
344/tmp/probe.ermitetest
345`)
346 if err != nil {
347 t.Fatal(err)
348 }
349 t.Log("\n" + r.out + r.errOut)
350 for _, w := range []string{"binfmt_misc /proc/sys/fs/binfmt_misc binfmt_misc", "status: enabled\n", "flags: F\n", "ran through binfmt_misc\n"} {
351 if !strings.Contains(r.out, w) {
352 t.Errorf("output lacks %q", w)
353 }
354 }
355}