1// SPDX-License-Identifier: MIT
2
3package integration
4
5import (
6 "context"
7 "os"
8 "strings"
9 "syscall"
10 "testing"
11 "time"
12
13 "bnema.dev/ermite-runner/internal/execstream"
14 "bnema.dev/ermite-runner/internal/execstream/execpb"
15 "bnema.dev/ermite-runner/internal/sandbox"
16 "bnema.dev/ermite-runner/internal/sandbox/drivers"
17)
18
19func TestVMStubExecutor(t *testing.T) {
20 if os.Getenv("ERMITE_RUNNER_IT_VM") != "1" {
21 t.Skip("set ERMITE_RUNNER_IT_VM=1 (needs KVM, firecracker, a kernel and a base image; see README)")
22 }
23 need(t, "mkfs.ext4")
24 needNetAdmin(t)
25 ctx := ctxWithLog(t)
26 cfg := loadConfig(t, sandbox.VM)
27 if why := sandbox.VMUnavailable(cfg, sandbox.Prober{}); len(why) > 0 {
28 t.Fatalf("vm isolation is not available here: %s", strings.Join(why, "; "))
29 }
30
31 scenario(t, ctx, cfg, sandbox.VM, func(t *testing.T, sb sandbox.Sandbox) {
32 // Kill every VMM process of ours (found like Reap finds them: by id).
33 pids := vmmPIDs()
34 if len(pids) == 0 {
35 t.Fatal("no VMM process to kill")
36 }
37 for _, pid := range pids {
38 _ = syscall.Kill(pid, syscall.SIGKILL)
39 }
40 })
41}
42
43// The guest must reach DNS on its gateway and nothing else on the host.
44func TestVMNetworkPolicy(t *testing.T) {
45 if os.Getenv("ERMITE_RUNNER_IT_VM") != "1" {
46 t.Skip("set ERMITE_RUNNER_IT_VM=1")
47 }
48 need(t, "mkfs.ext4")
49 needNetAdmin(t)
50 ctx := ctxWithLog(t)
51 cfg := loadConfig(t, sandbox.VM)
52 drv, err := sandbox.Select(ctx, cfg, drivers.Factories(), sandbox.Prober{})
53 if err != nil {
54 t.Fatal(err)
55 }
56 defer func() { _ = drv.Close() }()
57 _ = drv.Reap(ctx)
58 sb, err := drv.New()
59 if err != nil {
60 t.Fatal(err)
61 }
62 defer func() { _ = sb.Stop(context.Background()) }()
63 conn, err := sb.Start(ctx, sandbox.JobSpec{JobID: "netpolicy"})
64 if err != nil {
65 t.Fatal(err)
66 }
67 // 10.200.0.1 is the gateway of slot 0.
68 script := `
69probe() { timeout 4 bash -c "echo > /dev/tcp/$1/$2" >/dev/null 2>&1; echo "$1:$2 rc=$?"; }
70probe 10.200.0.1 22 # host SSH-like port: must be DROPPED (timeout, rc=124), not refused
71probe 10.200.0.1 8080
72probe 169.254.169.254 80 # cloud metadata
73probe 100.64.0.1 80 # CGNAT / VPN overlay
74probe 192.168.0.1 80 # RFC 1918
75probe 172.16.0.1 80
76probe 10.0.0.1 80
77python3 - <<'PY'
78import socket
79q=bytes.fromhex("abcd01000001000000000000076578616d706c6503636f6d0000010001")
80s=socket.socket(socket.AF_INET,socket.SOCK_DGRAM); s.settimeout(8); s.sendto(q,("10.200.0.1",53))
81try:
82 r=s.recv(512); print("dns answered", len(r))
83except Exception as e: print("dns silent", e)
84PY
85ip -6 addr show scope global | grep -c inet6 || true
86`
87 r, err := runJob(ctx, conn, script)
88 if err != nil {
89 t.Fatal(err)
90 }
91 t.Log("\n" + r.out)
92 for _, line := range strings.Split(r.out, "\n") {
93 if !strings.Contains(line, " rc=") {
94 continue
95 }
96 if strings.HasSuffix(line, "rc=0") {
97 t.Errorf("a guest reached a forbidden destination: %s", line)
98 }
99 }
100 if !strings.Contains(r.out, "10.200.0.1:22 rc=124") {
101 t.Errorf("host port must be dropped (timeout), got:\n%s", r.out)
102 }
103 if !strings.Contains(r.out, "dns answered") {
104 t.Errorf("the DNS forwarder on the gateway must answer:\n%s", r.out)
105 }
106}
107
108// TestVMRunsAJob drives a job (not the stub script) through the executor in a
109// micro-VM: the runner user, the work layout, file commands, outputs, a failing
110// step and cancellation by SIGINT. The VM is rootful-free like the other VM
111// tests (user + network namespace, jailer off).
112func TestVMRunsAJob(t *testing.T) {
113 if os.Getenv("ERMITE_RUNNER_IT_VM") != "1" {
114 t.Skip("set ERMITE_RUNNER_IT_VM=1 (needs KVM, firecracker, a kernel and a base image; see README)")
115 }
116 need(t, "mkfs.ext4")
117 needNetAdmin(t)
118 ctx := ctxWithLog(t)
119 cfg := loadConfig(t, sandbox.VM)
120 if why := sandbox.VMUnavailable(cfg, sandbox.Prober{}); len(why) > 0 {
121 t.Fatalf("vm isolation is not available here: %s", strings.Join(why, "; "))
122 }
123 drv, err := sandbox.Select(ctx, cfg, drivers.Factories(), sandbox.Prober{})
124 if err != nil {
125 t.Fatal(err)
126 }
127 t.Cleanup(func() {
128 _ = drv.Close()
129 if d2, err := sandbox.Select(context.Background(), cfg, drivers.Factories(), sandbox.Prober{}); err == nil {
130 _ = d2.Reap(context.Background())
131 _ = d2.Close()
132 }
133 })
134 if err := drv.Reap(ctx); err != nil {
135 t.Fatal(err)
136 }
137
138 run := func(yaml string, cancelAfter time.Duration) (steps []string, logs []string, res *execpb.JobResult, exit *execpb.Exit) {
139 t.Helper()
140 sb, err := drv.New()
141 if err != nil {
142 t.Fatal(err)
143 }
144 defer func() {
145 if err := sb.Stop(ctx); err != nil {
146 t.Errorf("stop: %v", err)
147 }
148 }()
149 conn, err := sb.Start(ctx, sandbox.JobSpec{JobID: "vm-job"})
150 if err != nil {
151 t.Fatal(err)
152 }
153 var cancel chan *execpb.Cancel
154 if cancelAfter > 0 {
155 cancel = make(chan *execpb.Cancel, 1)
156 time.AfterFunc(cancelAfter, func() { cancel <- &execpb.Cancel{Reason: "test"} })
157 }
158 _, exit, err = execstream.DriveJob(conn, &execpb.Run{Job: &execpb.JobSpec{
159 WorkflowPath: ".github/workflows/ci.yml", WorkflowYaml: yaml, JobKey: "j", Repo: "hello", RunnerName: "it", RunnerVersion: "test",
160 RunnerEnvironment: "self-hosted", Matrix: "{}",
161 Contexts: `{"github":{"repository":"octo/hello","event_name":"push","job":"j","ref":"refs/heads/main","sha":"0123456789abcdef0123456789abcdef01234567"}}`,
162 }}, cancel, execstream.JobHandler{
163 StepStarted: func(s *execpb.StepStarted) { steps = append(steps, s.Name) },
164 Log: func(l *execpb.Log) { logs = append(logs, l.Text) },
165 JobResult: func(r *execpb.JobResult) { res = r },
166 })
167 if err != nil {
168 t.Fatalf("driving the job: %v", err)
169 }
170 return
171 }
172
173 _, logs, res, exit := run(`on: push
174jobs:
175 j:
176 runs-on: x
177 outputs:
178 who: ${{ steps.a.outputs.who }}
179 steps:
180 - id: a
181 run: |
182 echo "who=$(id -un):$(id -u)" >> "$GITHUB_OUTPUT"
183 echo "pwd=$PWD home=$HOME"
184 echo "FOO=bar" >> "$GITHUB_ENV"
185 - run: echo "FOO=$FOO CI=$CI"
186 - run: exit 3
187 - if: always()
188 run: echo still-here
189`, 0)
190 all := strings.Join(logs, "\n")
191 if res == nil || res.Conclusion != execpb.StepConclusion_STEP_CONCLUSION_FAILURE || exit.Code == 0 {
192 t.Fatalf("result %v exit %v\n%s", res, exit, all)
193 }
194 if res.Outputs["who"] != "runner:1001" {
195 t.Errorf("steps ran as %q\n%s", res.Outputs["who"], all)
196 }
197 for _, w := range []string{"pwd=/home/runner/work/hello/hello home=/home/runner", "FOO=bar CI=true", "##[error]Process completed with exit code 3.", "still-here"} {
198 if !strings.Contains(all, w) {
199 t.Errorf("log lacks %q\n%s", w, all)
200 }
201 }
202
203 start := time.Now()
204 _, logs, res, _ = run("on: push\njobs:\n j:\n runs-on: x\n steps:\n - run: |\n trap 'echo got-int; exit 130' INT\n echo ready\n while :; do sleep 0.2; done\n", 8*time.Second)
205 all = strings.Join(logs, "\n")
206 if res == nil || res.Conclusion != execpb.StepConclusion_STEP_CONCLUSION_CANCELLED || !strings.Contains(all, "got-int") {
207 t.Fatalf("cancel: %v\n%s", res, all)
208 }
209 t.Logf("cancelled job ended %v after the start", time.Since(start))
210}