1// SPDX-License-Identifier: MIT
2
3// Package e2e runs jobs through the real executor in a real sandbox. The tests
4// are opt-in because they need Docker (and, for the VM, KVM and Firecracker):
5//
6// ERMITE_RUNNER_IT_E2E=1 ERMITE_RUNNER_TEST_IMAGE=ermite-runner-ubuntu-24.04:dev go test ./internal/e2e
7//
8// On a workstation with a rootless Docker daemon the container driver is used
9// with UnsafeAllowRootlessForTests (a test-only option): the jobs run as in production except
10// that the nested dockerd of the image cannot start, which these tests do not
11// need. The production paths that need root (jailer, rootful Docker, sysbox)
12// are not covered here.
13package e2e
14
15import (
16 "context"
17 "fmt"
18 "io"
19 "os"
20 "os/exec"
21 "path/filepath"
22 "strings"
23 "sync"
24 "testing"
25 "time"
26
27 "bnema.dev/ermite-runner/internal/execstream"
28 "bnema.dev/ermite-runner/internal/execstream/execpb"
29 "bnema.dev/ermite-runner/internal/sandbox"
30 "bnema.dev/ermite-runner/internal/sandbox/container"
31 "bnema.dev/ermite-runner/internal/sandbox/disk"
32 "bnema.dev/ermite-runner/internal/sandbox/sandboxtest"
33)
34
35func needE2E(t *testing.T) string {
36 t.Helper()
37 if os.Getenv("ERMITE_RUNNER_IT_E2E") != "1" {
38 t.Skip("set ERMITE_RUNNER_IT_E2E=1 and ERMITE_RUNNER_TEST_IMAGE (needs Docker; see the package comment)")
39 }
40 image := os.Getenv("ERMITE_RUNNER_TEST_IMAGE")
41 if image == "" {
42 t.Fatal("set ERMITE_RUNNER_TEST_IMAGE to the runner image")
43 }
44 if _, err := exec.LookPath("docker"); err != nil {
45 t.Skip("docker CLI not found")
46 }
47 return image
48}
49
50// buildRunner compiles the static runner binary that is attached to the sandbox.
51func buildRunner(t *testing.T, dir string) string {
52 t.Helper()
53 exe := filepath.Join(dir, "ermite-runner")
54 root, err := exec.Command("go", "list", "-m", "-f", "{{.Dir}}").Output()
55 if err != nil {
56 t.Fatal(err)
57 }
58 cmd := exec.Command("go", "build", "-o", exe, "bnema.dev/ermite-runner")
59 cmd.Env = append(os.Environ(), "CGO_ENABLED=0")
60 cmd.Dir = strings.TrimSpace(string(root))
61 if out, err := cmd.CombinedOutput(); err != nil {
62 t.Fatalf("build the runner: %v\n%s", err, out)
63 }
64 return exe
65}
66
67// newDriver starts the container driver with test-only settings: no firewall
68// plumbing, no loop mounts, a rootless daemon allowed.
69func newDriver(t *testing.T, image string) sandbox.Driver {
70 t.Helper()
71 dir := t.TempDir()
72 cfg := sandbox.Config{
73 Isolation: sandbox.Container, StateDir: filepath.Join(dir, "state"), Capacity: 2, Image: image,
74 JobCPUs: 2, JobMemoryMiB: 1024, JobDiskMiB: 2048, UIDBase: 900000,
75 Network: netPrefix("10.214.0.0/16"), Executable: buildRunner(t, dir), DNSPort: 0,
76 LockFile: filepath.Join(dir, "host.lock"), JobPids: 4096,
77 }
78 empty := filepath.Join(dir, "mountinfo")
79 _ = os.WriteFile(empty, nil, 0o644)
80 run := rootlessMountRunner{image: image, mountinfo: empty, mu: &sync.Mutex{}}
81 d, err := container.New(context.Background(), cfg, container.Options{
82 Net: sandboxtest.NewPlumbing(), UnsafeAllowRootlessForTests: true, ConnectTimeout: 90 * time.Second, Run: run,
83 Mounter: &disk.Mounter{Run: run, Mountinfo: empty},
84 })
85 if err != nil {
86 t.Fatal(err)
87 }
88 t.Cleanup(func() { _ = d.Close() })
89 if err := d.Reap(context.Background()); err != nil {
90 t.Fatal(err)
91 }
92 return d
93}
94
95// rootlessMountRunner stands in for mount and umount, which need root. The
96// scratch directory stays a plain directory; "mount" records it in a fake
97// mountinfo file so that the driver believes it is mounted and calls "umount"
98// at the end. On a rootless daemon the files that the runner user (uid 1001 in
99// the container) creates belong to a subordinate uid that the host user cannot
100// delete, so "umount" empties the directory from inside a container, where
101// root can. With the real loop mount the scratch image is simply unmounted and
102// deleted.
103type rootlessMountRunner struct {
104 image string
105 mountinfo string
106 mu *sync.Mutex
107}
108
109func (r rootlessMountRunner) Run(ctx context.Context, in io.Reader, name string, args ...string) ([]byte, error) {
110 switch name {
111 case "mount":
112 r.mu.Lock()
113 defer r.mu.Unlock()
114 f, err := os.OpenFile(r.mountinfo, os.O_APPEND|os.O_WRONLY, 0o644)
115 if err != nil {
116 return nil, err
117 }
118 defer func() { _ = f.Close() }()
119 _, err = fmt.Fprintf(f, "1 1 0:1 / %s rw - ext4 fake rw\n", args[len(args)-1])
120 return nil, err
121 case "umount":
122 dir := args[len(args)-1]
123 out, err := exec.CommandContext(ctx, "docker", "run", "--rm", "--entrypoint", "sh", "-v", dir+":/d", r.image,
124 "-c", "rm -rf /d/* /d/.[!.]* 2>/dev/null; true").CombinedOutput()
125 r.mu.Lock()
126 defer r.mu.Unlock()
127 b, _ := os.ReadFile(r.mountinfo)
128 var keep []string
129 for _, l := range strings.Split(string(b), "\n") {
130 if l != "" && !strings.Contains(l, " "+dir+" ") {
131 keep = append(keep, l)
132 }
133 }
134 _ = os.WriteFile(r.mountinfo, []byte(strings.Join(keep, "\n")+"\n"), 0o644)
135 return out, err
136 }
137 return sandboxtest.NoMountRunner{}.Run(ctx, in, name, args...)
138}
139
140// session is the outcome of one job driven through the stream.
141type session struct {
142 Steps []string
143 Concl map[string]execpb.StepConclusion
144 Logs []string // text of every log line, in order
145 LogStep []int32
146 Result *execpb.JobResult
147 Exit *execpb.Exit
148}
149
150// runJob starts a sandbox, drives the job and stops the sandbox.
151func runJob(t *testing.T, d sandbox.Driver, spec *execpb.JobSpec, cancel <-chan *execpb.Cancel) *session {
152 t.Helper()
153 sb, err := d.New()
154 if err != nil {
155 t.Fatal(err)
156 }
157 defer func() {
158 if err := sb.Stop(context.Background()); err != nil {
159 t.Errorf("stopping the sandbox: %v", err)
160 }
161 }()
162 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "e2e"})
163 if err != nil {
164 t.Fatal(err)
165 }
166 s := &session{Concl: map[string]execpb.StepConclusion{}}
167 _, exit, err := execstream.DriveJob(conn, &execpb.Run{Job: spec}, cancel, execstream.JobHandler{
168 StepStarted: func(x *execpb.StepStarted) { s.Steps = append(s.Steps, x.Name) },
169 StepCompleted: func(x *execpb.StepCompleted) {
170 s.Concl[x.Name] = x.Conclusion
171 },
172 Log: func(l *execpb.Log) { s.Logs = append(s.Logs, l.Text); s.LogStep = append(s.LogStep, l.Step) },
173 JobResult: func(r *execpb.JobResult) { s.Result = r },
174 })
175 if err != nil {
176 t.Fatalf("driving the job: %v", err)
177 }
178 s.Exit = exit
179 return s
180}
181
182func jobSpec(yaml string) *execpb.JobSpec {
183 return &execpb.JobSpec{
184 WorkflowPath: ".github/workflows/ci.yml", WorkflowYaml: yaml, JobKey: "j", Repo: "hello",
185 Contexts: `{"github":{"repository":"octo/hello","repository_owner":"octo","event_name":"push","job":"j","ref":"refs/heads/main","ref_name":"main","ref_type":"branch","sha":"0123456789abcdef0123456789abcdef01234567","run_id":"1","run_number":"1","run_attempt":"1","server_url":"https://ermite.example","workflow":"w","actor":"octo","event":{"ref":"refs/heads/main"}}}`,
186 Matrix: "{}", RunnerName: "e2e", RunnerVersion: "test", RunnerEnvironment: "self-hosted", TimeoutMinutes: 10,
187 }
188}
189
190func TestContainerRunsAJobAsTheRunnerUser(t *testing.T) {
191 image := needE2E(t)
192 d := newDriver(t, image)
193 s := runJob(t, d, jobSpec(`on: push
194jobs:
195 j:
196 runs-on: x
197 outputs:
198 who: ${{ steps.id.outputs.who }}
199 steps:
200 - id: id
201 run: |
202 echo "who=$(id -un):$(id -u):$(id -g)" >> "$GITHUB_OUTPUT"
203 echo "home=$HOME pwd=$PWD"
204 echo "ws=$GITHUB_WORKSPACE temp=$RUNNER_TEMP tool=$RUNNER_TOOL_CACHE"
205 touch "$RUNNER_TEMP/t" "$GITHUB_WORKSPACE/w" "$HOME/h"
206 echo "script=$0"
207 - name: env and path
208 run: |
209 mkdir -p "$RUNNER_TEMP/bin" && printf '#!/bin/sh\necho from-path\n' > "$RUNNER_TEMP/bin/mytool" && chmod +x "$RUNNER_TEMP/bin/mytool"
210 echo "$RUNNER_TEMP/bin" >> "$GITHUB_PATH"
211 echo "FOO=bar" >> "$GITHUB_ENV"
212 - run: mytool; echo "FOO=$FOO CI=$CI GITHUB_ACTIONS=$GITHUB_ACTIONS ERMITE_CONNECT=${ERMITE_CONNECT-unset}"
213 - name: python
214 shell: python3 {0}
215 run: print("py", 6*7)
216`), nil)
217 if s.Result == nil || s.Result.Conclusion != execpb.StepConclusion_STEP_CONCLUSION_SUCCESS || s.Exit.Code != 0 {
218 t.Fatalf("result %v exit %v\n%s", s.Result, s.Exit, strings.Join(s.Logs, "\n"))
219 }
220 if got := s.Result.Outputs["who"]; got != "runner:1001:1001" {
221 t.Errorf("steps run as %q, want runner:1001:1001", got)
222 }
223 all := strings.Join(s.Logs, "\n")
224 for _, want := range []string{
225 "home=/home/runner pwd=/home/runner/work/hello/hello",
226 "ws=/home/runner/work/hello/hello temp=/home/runner/work/_temp tool=/opt/hostedtoolcache",
227 "from-path", "FOO=bar CI=true GITHUB_ACTIONS=true ERMITE_CONNECT=unset", "py 42",
228 } {
229 if !strings.Contains(all, want) {
230 t.Errorf("log lacks %q\n%s", want, all)
231 }
232 }
233 if !strings.HasPrefix(strings.Join(s.Steps, "|"), "Set up job|Run echo") || s.Steps[len(s.Steps)-1] != "Complete job" {
234 t.Errorf("steps %v", s.Steps)
235 }
236}
237
238func TestContainerCancelStopsTheJobAndOrphans(t *testing.T) {
239 image := needE2E(t)
240 d := newDriver(t, image)
241 cancel := make(chan *execpb.Cancel, 1)
242 time.AfterFunc(4*time.Second, func() { cancel <- &execpb.Cancel{Reason: "test", Kind: execpb.CancelKind_CANCEL_KIND_USER} })
243 start := time.Now()
244 s := runJob(t, d, jobSpec(`on: push
245jobs:
246 j:
247 runs-on: x
248 steps:
249 - run: |
250 (sleep 300 &)
251 trap 'echo got-int; exit 130' INT
252 echo ready
253 while :; do sleep 0.2; done
254 - if: always()
255 run: echo cleanup-ran
256`), cancel)
257 if s.Result == nil || s.Result.Conclusion != execpb.StepConclusion_STEP_CONCLUSION_CANCELLED {
258 t.Fatalf("result %v\n%s", s.Result, strings.Join(s.Logs, "\n"))
259 }
260 all := strings.Join(s.Logs, "\n")
261 if !strings.Contains(all, "got-int") || !strings.Contains(all, "cleanup-ran") {
262 t.Errorf("SIGINT or the always() step missing:\n%s", all)
263 }
264 // "Complete job" kills the background sleep that the step left behind.
265 if !strings.Contains(all, "Terminate orphan process: pid (") || !strings.Contains(all, ") (sleep)") {
266 t.Errorf("orphan cleanup missing:\n%s", all)
267 }
268 if time.Since(start) > 40*time.Second {
269 t.Errorf("took %v", time.Since(start))
270 }
271}