1// SPDX-License-Identifier: MIT
2
3package executor
4
5import (
6 "encoding/json"
7 "fmt"
8 "os"
9 "os/exec"
10 "path/filepath"
11 "regexp"
12 "strings"
13 "sync"
14 "testing"
15 "time"
16)
17
18// These tests run jobs of the ermite-conformance corpus (the CI-1 scenarios)
19// through the executor on this machine and compare the steps, their
20// conclusions and their logs with the recordings made on github.com. They need
21// a checkout of ermite-conformance (the corpus and recordings directories),
22// found through ERMITE_CONFORMANCE_DIR or as a sibling of this repository;
23// they skip without it (and fail when ERMITE_CONFORMANCE_DIR is wrong).
24
25func conformanceDir(t *testing.T) string {
26 t.Helper()
27 return conformanceDirFor(t, "")
28}
29
30// conformanceDirFor is conformanceDir for a checkout that has the recording of
31// scenario (any checkout with a corpus when scenario is empty). The checkout is
32// ERMITE_CONFORMANCE_DIR when set (the test fails if it lacks the corpus or the
33// recording: a wrong path must not look like a pass), else the main checkout
34// ermite-conformance next to this repository (skipped when it has none).
35func conformanceDirFor(t *testing.T, scenario string) string {
36 t.Helper()
37 has := func(dir string) (corpus, rec bool) {
38 if st, err := os.Stat(filepath.Join(dir, "corpus")); err == nil && st.IsDir() {
39 corpus = true
40 }
41 if scenario == "" {
42 return corpus, true
43 }
44 _, err := os.Stat(filepath.Join(dir, "recordings", scenario+".json"))
45 return corpus, err == nil
46 }
47 if dir := os.Getenv("ERMITE_CONFORMANCE_DIR"); dir != "" {
48 corpus, rec := has(dir)
49 if !corpus {
50 t.Fatalf("ERMITE_CONFORMANCE_DIR=%s has no corpus directory", dir)
51 }
52 if !rec {
53 t.Fatalf("ERMITE_CONFORMANCE_DIR=%s has no recordings/%s.json", dir, scenario)
54 }
55 return dir
56 }
57 wd, _ := os.Getwd()
58 for d := wd; d != "/" && d != "."; d = filepath.Dir(d) {
59 c := filepath.Join(d, "ermite-conformance")
60 if corpus, rec := has(c); corpus && rec {
61 return c
62 }
63 }
64 t.Skip("ermite-conformance checkout not found next to this repository (set ERMITE_CONFORMANCE_DIR)")
65 return ""
66}
67
68func needTools(t *testing.T, names ...string) {
69 t.Helper()
70 for _, n := range names {
71 if _, err := exec.LookPath(n); err != nil {
72 t.Skipf("%s not installed", n)
73 }
74 }
75}
76
77// recorder is a Reporter that keeps everything.
78type recorder struct {
79 mu sync.Mutex
80 steps []*recStep
81 byNum map[int]*recStep
82 result Result
83 done bool
84 lastLogTime time.Time
85}
86
87type recStep struct {
88 Num int
89 Name string
90 Conclusion Conclusion
91 Log []string
92 Started time.Time
93 Completed time.Time
94}
95
96func newRecorder() *recorder { return &recorder{byNum: map[int]*recStep{}} }
97
98func (r *recorder) StepStarted(n int, name string, at time.Time) {
99 r.mu.Lock()
100 defer r.mu.Unlock()
101 s := &recStep{Num: n, Name: name, Started: at}
102 r.steps = append(r.steps, s)
103 r.byNum[n] = s
104}
105func (r *recorder) Log(n int, at time.Time, line string) {
106 r.mu.Lock()
107 defer r.mu.Unlock()
108 if s := r.byNum[n]; s != nil {
109 s.Log = append(s.Log, line)
110 }
111 r.lastLogTime = at
112}
113func (r *recorder) StepCompleted(n int, name string, c Conclusion, at time.Time) {
114 r.mu.Lock()
115 defer r.mu.Unlock()
116 s := r.byNum[n]
117 s.Conclusion, s.Completed = c, at
118 if name != "" {
119 s.Name = name
120 }
121}
122func (r *recorder) JobCompleted(res Result) {
123 r.mu.Lock()
124 defer r.mu.Unlock()
125 r.result, r.done = res, true
126}
127
128func (r *recorder) step(t *testing.T, name string) *recStep {
129 t.Helper()
130 for _, s := range r.steps {
131 if s.Name == name {
132 return s
133 }
134 }
135 t.Fatalf("no step %q in %v", name, r.names())
136 return nil
137}
138
139func (r *recorder) names() []string {
140 var out []string
141 for _, s := range r.steps {
142 out = append(out, s.Name)
143 }
144 return out
145}
146
147// testGitHubContext is a push on a branch, with the keys the corpus reads.
148func testGitHubContext(job string) string {
149 return fmt.Sprintf(`{"github":{"repository":"octo/hello","repository_owner":"octo","run_id":"1","run_number":"1","run_attempt":"1",
150 "ref":"refs/heads/scn/x/r1","ref_name":"scn/x/r1","ref_type":"branch","ref_protected":false,"sha":"0123456789abcdef0123456789abcdef01234567",
151 "event_name":"push","event":{"ref":"refs/heads/scn/x/r1"},"job":%q,"workflow":"w","actor":"octo","triggering_actor":"octo",
152 "server_url":"https://ermite.example","retention_days":"90"}}`, job)
153}
154
155// runLocal runs one job of a workflow file on this machine under a temporary
156// home. The environment is minimal and fixed.
157func runLocal(t *testing.T, yaml, jobKey string, tweak func(*Config)) (*recorder, Result) {
158 t.Helper()
159 return runLocalC(t, nil, yaml, jobKey, tweak)
160}
161
162// runLocalC is runLocal with a canceller.
163func runLocalC(t *testing.T, cancel *Canceller, yaml, jobKey string, tweak func(*Config)) (*recorder, Result) {
164 t.Helper()
165 home := t.TempDir()
166 layout := NewLayout(home, "hello", filepath.Join(home, "toolcache"))
167 rec := newRecorder()
168 cfg := Config{
169 WorkflowPath: ".github/workflows/ci.yml", WorkflowYAML: []byte(yaml), JobKey: jobKey,
170 ContextsJSON: testGitHubContext(jobKey), Runner: RunnerInfo{Name: "test-runner", Environment: "github-hosted"},
171 RunnerVersion: "test", Layout: &layout, Reporter: rec,
172 BaseEnv: []string{"PATH=" + os.Getenv("PATH"), "HOME=" + home, "LANG=C.UTF-8"},
173 TermAfter: 200 * time.Millisecond, KillAfter: 200 * time.Millisecond, WaitDelay: 500 * time.Millisecond,
174 }
175 if tweak != nil {
176 tweak(&cfg)
177 }
178 res := Run(cancel, cfg)
179 return rec, res
180}
181
182func readFile(t *testing.T, p string) string {
183 t.Helper()
184 b, err := os.ReadFile(p)
185 if err != nil {
186 t.Fatal(err)
187 }
188 return string(b)
189}
190
191type recording struct {
192 Pushes []struct {
193 Runs []struct {
194 Workflow string `json:"workflow"`
195 Conclusion string `json:"conclusion"`
196 Jobs []struct {
197 Name string `json:"name"`
198 Conclusion string `json:"conclusion"`
199 Steps []struct {
200 Name string `json:"name"`
201 Conclusion string `json:"conclusion"`
202 Log []string `json:"log"`
203 } `json:"steps"`
204 } `json:"jobs"`
205 } `json:"runs"`
206 } `json:"pushes"`
207}
208
209var (
210 shellPathRe = regexp.MustCompile(`^shell: \S*/(\w+) `)
211 startPathRe = regexp.MustCompile(`process '\S*/(\w+)'`)
212 uuidRe = regexp.MustCompile(`[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}`)
213 versionRe = regexp.MustCompile(`(Current runner version: ')[^']*(')`)
214)
215
216// normalizeLine maps the machine-dependent parts of a line to the form of the
217// recordings, like the normalisation rules of ermite-conformance: the
218// directory of the shell binary, the home directory, the repository name and
219// the random names of script files.
220func normalizeLine(l, home string) string {
221 l = shellPathRe.ReplaceAllString(l, "shell: /usr/bin/$1 ")
222 l = startPathRe.ReplaceAllString(l, "process '/usr/bin/$1'")
223 l = versionRe.ReplaceAllString(l, "${1}<runner-version>${2}")
224 l = strings.ReplaceAll(l, home, "/home/runner")
225 l = strings.ReplaceAll(l, "/work/hello/hello", "/work/<repo>/<repo>")
226 return uuidRe.ReplaceAllString(l, "<uuid>")
227}
228
229// shIsBash reports whether sh on this machine is bash (Arch, Fedora): the
230// image of ermite has dash, so a step that checks BASH_VERSION under sh would
231// differ here for a reason that has nothing to do with the executor.
232func shIsBash() bool {
233 out, err := exec.Command("sh", "-c", `echo "${BASH_VERSION-unset}"`).Output()
234 return err == nil && strings.TrimSpace(string(out)) != "unset"
235}
236
237// compareWithRecording runs every job without needs of a scenario and checks
238// that the steps, conclusions and logs equal the recording of github.com. The
239// dump step the harness appends is not compared (the recordings have no fleet
240// and image groups in Set up job, rule infra-groups of ermite-conformance).
241func compareWithRecording(t *testing.T, scenario string, skipJobs ...string) {
242 t.Helper()
243 needTools(t, "bash", "sh")
244 dir := conformanceDir(t)
245 files, _ := filepath.Glob(filepath.Join(dir, "corpus", scenario, "tree", ".github", "workflows", "*.yml"))
246 if len(files) != 1 {
247 t.Fatalf("scenario %s: workflow files %v", scenario, files)
248 }
249 yaml := readFile(t, files[0])
250 var rec recording
251 if err := json.Unmarshal([]byte(readFile(t, filepath.Join(dir, "recordings", scenario+".json"))), &rec); err != nil {
252 t.Fatal(err)
253 }
254 if len(rec.Pushes) == 0 || len(rec.Pushes[0].Runs) == 0 {
255 t.Skip("scenario not recorded")
256 }
257 skip := map[string]bool{}
258 for _, s := range skipJobs {
259 skip[s] = true
260 }
261 n := 0
262 for _, jr := range rec.Pushes[0].Runs[0].Jobs {
263 if skip[jr.Name] {
264 continue
265 }
266 jr := jr
267 t.Run(jr.Name, func(t *testing.T) {
268 var home string
269 got, res := runLocal(t, yaml, jr.Name, func(c *Config) { home = c.Layout.Home })
270 // Expected steps: the recording without Set up job and the dump step.
271 type es struct {
272 Name, Conclusion string
273 Log []string
274 }
275 var want []es
276 for _, s := range jr.Steps {
277 if s.Name == "ermite-dump" {
278 continue
279 }
280 want = append(want, es{s.Name, s.Conclusion, s.Log})
281 }
282 var have []es
283 for _, s := range got.steps {
284 var log []string
285 for _, l := range s.Log {
286 log = append(log, normalizeLine(l, home))
287 }
288 have = append(have, es{s.Name, string(s.Conclusion), log})
289 }
290 if len(have) != len(want) {
291 t.Fatalf("steps: got %v, want %d steps", got.names(), len(want))
292 }
293 for i := range want {
294 w, h := want[i], have[i]
295 if h.Name != w.Name || h.Conclusion != w.Conclusion {
296 t.Errorf("step %d: got %q (%s), want %q (%s)", i+1, h.Name, h.Conclusion, w.Name, w.Conclusion)
297 continue
298 }
299 if shIsBash() && strings.Contains(strings.Join(w.Log, "\n"), "shell: /usr/bin/sh ") {
300 continue // sh is bash here, dash on the image: the output is not comparable
301 }
302 if strings.Join(h.Log, "\n") != strings.Join(w.Log, "\n") {
303 t.Errorf("step %q log differs:\n--- got\n%s\n--- want\n%s", w.Name, strings.Join(h.Log, "\n"), strings.Join(w.Log, "\n"))
304 }
305 }
306 if string(res.Conclusion) != jr.Conclusion && jr.Conclusion != "" {
307 // The dump step fails on GitHub when an earlier step failed in
308 // some scenarios; the job conclusion is still the one of the steps.
309 t.Errorf("job conclusion %s, recording %s", res.Conclusion, jr.Conclusion)
310 }
311 })
312 n++
313 }
314 if n == 0 {
315 t.Fatal("no job compared")
316 }
317}
318
319// Scenarios whose jobs have no dependencies and whose logs do not depend on
320// the machine, in the order of the plan's list (shells, env, outputs, file
321// commands, conditions).
322func TestCorpusEnvPrecedence(t *testing.T) { compareWithRecording(t, "env-precedence") }
323func TestCorpusEnvExpressions(t *testing.T) { compareWithRecording(t, "env-expressions") }
324func TestCorpusEnvSpecialValues(t *testing.T) { compareWithRecording(t, "env-special-values") }
325func TestCorpusOutGithubEnv(t *testing.T) { compareWithRecording(t, "out-github-env") }
326func TestCorpusOutGithubEnvReserved(t *testing.T) { compareWithRecording(t, "out-github-env-reserved") }
327func TestCorpusOutGithubPath(t *testing.T) { compareWithRecording(t, "out-github-path") }
328func TestCorpusOutStepOutputs(t *testing.T) { compareWithRecording(t, "out-step-outputs") }
329func TestCorpusCtxSteps(t *testing.T) { compareWithRecording(t, "ctx-steps") }
330func TestCorpusIfStepConditions(t *testing.T) { compareWithRecording(t, "if-step-conditions") }
331func TestCorpusShellWorkingDirectory(t *testing.T) {
332 compareWithRecording(t, "shell-working-directory")
333}
334func TestCorpusShellExitCodes(t *testing.T) { compareWithRecording(t, "shell-exit-codes") }
335func TestCorpusIfJobLevel(t *testing.T) {
336 compareWithRecording(t, "if-job-level-fail-effects", "after-job-continue")
337}
338
339func envPath() string { return os.Getenv("PATH") }
340
341func TestCorpusShellBuiltin(t *testing.T) { compareWithRecording(t, "shell-builtin") }
342func TestCorpusShellCustom(t *testing.T) { compareWithRecording(t, "shell-custom") }
343func TestCorpusStatusFunctions(t *testing.T) { compareWithRecording(t, "expr-status-functions") }
344
345// GitHub rejects the workflow of expr-status-in-env (status functions are not
346// allowed in a step's env): the recorded run has no jobs. The executor must
347// refuse it at "Set up job" too.
348func TestCorpusStatusInEnvIsRejected(t *testing.T) {
349 dir := conformanceDir(t)
350 yaml := readFile(t, filepath.Join(dir, "corpus", "expr-status-in-env", "tree", ".github", "workflows", "ci.yml"))
351 rec, res := runLocal(t, yaml, "main", nil)
352 if res.Conclusion != ConclusionFailure || !strings.Contains(res.Error, "The workflow is not valid") || len(rec.steps) != 1 {
353 t.Fatalf("%+v %v", res, rec.names())
354 }
355}
356
357// TestCorpusPermissionsDisplay runs every job of every workflow of the
358// permissions-display scenario and compares the GITHUB_TOKEN Permissions group
359// of Set up job with the recording. The scenario has one workflow file per
360// case, so compareWithRecording does not apply. Workflows GitHub rejects have
361// no jobs in the recording and are not run.
362func TestCorpusPermissionsDisplay(t *testing.T) {
363 needTools(t, "bash")
364 dir := conformanceDirFor(t, "permissions-display")
365 var rec recording
366 if err := json.Unmarshal([]byte(readFile(t, filepath.Join(dir, "recordings", "permissions-display.json"))), &rec); err != nil {
367 t.Fatal(err)
368 }
369 n := 0
370 for _, run := range rec.Pushes[0].Runs {
371 if len(run.Jobs) == 0 {
372 continue
373 }
374 file := run.Workflow
375 yaml := readFile(t, filepath.Join(dir, "corpus", "permissions-display", "tree", filepath.FromSlash(file)))
376 for _, jr := range run.Jobs {
377 t.Run(strings.TrimSuffix(filepath.Base(file), ".yml")+"/"+jr.Name, func(t *testing.T) {
378 if len(jr.Steps) == 0 {
379 t.Fatal("no steps in the recording")
380 }
381 want := permissionsGroup(jr.Steps[0].Log)
382 if want == nil {
383 t.Fatalf("no group in the recording: %v", jr.Steps[0].Log)
384 }
385 got, res := runLocal(t, yaml, jr.Name, nil)
386 if res.Conclusion != ConclusionSuccess || len(got.steps) == 0 {
387 t.Fatalf("job did not run: %+v %v", res, got.names())
388 }
389 if have := permissionsGroup(got.steps[0].Log); strings.Join(have, "\n") != strings.Join(want, "\n") {
390 t.Errorf("got\n%s\nwant\n%s", strings.Join(have, "\n"), strings.Join(want, "\n"))
391 }
392 })
393 n++
394 }
395 }
396 if n < 20 {
397 t.Fatalf("only %d jobs in the recording", n)
398 }
399}
400
401// permissionsGroup returns the lines of the GITHUB_TOKEN Permissions group.
402func permissionsGroup(log []string) []string {
403 for i, l := range log {
404 if l != "##[group]GITHUB_TOKEN Permissions" {
405 continue
406 }
407 for k := i + 1; k < len(log); k++ {
408 if log[k] == "##[endgroup]" {
409 return log[i+1 : k]
410 }
411 }
412 }
413 return nil
414}