1// SPDX-License-Identifier: MIT
2
3package executor
4
5import (
6 "encoding/json"
7 "fmt"
8 "os"
9 "os/exec"
10 "path/filepath"
11 "regexp"
12 "strings"
13 "sync"
14 "testing"
15 "time"
16)
17
18// These tests run jobs of the ermite-conformance corpus (the CI-1 scenarios)
19// through the executor on this machine and compare the steps, their
20// conclusions and their logs with the recordings made on github.com. They need
21// a checkout of ermite-conformance (the corpus and recordings directories),
22// found through ERMITE_CONFORMANCE_DIR or as a sibling of this repository;
23// they skip without it (and fail when ERMITE_CONFORMANCE_DIR is wrong).
24
25func conformanceDir(t *testing.T) string {
26 t.Helper()
27 return conformanceDirFor(t, "")
28}
29
30// conformanceDirFor is conformanceDir for a checkout that has the recording of
31// scenario (any checkout with a corpus when scenario is empty). The checkout is
32// ERMITE_CONFORMANCE_DIR when set (the test fails if it lacks the corpus or the
33// recording: a wrong path must not look like a pass), else the main checkout
34// ermite-conformance next to this repository (skipped when it has none).
35func conformanceDirFor(t *testing.T, scenario string) string {
36 t.Helper()
37 has := func(dir string) (corpus, rec bool) {
38 if st, err := os.Stat(filepath.Join(dir, "corpus")); err == nil && st.IsDir() {
39 corpus = true
40 }
41 if scenario == "" {
42 return corpus, true
43 }
44 _, err := os.Stat(filepath.Join(dir, "recordings", scenario+".json"))
45 return corpus, err == nil
46 }
47 if dir := os.Getenv("ERMITE_CONFORMANCE_DIR"); dir != "" {
48 corpus, rec := has(dir)
49 if !corpus {
50 t.Fatalf("ERMITE_CONFORMANCE_DIR=%s has no corpus directory", dir)
51 }
52 if !rec {
53 t.Fatalf("ERMITE_CONFORMANCE_DIR=%s has no recordings/%s.json", dir, scenario)
54 }
55 return dir
56 }
57 wd, _ := os.Getwd()
58 for d := wd; d != "/" && d != "."; d = filepath.Dir(d) {
59 c := filepath.Join(d, "ermite-conformance")
60 if corpus, rec := has(c); corpus && rec {
61 return c
62 }
63 }
64 t.Skip("ermite-conformance checkout not found next to this repository (set ERMITE_CONFORMANCE_DIR)")
65 return ""
66}
67
68func needTools(t *testing.T, names ...string) {
69 t.Helper()
70 for _, n := range names {
71 if _, err := exec.LookPath(n); err != nil {
72 t.Skipf("%s not installed", n)
73 }
74 }
75}
76
77// recorder is a Reporter that keeps everything.
78type recorder struct {
79 mu sync.Mutex
80 steps []*recStep
81 byNum map[int]*recStep
82 result Result
83 done bool
84 lastLogTime time.Time
85}
86
87type recStep struct {
88 Num int
89 Name string
90 Conclusion Conclusion
91 Log []string
92 Started time.Time
93 Completed time.Time
94}
95
96func newRecorder() *recorder { return &recorder{byNum: map[int]*recStep{}} }
97
98func (r *recorder) StepStarted(n int, name string, at time.Time) {
99 r.mu.Lock()
100 defer r.mu.Unlock()
101 s := &recStep{Num: n, Name: name, Started: at}
102 r.steps = append(r.steps, s)
103 r.byNum[n] = s
104}
105func (r *recorder) Log(n int, at time.Time, line string) {
106 r.mu.Lock()
107 defer r.mu.Unlock()
108 if s := r.byNum[n]; s != nil {
109 s.Log = append(s.Log, line)
110 }
111 r.lastLogTime = at
112}
113func (r *recorder) StepCompleted(n int, name string, c Conclusion, at time.Time) {
114 r.mu.Lock()
115 defer r.mu.Unlock()
116 s := r.byNum[n]
117 s.Conclusion, s.Completed = c, at
118 if name != "" {
119 s.Name = name
120 }
121}
122func (r *recorder) JobCompleted(res Result) {
123 r.mu.Lock()
124 defer r.mu.Unlock()
125 r.result, r.done = res, true
126}
127
128func (r *recorder) step(t *testing.T, name string) *recStep {
129 t.Helper()
130 for _, s := range r.steps {
131 if s.Name == name {
132 return s
133 }
134 }
135 t.Fatalf("no step %q in %v", name, r.names())
136 return nil
137}
138
139func (r *recorder) names() []string {
140 var out []string
141 for _, s := range r.steps {
142 out = append(out, s.Name)
143 }
144 return out
145}
146
147// testGitHubContext is a push on a branch, with the keys the corpus reads.
148func testGitHubContext(job string) string {
149 return fmt.Sprintf(`{"github":{"repository":"octo/hello","repository_owner":"octo","run_id":"1","run_number":"1","run_attempt":"1",
150 "ref":"refs/heads/scn/x/r1","ref_name":"scn/x/r1","ref_type":"branch","ref_protected":false,"sha":"0123456789abcdef0123456789abcdef01234567",
151 "event_name":"push","event":{"ref":"refs/heads/scn/x/r1"},"job":%q,"workflow":"w","actor":"octo","triggering_actor":"octo",
152 "server_url":"https://ermite.example","retention_days":"90"}}`, job)
153}
154
155// runLocal runs one job of a workflow file on this machine under a temporary
156// home. The environment is minimal and fixed.
157func runLocal(t *testing.T, yaml, jobKey string, tweak func(*Config)) (*recorder, Result) {
158 t.Helper()
159 home := t.TempDir()
160 layout := NewLayout(home, "hello", filepath.Join(home, "toolcache"))
161 rec := newRecorder()
162 cfg := Config{
163 WorkflowPath: ".github/workflows/ci.yml", WorkflowYAML: []byte(yaml), JobKey: jobKey,
164 ContextsJSON: testGitHubContext(jobKey), Runner: RunnerInfo{Name: "test-runner", Environment: "github-hosted"},
165 RunnerVersion: "test", Layout: &layout, Reporter: rec,
166 BaseEnv: []string{"PATH=" + os.Getenv("PATH"), "HOME=" + home, "LANG=C.UTF-8"},
167 TermAfter: 200 * time.Millisecond, KillAfter: 200 * time.Millisecond, WaitDelay: 500 * time.Millisecond,
168 }
169 if tweak != nil {
170 tweak(&cfg)
171 }
172 res := Run(nil, cfg)
173 return rec, res
174}
175
176func readFile(t *testing.T, p string) string {
177 t.Helper()
178 b, err := os.ReadFile(p)
179 if err != nil {
180 t.Fatal(err)
181 }
182 return string(b)
183}
184
185type recording struct {
186 Pushes []struct {
187 Runs []struct {
188 Workflow string `json:"workflow"`
189 Conclusion string `json:"conclusion"`
190 Jobs []struct {
191 Name string `json:"name"`
192 Conclusion string `json:"conclusion"`
193 Steps []struct {
194 Name string `json:"name"`
195 Conclusion string `json:"conclusion"`
196 Log []string `json:"log"`
197 } `json:"steps"`
198 } `json:"jobs"`
199 } `json:"runs"`
200 } `json:"pushes"`
201}
202
203var (
204 shellPathRe = regexp.MustCompile(`^shell: \S*/(\w+) `)
205 startPathRe = regexp.MustCompile(`process '\S*/(\w+)'`)
206 uuidRe = regexp.MustCompile(`[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}`)
207 versionRe = regexp.MustCompile(`(Current runner version: ')[^']*(')`)
208)
209
210// normalizeLine maps the machine-dependent parts of a line to the form of the
211// recordings, like the normalisation rules of ermite-conformance: the
212// directory of the shell binary, the home directory, the repository name and
213// the random names of script files.
214func normalizeLine(l, home string) string {
215 l = shellPathRe.ReplaceAllString(l, "shell: /usr/bin/$1 ")
216 l = startPathRe.ReplaceAllString(l, "process '/usr/bin/$1'")
217 l = versionRe.ReplaceAllString(l, "${1}<runner-version>${2}")
218 l = strings.ReplaceAll(l, home, "/home/runner")
219 l = strings.ReplaceAll(l, "/work/hello/hello", "/work/<repo>/<repo>")
220 return uuidRe.ReplaceAllString(l, "<uuid>")
221}
222
223// shIsBash reports whether sh on this machine is bash (Arch, Fedora): the
224// image of ermite has dash, so a step that checks BASH_VERSION under sh would
225// differ here for a reason that has nothing to do with the executor.
226func shIsBash() bool {
227 out, err := exec.Command("sh", "-c", `echo "${BASH_VERSION-unset}"`).Output()
228 return err == nil && strings.TrimSpace(string(out)) != "unset"
229}
230
231// compareWithRecording runs every job without needs of a scenario and checks
232// that the steps, conclusions and logs equal the recording of github.com. The
233// dump step the harness appends is not compared (the recordings have no fleet
234// and image groups in Set up job, rule infra-groups of ermite-conformance).
235func compareWithRecording(t *testing.T, scenario string, skipJobs ...string) {
236 t.Helper()
237 needTools(t, "bash", "sh")
238 dir := conformanceDir(t)
239 files, _ := filepath.Glob(filepath.Join(dir, "corpus", scenario, "tree", ".github", "workflows", "*.yml"))
240 if len(files) != 1 {
241 t.Fatalf("scenario %s: workflow files %v", scenario, files)
242 }
243 yaml := readFile(t, files[0])
244 var rec recording
245 if err := json.Unmarshal([]byte(readFile(t, filepath.Join(dir, "recordings", scenario+".json"))), &rec); err != nil {
246 t.Fatal(err)
247 }
248 if len(rec.Pushes) == 0 || len(rec.Pushes[0].Runs) == 0 {
249 t.Skip("scenario not recorded")
250 }
251 skip := map[string]bool{}
252 for _, s := range skipJobs {
253 skip[s] = true
254 }
255 n := 0
256 for _, jr := range rec.Pushes[0].Runs[0].Jobs {
257 if skip[jr.Name] {
258 continue
259 }
260 jr := jr
261 t.Run(jr.Name, func(t *testing.T) {
262 var home string
263 got, res := runLocal(t, yaml, jr.Name, func(c *Config) { home = c.Layout.Home })
264 // Expected steps: the recording without Set up job and the dump step.
265 type es struct {
266 Name, Conclusion string
267 Log []string
268 }
269 var want []es
270 for _, s := range jr.Steps {
271 if s.Name == "ermite-dump" {
272 continue
273 }
274 want = append(want, es{s.Name, s.Conclusion, s.Log})
275 }
276 var have []es
277 for _, s := range got.steps {
278 var log []string
279 for _, l := range s.Log {
280 log = append(log, normalizeLine(l, home))
281 }
282 have = append(have, es{s.Name, string(s.Conclusion), log})
283 }
284 if len(have) != len(want) {
285 t.Fatalf("steps: got %v, want %d steps", got.names(), len(want))
286 }
287 for i := range want {
288 w, h := want[i], have[i]
289 if h.Name != w.Name || h.Conclusion != w.Conclusion {
290 t.Errorf("step %d: got %q (%s), want %q (%s)", i+1, h.Name, h.Conclusion, w.Name, w.Conclusion)
291 continue
292 }
293 if shIsBash() && strings.Contains(strings.Join(w.Log, "\n"), "shell: /usr/bin/sh ") {
294 continue // sh is bash here, dash on the image: the output is not comparable
295 }
296 if strings.Join(h.Log, "\n") != strings.Join(w.Log, "\n") {
297 t.Errorf("step %q log differs:\n--- got\n%s\n--- want\n%s", w.Name, strings.Join(h.Log, "\n"), strings.Join(w.Log, "\n"))
298 }
299 }
300 if string(res.Conclusion) != jr.Conclusion && jr.Conclusion != "" {
301 // The dump step fails on GitHub when an earlier step failed in
302 // some scenarios; the job conclusion is still the one of the steps.
303 t.Errorf("job conclusion %s, recording %s", res.Conclusion, jr.Conclusion)
304 }
305 })
306 n++
307 }
308 if n == 0 {
309 t.Fatal("no job compared")
310 }
311}
312
313// Scenarios whose jobs have no dependencies and whose logs do not depend on
314// the machine, in the order of the plan's list (shells, env, outputs, file
315// commands, conditions).
316func TestCorpusEnvPrecedence(t *testing.T) { compareWithRecording(t, "env-precedence") }
317func TestCorpusEnvExpressions(t *testing.T) { compareWithRecording(t, "env-expressions") }
318func TestCorpusEnvSpecialValues(t *testing.T) { compareWithRecording(t, "env-special-values") }
319func TestCorpusOutGithubEnv(t *testing.T) { compareWithRecording(t, "out-github-env") }
320func TestCorpusOutGithubEnvReserved(t *testing.T) { compareWithRecording(t, "out-github-env-reserved") }
321func TestCorpusOutGithubPath(t *testing.T) { compareWithRecording(t, "out-github-path") }
322func TestCorpusOutStepOutputs(t *testing.T) { compareWithRecording(t, "out-step-outputs") }
323func TestCorpusCtxSteps(t *testing.T) { compareWithRecording(t, "ctx-steps") }
324func TestCorpusIfStepConditions(t *testing.T) { compareWithRecording(t, "if-step-conditions") }
325func TestCorpusShellWorkingDirectory(t *testing.T) {
326 compareWithRecording(t, "shell-working-directory")
327}
328func TestCorpusShellExitCodes(t *testing.T) { compareWithRecording(t, "shell-exit-codes") }
329func TestCorpusIfJobLevel(t *testing.T) {
330 compareWithRecording(t, "if-job-level-fail-effects", "after-job-continue")
331}
332
333func envPath() string { return os.Getenv("PATH") }
334
335func TestCorpusShellBuiltin(t *testing.T) { compareWithRecording(t, "shell-builtin") }
336func TestCorpusShellCustom(t *testing.T) { compareWithRecording(t, "shell-custom") }
337func TestCorpusStatusFunctions(t *testing.T) { compareWithRecording(t, "expr-status-functions") }
338
339// GitHub rejects the workflow of expr-status-in-env (status functions are not
340// allowed in a step's env): the recorded run has no jobs. The executor must
341// refuse it at "Set up job" too.
342func TestCorpusStatusInEnvIsRejected(t *testing.T) {
343 dir := conformanceDir(t)
344 yaml := readFile(t, filepath.Join(dir, "corpus", "expr-status-in-env", "tree", ".github", "workflows", "ci.yml"))
345 rec, res := runLocal(t, yaml, "main", nil)
346 if res.Conclusion != ConclusionFailure || !strings.Contains(res.Error, "The workflow is not valid") || len(rec.steps) != 1 {
347 t.Fatalf("%+v %v", res, rec.names())
348 }
349}
350
351// TestCorpusPermissionsDisplay runs every job of every workflow of the
352// permissions-display scenario and compares the GITHUB_TOKEN Permissions group
353// of Set up job with the recording. The scenario has one workflow file per
354// case, so compareWithRecording does not apply. Workflows GitHub rejects have
355// no jobs in the recording and are not run.
356func TestCorpusPermissionsDisplay(t *testing.T) {
357 needTools(t, "bash")
358 dir := conformanceDirFor(t, "permissions-display")
359 var rec recording
360 if err := json.Unmarshal([]byte(readFile(t, filepath.Join(dir, "recordings", "permissions-display.json"))), &rec); err != nil {
361 t.Fatal(err)
362 }
363 n := 0
364 for _, run := range rec.Pushes[0].Runs {
365 if len(run.Jobs) == 0 {
366 continue
367 }
368 file := run.Workflow
369 yaml := readFile(t, filepath.Join(dir, "corpus", "permissions-display", "tree", filepath.FromSlash(file)))
370 for _, jr := range run.Jobs {
371 t.Run(strings.TrimSuffix(filepath.Base(file), ".yml")+"/"+jr.Name, func(t *testing.T) {
372 if len(jr.Steps) == 0 {
373 t.Fatal("no steps in the recording")
374 }
375 want := permissionsGroup(jr.Steps[0].Log)
376 if want == nil {
377 t.Fatalf("no group in the recording: %v", jr.Steps[0].Log)
378 }
379 got, res := runLocal(t, yaml, jr.Name, nil)
380 if res.Conclusion != ConclusionSuccess || len(got.steps) == 0 {
381 t.Fatalf("job did not run: %+v %v", res, got.names())
382 }
383 if have := permissionsGroup(got.steps[0].Log); strings.Join(have, "\n") != strings.Join(want, "\n") {
384 t.Errorf("got\n%s\nwant\n%s", strings.Join(have, "\n"), strings.Join(want, "\n"))
385 }
386 })
387 n++
388 }
389 }
390 if n < 20 {
391 t.Fatalf("only %d jobs in the recording", n)
392 }
393}
394
395// permissionsGroup returns the lines of the GITHUB_TOKEN Permissions group.
396func permissionsGroup(log []string) []string {
397 for i, l := range log {
398 if l != "##[group]GITHUB_TOKEN Permissions" {
399 continue
400 }
401 for k := i + 1; k < len(log); k++ {
402 if log[k] == "##[endgroup]" {
403 return log[i+1 : k]
404 }
405 }
406 }
407 return nil
408}