1// SPDX-License-Identifier: MIT
2
3package e2e
4
5import (
6 "context"
7 "encoding/json"
8 "fmt"
9 "net/http"
10 "os"
11 "path/filepath"
12 "regexp"
13 "sort"
14 "strings"
15 "testing"
16 "time"
17
18 "github.com/bnema/zerowrap"
19
20 ermite "bnema.dev/ermite-api/gen/ermite"
21 "bnema.dev/ermite-api/gen/ermite/ermiteconnect"
22
23 "bnema.dev/ermite-runner/internal/host"
24 "bnema.dev/ermite-runner/internal/host/hosttest"
25)
26
27// The chain test runs the whole path of a job: a fake RunnerService hands out a
28// corpus job, the real host loop starts a real sandbox (the runner image in a
29// container), the real executor runs the steps, and the host reports to the
30// fake with AppendLog, UpdateJob and CompleteJob. What the fake stored is
31// compared with the recording of the same job made on github.com.
32//
33// It needs a checkout of ermite-conformance (corpus/ and recordings/), found
34// through ERMITE_CONFORMANCE_DIR or as a sibling of this repository, and is
35// read-only on it.
36
37func conformanceDir(t *testing.T) string {
38 t.Helper()
39 cands := []string{os.Getenv("ERMITE_CONFORMANCE_DIR")}
40 wd, _ := os.Getwd()
41 for d := wd; d != "/" && d != "."; d = filepath.Dir(d) {
42 cands = append(cands, filepath.Join(d, "ermite-conformance", ".worktrees", "corpus"), filepath.Join(d, "ermite-conformance"))
43 }
44 for _, c := range cands {
45 if c == "" {
46 continue
47 }
48 if st, err := os.Stat(filepath.Join(c, "recordings")); err == nil && st.IsDir() {
49 return c
50 }
51 }
52 t.Skip("ermite-conformance checkout not found (set ERMITE_CONFORMANCE_DIR)")
53 return ""
54}
55
56type recStep struct {
57 Name string `json:"name"`
58 Conclusion string `json:"conclusion"`
59 Log []string `json:"log"`
60}
61
62type recJob struct {
63 Name string `json:"name"`
64 Conclusion string `json:"conclusion"`
65 Steps []recStep `json:"steps"`
66}
67
68type recording struct {
69 Pushes []struct {
70 Runs []struct {
71 Jobs []recJob `json:"jobs"`
72 } `json:"runs"`
73 } `json:"pushes"`
74}
75
76var (
77 uuidRe = regexp.MustCompile(`[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}`)
78 // Machine-dependent parts of the recordings, like the normalisation rules of
79 // ermite-conformance: random script names and the repository name.
80 repoRe = regexp.MustCompile(`/work/hello/hello`)
81 // The runner version moves with each release (rule runner-version of
82 // ermite-conformance).
83 versionRe = regexp.MustCompile(`(Current runner version: ')[^']*(')`)
84)
85
86func normalize(l string) string {
87 l = versionRe.ReplaceAllString(l, "${1}<runner-version>${2}")
88 l = repoRe.ReplaceAllString(l, "/work/<repo>/<repo>")
89 return uuidRe.ReplaceAllString(l, "<uuid>")
90}
91
92func chainContexts(job string) string {
93 return fmt.Sprintf(`{"github":{"repository":"octo/hello","repository_owner":"octo","run_id":"1","run_number":"1","run_attempt":"1",
94 "ref":"refs/heads/scn/x/r1","ref_name":"scn/x/r1","ref_type":"branch","ref_protected":false,"sha":"0123456789abcdef0123456789abcdef01234567",
95 "event_name":"push","event":{"ref":"refs/heads/scn/x/r1"},"job":%q,"workflow":"w","actor":"octo","triggering_actor":"octo",
96 "server_url":"https://ermite.example","retention_days":"90"}}`, job)
97}
98
99func TestChainCorpus(t *testing.T) {
100 image := needE2E(t)
101 dir := conformanceDir(t)
102 drv := newDriver(t, image)
103
104 // scenario -> jobs to run (jobs with `needs` depend on the server and are left out)
105 scenarios := []struct {
106 name string
107 jobs []string
108 }{
109 {"env-precedence", []string{"main", "other"}},
110 {"env-expressions", []string{"main"}},
111 {"out-github-env", []string{"main"}},
112 {"out-github-env-reserved", []string{"node-options", "github-prefix"}},
113 {"out-github-path", []string{"main"}},
114 {"out-step-outputs", []string{"main"}},
115 {"ctx-steps", []string{"main"}},
116 {"if-step-conditions", []string{"main"}},
117 {"shell-builtin", []string{"main"}},
118 {"shell-custom", []string{"main"}},
119 {"shell-exit-codes", nil}, // every job of the workflow
120 {"shell-working-directory", nil},
121 {"out-job-outputs", []string{"producer"}},
122 }
123
124 srv := hosttest.New()
125 t.Cleanup(srv.Close)
126
127 type want struct {
128 scenario string
129 job recJob
130 }
131 wants := map[string]want{}
132 var queue []*ermite.Job
133 for _, sc := range scenarios {
134 files, _ := filepath.Glob(filepath.Join(dir, "corpus", sc.name, "tree", ".github", "workflows", "*.yml"))
135 if len(files) != 1 {
136 t.Fatalf("%s: workflow files %v", sc.name, files)
137 }
138 yaml, err := os.ReadFile(files[0])
139 if err != nil {
140 t.Fatal(err)
141 }
142 raw, err := os.ReadFile(filepath.Join(dir, "recordings", sc.name+".json"))
143 if err != nil {
144 t.Fatal(err)
145 }
146 var rec recording
147 if err := json.Unmarshal(raw, &rec); err != nil {
148 t.Fatal(err)
149 }
150 for _, rj := range rec.Pushes[0].Runs[0].Jobs {
151 if sc.jobs != nil && !contains(sc.jobs, rj.Name) {
152 continue
153 }
154 id := sc.name + "/" + rj.Name
155 queue = append(queue, &ermite.Job{
156 JobId: id, RunId: 1, Attempt: 1, JobToken: "ermitejob_" + strings.ReplaceAll(id, "/", "_"),
157 WorkflowPath: ".github/workflows/ci.yml", WorkflowYaml: string(yaml), JobKey: rj.Name, Matrix: "{}",
158 Contexts: chainContexts(rj.Name), Repo: "hello", TimeoutMinutes: 10,
159 })
160 wants[id] = want{sc.name, rj}
161 }
162 }
163 srv.Queue(queue...)
164
165 r := &host.Runner{
166 Cfg: host.Config{URL: srv.URL, Name: "chain", Capacity: 2, Version: "test"},
167 Driver: drv,
168 Poll: host.NewClient(srv.URL, hosttest.RunnerToken, http.DefaultTransport),
169 Job: func(tok string) ermiteconnect.RunnerServiceClient {
170 return host.NewClient(srv.URL, tok, http.DefaultTransport)
171 },
172 Timing: host.Timing{LogFlush: 20 * time.Millisecond, NoJobPause: 100 * time.Millisecond},
173 }
174 ctx, cancel := context.WithCancel(zerowrap.WithCtx(context.Background(), zerowrap.New(zerowrap.Config{Level: "info", Format: "console", Output: testWriter{t}})))
175 done := make(chan error, 1)
176 go func() { done <- r.Run(ctx) }()
177 defer func() {
178 cancel()
179 select {
180 case err := <-done:
181 if err != nil {
182 t.Errorf("Run: %v", err)
183 }
184 case <-time.After(60 * time.Second):
185 t.Error("Run did not stop")
186 }
187 }()
188
189 deadline := time.After(5 * time.Minute)
190 for n := 0; n < len(queue); n++ {
191 select {
192 case <-srv.Done():
193 case <-deadline:
194 t.Fatalf("only %d of %d jobs finished", n, len(queue))
195 }
196 }
197
198 var ids []string
199 for id := range wants {
200 ids = append(ids, id)
201 }
202 sort.Strings(ids)
203 for _, id := range ids {
204 w := wants[id]
205 t.Run(id, func(t *testing.T) { compareJob(t, srv.Result(id), w.job, w.scenario) })
206 }
207 for _, e := range srv.Violations() {
208 t.Errorf("the fake server saw a contract violation: %s", e)
209 }
210}
211
212func contains(l []string, s string) bool {
213 for _, x := range l {
214 if x == s {
215 return true
216 }
217 }
218 return false
219}
220
221// compareJob checks the steps, conclusions, outputs and log lines reported for
222// a job against the recording. The ermite-dump step that the harness appends to
223// every job is not compared, and neither are the three groups of "Set up job"
224// that describe GitHub's fleet and image (ermite-conformance drops them from
225// the recordings, rule infra-groups): the other lines of "Set up job" are.
226func compareJob(t *testing.T, got hosttest.Result, want recJob, scenario string) {
227 if got.Complete == nil {
228 t.Fatal("the job did not complete")
229 }
230 // The conclusion of the job as GitHub recorded it, except where the dump
231 // step failed on GitHub (it only fails after a failed step in some
232 // scenarios and is not part of the job here).
233 wantConcl := map[string]ermite.CiConclusion{
234 "success": ermite.CiConclusion_CI_CONCLUSION_SUCCESS, "failure": ermite.CiConclusion_CI_CONCLUSION_FAILURE,
235 "cancelled": ermite.CiConclusion_CI_CONCLUSION_CANCELLED,
236 }
237 jobWant := wantConcl[want.Conclusion]
238 for _, s := range want.Steps {
239 if s.Name == "ermite-dump" && s.Conclusion != "success" && want.Conclusion == "failure" {
240 // the job failed only because of the dump step
241 jobWant = ermite.CiConclusion_CI_CONCLUSION_SUCCESS
242 for _, o := range want.Steps {
243 if o.Name != "ermite-dump" && o.Conclusion == "failure" {
244 jobWant = ermite.CiConclusion_CI_CONCLUSION_FAILURE
245 }
246 }
247 }
248 }
249 if got.Complete.Conclusion != jobWant {
250 t.Errorf("job conclusion %v, want %v", got.Complete.Conclusion, jobWant)
251 }
252
253 // Steps by number, with their lines.
254 byStep := map[int32][]string{}
255 for _, l := range got.Lines {
256 byStep[l.Step] = append(byStep[l.Step], normalize(l.Text))
257 }
258 var steps []int32
259 for n := range got.Steps {
260 steps = append(steps, n)
261 }
262 sort.Slice(steps, func(i, j int) bool { return steps[i] < steps[j] })
263
264 type es = recStep
265 var wantSteps, gotSteps []es
266 for _, s := range want.Steps {
267 if s.Name == "ermite-dump" {
268 continue
269 }
270 wantSteps = append(wantSteps, s)
271 }
272 conc := map[ermite.CiConclusion]string{
273 ermite.CiConclusion_CI_CONCLUSION_SUCCESS: "success", ermite.CiConclusion_CI_CONCLUSION_FAILURE: "failure",
274 ermite.CiConclusion_CI_CONCLUSION_CANCELLED: "cancelled", ermite.CiConclusion_CI_CONCLUSION_SKIPPED: "skipped",
275 }
276 for _, n := range steps {
277 u := got.Steps[n]
278 if u.State != ermite.StepState_STEP_STATE_COMPLETED || u.StartedAt == nil || u.CompletedAt == nil {
279 t.Errorf("step %d %q is not a complete snapshot: %v", n, u.Name, u)
280 }
281 gotSteps = append(gotSteps, es{Name: u.Name, Conclusion: conc[u.Conclusion], Log: byStep[n]})
282 }
283 if len(gotSteps) != len(wantSteps) {
284 var names []string
285 for _, s := range gotSteps {
286 names = append(names, s.Name)
287 }
288 t.Fatalf("steps %q, want %d steps", names, len(wantSteps))
289 }
290 for i := range wantSteps {
291 w, g := wantSteps[i], gotSteps[i]
292 if g.Name != w.Name || g.Conclusion != w.Conclusion {
293 t.Errorf("step %d: %q (%s), want %q (%s)", i+1, g.Name, g.Conclusion, w.Name, w.Conclusion)
294 continue
295 }
296 if strings.Join(g.Log, "\n") != strings.Join(w.Log, "\n") {
297 t.Errorf("step %q log differs:\n--- got\n%s\n--- want\n%s", w.Name, strings.Join(g.Log, "\n"), strings.Join(w.Log, "\n"))
298 }
299 }
300
301 if scenario == "out-job-outputs" {
302 checkProducerOutputs(t, got)
303 }
304}
305
306// checkProducerOutputs compares the outputs of the producer job of
307// out-job-outputs with what the recorded consumer printed from `needs`.
308func checkProducerOutputs(t *testing.T, got hosttest.Result) {
309 t.Helper()
310 // The values the consumer printed on GitHub (recordings/out-job-outputs.json,
311 // step "print needs outputs"): empty outputs are absent.
312 want := map[string]string{
313 "composed": "hello-push", "from-env": "env-value", "from-failed": "written-before-failure",
314 "overwritten": "second", "simple": "hello", "multi": "line one\nline two", "literal": "fixed-text",
315 }
316 if len(got.Complete.Outputs) != len(want) {
317 t.Errorf("outputs %v, want %v", got.Complete.Outputs, want)
318 }
319 for k, v := range want {
320 if got.Complete.Outputs[k] != v {
321 t.Errorf("output %q = %q, want %q", k, got.Complete.Outputs[k], v)
322 }
323 }
324}
325
326type testWriter struct{ t *testing.T }
327
328func (w testWriter) Write(p []byte) (int, error) {
329 w.t.Log(strings.TrimRight(string(p), "\n"))
330 return len(p), nil
331}