1// SPDX-License-Identifier: AGPL-3.0-or-later
2
3package app
4
5import (
6 "encoding/json"
7 "net/http"
8 "os"
9 "slices"
10 "strings"
11 "testing"
12
13 "connectrpc.com/connect"
14 "github.com/stretchr/testify/assert"
15 "github.com/stretchr/testify/require"
16
17 pb "bnema.dev/ermite-api/gen/ermite"
18 "bnema.dev/ermite/internal/domain"
19)
20
21// conformance is testdata/conformance/ctx-job-runner.contexts.json: the
22// contexts github.com gave to the ctx-job-runner scenario of ermite-conformance
23// (see its _source).
24type conformance struct {
25 GithubKeyOrder []string `json:"_github_key_order"`
26 JobKeyOrder []string `json:"_job_key_order"`
27 RunnerOwned []string `json:"_runner_owned_github_keys"`
28 Github map[string]any `json:"github"`
29 Job map[string]any `json:"job"`
30 Matrix any `json:"matrix"`
31 Strategy map[string]any `json:"strategy"`
32}
33
34func loadConformance(t *testing.T) conformance {
35 t.Helper()
36 b, err := os.ReadFile("testdata/conformance/ctx-job-runner.contexts.json")
37 require.NoError(t, err)
38 var c conformance
39 require.NoError(t, json.Unmarshal(b, &c))
40 return c
41}
42
43// numericPlaceholders are the placeholders of the normaliser of
44// ermite-conformance for values that are numbers (or times) on github.com: ids,
45// times and the repository size. In the raw bundle of the same scenario id,
46// created_at, pushed_at and size are JSON numbers; the fixture only has the
47// placeholder, a string. They are matched by "number or string".
48var numericPlaceholders = []string{"<id:", "<time>", "<size>", "<actor_id:", "<repository_id:", "<repository_owner_id:", "<check_run_id:"}
49
50// shape is the type of each value of an object. A placeholder of a number
51// has the type "scalar".
52func shape(m map[string]any) map[string]string {
53 out := map[string]string{}
54 for k, v := range m {
55 if s, ok := v.(string); ok && slices.ContainsFunc(numericPlaceholders, func(p string) bool { return strings.HasPrefix(s, p) }) {
56 out[k] = "scalar"
57 continue
58 }
59 switch v.(type) {
60 case nil:
61 out[k] = "null"
62 case string:
63 out[k] = "string"
64 case float64:
65 out[k] = "number"
66 case bool:
67 out[k] = "bool"
68 case []any:
69 out[k] = "array"
70 case map[string]any:
71 out[k] = "object"
72 }
73 }
74 return out
75}
76
77// orderedKeys returns the keys of a JSON object in document order.
78func orderedKeys(t *testing.T, raw json.RawMessage) []string {
79 t.Helper()
80 dec := json.NewDecoder(strings.NewReader(string(raw)))
81 tok, err := dec.Token()
82 require.NoError(t, err)
83 require.Equal(t, json.Delim('{'), tok)
84 var keys []string
85 for dec.More() {
86 k, err := dec.Token()
87 require.NoError(t, err)
88 keys = append(keys, k.(string))
89 var skip json.RawMessage
90 require.NoError(t, dec.Decode(&skip))
91 }
92 return keys
93}
94
95// sameShape compares the shape of a recorded object with ours: a "scalar"
96// (numeric placeholder) accepts a number or a string. The description is a
97// property of the recorded repository, which ermite's test repository lacks.
98func sameShape(t *testing.T, want, got map[string]any, what string) {
99 t.Helper()
100 ws, gs := shape(want), shape(got)
101 for k, wt := range ws {
102 if k == "description" {
103 continue
104 }
105 if wt == "scalar" {
106 assert.Contains(t, []string{"number", "string"}, gs[k], "%s.%s", what, k)
107 continue
108 }
109 assert.Equal(t, wt, gs[k], "%s.%s", what, k)
110 }
111}
112
113// The github and job contexts of a push have the keys, the key order and the
114// value types of the ones github.com recorded for the same scenario shape
115// (ermite-conformance, ctx-job-runner), apart from the keys the runner adds.
116func TestCIContextsMatchTheGitHubRecording(t *testing.T) {
117 want := loadConformance(t)
118 c := newCIHarness(t, nil)
119 runner := c.registerRunner("alpha", "ubuntu-latest")
120 c.push("main", map[string]string{".github/workflows/ci.yml": workflowWith("branches: [main]")}, "")
121 job := runner.await()
122 var raw map[string]json.RawMessage
123 require.NoError(t, json.Unmarshal([]byte(job.GetContexts()), &raw))
124 var ctxs map[string]map[string]any
125 require.NoError(t, json.Unmarshal([]byte(job.GetContexts()), &ctxs))
126 runner.runJob(job, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
127
128 t.Run("github", func(t *testing.T) {
129 gh := ctxs["github"]
130 // The runner owns some keys and adds job and token; the server sends the rest.
131 serverKeys := slices.DeleteFunc(slices.Clone(want.GithubKeyOrder), func(k string) bool { return slices.Contains(want.RunnerOwned, k) })
132 assert.Equal(t, serverKeys, orderedKeys(t, raw["github"]), "key set and order")
133 serverGithub := map[string]any{}
134 for k, v := range want.Github {
135 if !slices.Contains(want.RunnerOwned, k) {
136 serverGithub[k] = v
137 }
138 }
139 sameShape(t, serverGithub, gh, "github")
140 assert.Equal(t, "Actions", gh["secret_source"])
141 assert.Equal(t, "10", gh["artifact_cache_size_limit"])
142 assert.Equal(t, "private", gh["repository_visibility"], "from the repository")
143 assert.Equal(t, "owner", gh["actor"])
144 assert.Equal(t, "owner", gh["repository_owner"])
145 assert.Regexp(t, `^[1-9][0-9]*$`, gh["actor_id"])
146 assert.Regexp(t, `^[1-9][0-9]*$`, gh["repository_id"])
147 assert.Regexp(t, `^[1-9][0-9]*$`, gh["repository_owner_id"])
148 assert.Equal(t, gh["actor_id"], gh["repository_owner_id"], "the owner pushed")
149 assert.Equal(t, "git://example.test/owner/proj.git", gh["repositoryUrl"], "git://<host>/<owner>/<repo>.git, as GitHub's")
150 })
151
152 t.Run("event", func(t *testing.T) {
153 ev := ctxs["github"]["event"].(map[string]any)
154 wev := want.Github["event"].(map[string]any)
155 assert.ElementsMatch(t, keysSlice(wev), keysSlice(ev), "event keys")
156 sameShape(t, wev, ev, "event")
157 repo, wrepo := ev["repository"].(map[string]any), wev["repository"].(map[string]any)
158 assert.Len(t, wrepo, 83, "the recording has 83 repository keys")
159 assert.ElementsMatch(t, keysSlice(wrepo), keysSlice(repo), "repository keys")
160 sameShape(t, wrepo, repo, "event.repository")
161 assert.ElementsMatch(t, keysSlice(wrepo["owner"].(map[string]any)), keysSlice(repo["owner"].(map[string]any)), "repository.owner keys")
162 sender, wsender := ev["sender"].(map[string]any), wev["sender"].(map[string]any)
163 assert.Len(t, wsender, 19)
164 assert.ElementsMatch(t, keysSlice(wsender), keysSlice(sender), "sender keys")
165 sameShape(t, wsender, sender, "event.sender")
166 assert.ElementsMatch(t, keysSlice(wev["pusher"].(map[string]any)), keysSlice(ev["pusher"].(map[string]any)))
167 for _, got := range []map[string]any{ev["head_commit"].(map[string]any), ev["commits"].([]any)[0].(map[string]any)} {
168 wc := wev["head_commit"].(map[string]any)
169 assert.ElementsMatch(t, keysSlice(wc), keysSlice(got), "commit keys: no added/removed/modified/username")
170 assert.ElementsMatch(t, keysSlice(wc["author"].(map[string]any)), keysSlice(got["author"].(map[string]any)))
171 assert.ElementsMatch(t, keysSlice(wc["committer"].(map[string]any)), keysSlice(got["committer"].(map[string]any)))
172 assert.True(t, strings.HasPrefix(got["url"].(string), publicOrigin+"/owner/proj/commit/"), got["url"])
173 }
174 assert.Equal(t, "proj", repo["name"])
175 assert.Equal(t, "owner/proj", repo["full_name"])
176 assert.Equal(t, true, repo["private"])
177 assert.Equal(t, "private", repo["visibility"])
178 assert.Equal(t, "main", repo["default_branch"])
179 assert.Equal(t, ctxs["github"]["repository_id"], jsonNumber(repo["id"]), "repository_id is repository.id")
180 assert.Equal(t, ctxs["github"]["actor_id"], jsonNumber(sender["id"]))
181 assert.Equal(t, ctxs["github"]["repository_owner_id"], jsonNumber(repo["owner"].(map[string]any)["id"]))
182 })
183
184 t.Run("job", func(t *testing.T) {
185 j := ctxs["job"]
186 // The runner adds status first; the server sends the rest, in GitHub's order.
187 serverKeys := slices.DeleteFunc(slices.Clone(want.JobKeyOrder), func(k string) bool { return k == "status" })
188 assert.Equal(t, serverKeys, orderedKeys(t, raw["job"]))
189 serverJob := map[string]any{}
190 for k, v := range want.Job {
191 if k != "status" {
192 serverJob[k] = v
193 }
194 }
195 sameShape(t, serverJob, j, "job")
196 assert.Equal(t, ".github/workflows/ci.yml", j["workflow_file_path"])
197 assert.Equal(t, "owner/proj", j["workflow_repository"])
198 assert.Equal(t, "owner/proj/.github/workflows/ci.yml@refs/heads/main", j["workflow_ref"])
199 assert.Equal(t, ctxs["github"]["sha"], j["workflow_sha"])
200 assert.Equal(t, ctxs["github"]["workflow_ref"], j["workflow_ref"])
201 })
202
203 t.Run("matrix is null without a matrix", func(t *testing.T) {
204 assert.Equal(t, "null", string(raw["matrix"]))
205 assert.Nil(t, want.Matrix)
206 assert.ElementsMatch(t, keysSlice(want.Strategy), keysSlice(ctxs["strategy"]))
207 })
208}
209
210func keysSlice(m map[string]any) []string {
211 out := make([]string, 0, len(m))
212 for k := range m {
213 out = append(out, k)
214 }
215 return out
216}
217
218func jsonNumber(v any) string {
219 b, _ := json.Marshal(v)
220 return string(b)
221}
222
223// The run title is the evaluated run-name, else the head commit message. The
224// name of the run is the evaluated run-name too (recorded: ctx-github-names,
225// name = title = the run-name), else the workflow name; github.workflow stays
226// the workflow name.
227func TestCIRunTitle(t *testing.T) {
228 c := newCIHarness(t, nil)
229 c.registerRunner("alpha", "ubuntu-latest")
230 c.push("main", map[string]string{".github/workflows/named.yml": `name: Named
231run-name: "run ${{ github.event_name }} on ${{ github.ref_type }} by ${{ github.actor == '' && 'nobody' || 'someone' }}"
232on: push
233jobs:
234 j:
235 runs-on: ubuntu-latest
236 steps:
237 - run: echo hi
238`, ".github/workflows/plain.yml": workflowWith("branches: [main]")}, "the head commit\n\nbody")
239 c.waitFor("two runs", func() bool { return len(c.runs()) == 2 })
240 titles, names, workflows := map[string]string{}, map[string]string{}, map[string]string{}
241 for _, r := range c.runs() {
242 titles[r.GetWorkflowPath()] = r.GetTitle()
243 names[r.GetWorkflowPath()] = r.GetName()
244 workflows[r.GetWorkflowPath()] = r.GetWorkflowName()
245 }
246 assert.Equal(t, map[string]string{
247 ".github/workflows/named.yml": "run push on branch by someone",
248 ".github/workflows/plain.yml": "the head commit",
249 }, titles)
250 assert.Equal(t, map[string]string{
251 ".github/workflows/named.yml": "run push on branch by someone",
252 ".github/workflows/plain.yml": ".github/workflows/plain.yml",
253 }, names, "the run-name, else the workflow name")
254 assert.Equal(t, map[string]string{
255 ".github/workflows/named.yml": "Named",
256 ".github/workflows/plain.yml": ".github/workflows/plain.yml",
257 }, workflows, "workflow_name is unchanged by a run-name")
258}
259
260// An over-256 matrix is an invalid workflow, like on github.com
261// (matrix-over-256): a failed run with no job.
262func TestCIMatrixOver256IsAnInvalidWorkflow(t *testing.T) {
263 c := newCIHarness(t, nil)
264 c.registerRunner("alpha", "ubuntu-latest")
265 var a, b []string
266 for i := 1; i <= 17; i++ {
267 a = append(a, itoa(i))
268 }
269 for i := 1; i <= 16; i++ {
270 b = append(b, itoa(i))
271 }
272 c.push("main", map[string]string{".github/workflows/m.yml": `on: push
273jobs:
274 too-many:
275 runs-on: ubuntu-latest
276 strategy:
277 matrix:
278 a: [` + strings.Join(a, ", ") + `]
279 b: [` + strings.Join(b, ", ") + `]
280 steps:
281 - run: echo
282`}, "")
283 c.waitFor("the run fails", func() bool { return len(c.runs()) == 1 && c.runs()[0].GetStatus() == pb.CiStatus_CI_STATUS_COMPLETED })
284 run := c.getRun(c.runs()[0].GetId())
285 assert.Equal(t, pb.CiConclusion_CI_CONCLUSION_FAILURE, run.GetRun().GetConclusion())
286 assert.Empty(t, run.GetJobs(), "no job")
287 assert.Equal(t, ".github/workflows/m.yml", run.GetRun().GetWorkflowName(), "no name: in this file: the path")
288 assert.Contains(t, run.GetRun().GetError(), "Strategy for job 'too-many' produced 272 configurations which exceeds the maximum of 256 configurations")
289}
290
291func itoa(i int) string { b, _ := json.Marshal(i); return string(b) }
292
293// A step with continue-on-error is no longer refused; the runner runs it.
294func TestCIStepContinueOnErrorIsPlanned(t *testing.T) {
295 c := newCIHarness(t, nil)
296 runner := c.registerRunner("alpha", "ubuntu-latest")
297 c.push("main", map[string]string{".github/workflows/s.yml": `on: push
298jobs:
299 j:
300 runs-on: ubuntu-latest
301 steps:
302 - continue-on-error: ${{ github.event_name == 'push' }}
303 run: exit 1
304`}, "")
305 job := runner.await()
306 assert.Equal(t, "j", job.GetJobKey())
307 runner.runJob(job, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
308}
309
310// CI-2: planning runs what CI-1 refused (uses, container, services, a step
311// timeout) and CI-3 environments, and still refuses job-level uses.
312func TestCIPlanningAcceptsCI2Features(t *testing.T) {
313 c := newCIHarness(t, nil)
314 runner := c.registerRunner("alpha", "ubuntu-latest")
315 c.push("main", map[string]string{".github/workflows/s.yml": `on: push
316jobs:
317 j:
318 runs-on: ubuntu-latest
319 container: alpine:3
320 services:
321 db:
322 image: postgres:16
323 steps:
324 - uses: actions/checkout@v4
325 - timeout-minutes: 1
326 run: echo
327`}, "")
328 job := runner.await()
329 assert.Equal(t, "j", job.GetJobKey())
330
331 for _, tc := range []struct{ name, job, want string }{
332 {"reusable", "uses: ./.github/workflows/x.yml", "not supported yet: reusable workflows (uses at the job level)"},
333 } {
334 t.Run(tc.name, func(t *testing.T) {
335 body := "on: push\njobs:\n r:\n " + tc.job + "\n"
336 c.push("main", map[string]string{".github/workflows/" + tc.name + ".yml": body}, "")
337 var got string
338 c.waitFor(tc.name+" fails", func() bool {
339 for _, r := range c.runs() {
340 if r.GetWorkflowPath() == ".github/workflows/"+tc.name+".yml" && r.GetStatus() == pb.CiStatus_CI_STATUS_COMPLETED {
341 if jobs := c.getRun(r.GetId()).GetJobs(); len(jobs) == 1 {
342 got = jobs[0].GetError()
343 return true
344 }
345 }
346 }
347 return false
348 })
349 assert.Equal(t, tc.want, got)
350 })
351 }
352}
353
354// A failed job with continue-on-error does not fail the run, keeps its own
355// failure conclusion, and shows as success to the jobs that need it (recorded:
356// if-job-level-fail-effects, where the dependent job printed result=success).
357func TestCIJobContinueOnError(t *testing.T) {
358 c := newCIHarness(t, nil)
359 runner := c.registerRunner("alpha", "ubuntu-latest")
360 c.push("main", map[string]string{".github/workflows/c.yml": `on: push
361jobs:
362 soft:
363 runs-on: ubuntu-latest
364 continue-on-error: ${{ github.event_name == 'push' }}
365 steps:
366 - run: exit 5
367 after:
368 needs: soft
369 runs-on: ubuntu-latest
370 steps:
371 - run: echo "result=${{ needs.soft.result }}"
372`}, "")
373 soft := runner.await()
374 assert.Equal(t, "soft", soft.GetJobKey())
375 runner.runJob(soft, pb.CiConclusion_CI_CONCLUSION_FAILURE, nil, "x")
376 after := runner.await()
377 assert.Equal(t, "after", after.GetJobKey(), "a needed job that failed softly does not skip its dependents")
378 assert.Contains(t, after.GetContexts(), `"result":"success"`)
379 runner.runJob(after, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
380 c.waitFor("the run ends", func() bool { return c.runs()[0].GetStatus() == pb.CiStatus_CI_STATUS_COMPLETED })
381 assert.Equal(t, pb.CiConclusion_CI_CONCLUSION_SUCCESS, c.runs()[0].GetConclusion(), "the run does not fail")
382 byKey := map[string]pb.CiConclusion{}
383 for _, j := range c.getRun(c.runs()[0].GetId()).GetJobs() {
384 byKey[j.GetName()] = j.GetConclusion()
385 }
386 assert.Equal(t, pb.CiConclusion_CI_CONCLUSION_FAILURE, byKey["soft"], "the job itself failed")
387}
388
389// A matrix leg with continue-on-error that fails does not cancel its siblings
390// (fail-fast), and does not fail the run; a leg without it still does.
391func TestCIJobContinueOnErrorLegDoesNotTriggerFailFast(t *testing.T) {
392 c := newCIHarness(t, nil)
393 runner := c.registerRunner("alpha", "ubuntu-latest")
394 c.push("main", map[string]string{".github/workflows/m.yml": `on: push
395jobs:
396 build:
397 runs-on: ubuntu-latest
398 continue-on-error: ${{ matrix.experimental }}
399 strategy:
400 matrix:
401 experimental: [true, false]
402 steps:
403 - run: echo ${{ matrix.experimental }}
404`}, "")
405 a, b := runner.await(), runner.await()
406 soft, hard := a, b
407 if strings.Contains(b.GetMatrix(), "true") {
408 soft, hard = b, a
409 }
410 apiHard := runner.jobAPI(hard)
411 _, err := apiHard.runner.Heartbeat(c.ctx, connect.NewRequest(&pb.HeartbeatRequest{JobId: hard.GetJobId()}))
412 require.NoError(t, err)
413 runner.runJob(soft, pb.CiConclusion_CI_CONCLUSION_FAILURE, nil, "x")
414 // The sibling must not have been asked to stop.
415 watch, err := apiHard.runner.WatchJob(c.ctx, connect.NewRequest(&pb.WatchJobRequest{JobId: hard.GetJobId()}))
416 require.NoError(t, err)
417 done := make(chan *pb.CancelReason, 1)
418 go func() {
419 for watch.Receive() {
420 if m := watch.Msg().GetCancel(); m != nil {
421 r := m.GetReason()
422 done <- &r
423 return
424 }
425 }
426 done <- nil
427 }()
428 runner.runJob(hard, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
429 c.waitFor("the run ends", func() bool { return c.runs()[0].GetStatus() == pb.CiStatus_CI_STATUS_COMPLETED })
430 assert.Equal(t, pb.CiConclusion_CI_CONCLUSION_SUCCESS, c.runs()[0].GetConclusion())
431 if r := <-done; r != nil {
432 t.Fatalf("the sibling of a continue-on-error leg was cancelled: %v", *r)
433 }
434}
435
436// GitHub-shaped URLs, /<owner>/<repo>/..., redirect to ermite's pages, only for
437// the owner of the instance and only where there is no page of its own.
438func TestWebRedirectsOwnerURLs(t *testing.T) {
439 c := newCIHarness(t, nil)
440 b := browser{c.harness}
441 c.push("main", map[string]string{"README": "x\n"}, "")
442 sha := strings.TrimSpace(c.mustGit(c.wt, "rev-parse", "HEAD"))
443 for path, to := range map[string]string{
444 "/owner/proj": "/proj",
445 "/owner/proj/commit/" + sha: "/proj/commit/" + sha,
446 "/owner/proj/compare/a...b?x=1": "/proj/compare/a...b?x=1",
447 } {
448 res := b.get(path, nil)
449 assert.Equal(t, http.StatusPermanentRedirect, res.status, path)
450 assert.Equal(t, to, res.hdr.Get("Location"), path)
451 }
452 // The git URLs of the owner are served by git over HTTP, not redirected
453 // (checkout clones <server>/<owner>/<repo>): the private repository asks
454 // for credentials, a dumb request has no route.
455 for path, want := range map[string]int{
456 "/owner/proj.git/info/refs?service=git-upload-pack": http.StatusUnauthorized,
457 "/owner/proj/info/refs?service=git-upload-pack": http.StatusUnauthorized,
458 "/other/proj/info/refs?service=git-upload-pack": http.StatusNotFound,
459 "/owner/proj.git/info/refs": http.StatusNotFound,
460 } {
461 assert.Equal(t, want, b.get(path, nil).status, path)
462 }
463 assert.Equal(t, http.StatusNotFound, b.get("/other/proj/commit/"+sha, nil).status, "not the owner")
464 assert.Equal(t, http.StatusNotFound, b.get("/owner", nil).status, "a bare owner is no page")
465 assert.Equal(t, http.StatusNotFound, b.get("/owner/Bad Name", nil).status)
466 post := b.do("POST", "/owner/proj", map[string]string{"Origin": publicOrigin}, "")
467 assert.Equal(t, http.StatusForbidden, post.status, "a write is stopped by the gate, never redirected")
468 assert.Empty(t, post.hdr.Get("Location"))
469}
470
471// github.job is empty while a job is evaluated, so a name that reads it loses
472// it (recorded: ctx-github-names), but the job gets its key in the contexts.
473func TestCIJobNameReadsAnEmptyGithubJob(t *testing.T) {
474 c := newCIHarness(t, nil)
475 runner := c.registerRunner("alpha", "ubuntu-latest")
476 c.push("main", map[string]string{".github/workflows/n.yml": `on: push
477jobs:
478 first-job:
479 name: Display name of ${{ github.job }}
480 runs-on: ubuntu-latest
481 steps:
482 - run: echo hi
483`}, "")
484 job := runner.await()
485 assert.Contains(t, job.GetContexts(), `"job":"first-job"`)
486 runner.runJob(job, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
487 c.waitFor("the run ends", func() bool { return c.runs()[0].GetStatus() == pb.CiStatus_CI_STATUS_COMPLETED })
488 assert.Equal(t, "Display name of", c.getRun(c.runs()[0].GetId()).GetJobs()[0].GetName())
489}
490
491// A tag push carries the commits that no branch has, newest as head_commit, and
492// the compare URL of several commits is "<first>^...<last>" (recorded:
493// skip-directive-tag, where no branch exists).
494func TestCITagEventCarriesTheNewCommits(t *testing.T) {
495 c := newCIHarness(t, nil)
496 runner := c.registerRunner("alpha", "ubuntu-latest")
497 c.push("main", map[string]string{".github/workflows/rel.yml": workflowWith("tags: ['v*']")}, "")
498 c.mustGit(c.wt, "checkout", "-q", "-b", "rel")
499 c.writeFile(c.wt, "a.txt", []byte("a\n")) // commits "add a.txt"
500 first := strings.TrimSpace(c.mustGit(c.wt, "rev-parse", "HEAD"))
501 c.writeFile(c.wt, "b.txt", []byte("b\n")) // commits "add b.txt"
502 last := strings.TrimSpace(c.mustGit(c.wt, "rev-parse", "HEAD"))
503 c.mustGit(c.wt, "tag", "v1")
504 c.mustGit(c.wt, "push", "-q", c.urlAuth(adminToken, "/proj.git"), "v1")
505
506 job := runner.await()
507 var ctxs map[string]map[string]any
508 require.NoError(t, json.Unmarshal([]byte(job.GetContexts()), &ctxs))
509 runner.runJob(job, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
510 ev := ctxs["github"]["event"].(map[string]any)
511 var msgs []string
512 for _, cm := range ev["commits"].([]any) {
513 msgs = append(msgs, cm.(map[string]any)["message"].(string))
514 }
515 assert.Equal(t, []string{"add a.txt", "add b.txt"}, msgs)
516 assert.Equal(t, "add b.txt", ev["head_commit"].(map[string]any)["message"])
517 assert.Equal(t, publicOrigin+"/owner/proj/compare/"+first[:12]+"^..."+last[:12], ev["compare"])
518 assert.Equal(t, "tag", ctxs["github"]["ref_type"])
519 assert.Equal(t, true, ev["created"])
520}
521
522// tagEvent pushes the tag and returns the push event and the title of the run
523// it starts, after `want` runs exist.
524func (c *ciHarness) tagEvent(runner *fakeRunner, tag string, want int) (map[string]any, string) {
525 c.t.Helper()
526 c.mustGit(c.wt, "tag", tag)
527 c.mustGit(c.wt, "push", "-q", c.urlAuth(adminToken, "/proj.git"), tag)
528 job := runner.await()
529 var ctxs map[string]map[string]any
530 require.NoError(c.t, json.Unmarshal([]byte(job.GetContexts()), &ctxs))
531 runner.runJob(job, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
532 c.waitFor("the run exists", func() bool { return len(c.runs()) == want })
533 var title string
534 for _, r := range c.runs() {
535 if r.GetRef() == "refs/tags/"+tag {
536 title = r.GetTitle()
537 }
538 }
539 return ctxs["github"]["event"].(map[string]any), title
540}
541
542func eventMessages(ev map[string]any) []string {
543 var msgs []string
544 for _, cm := range ev["commits"].([]any) {
545 msgs = append(msgs, cm.(map[string]any)["message"].(string))
546 }
547 return msgs
548}
549
550// A tag on a commit that a branch has carries that commit alone, and the
551// compare URL is its page, not a range from the previous push (recorded:
552// filter-tags, filter-paths-tag, filter-branches-ignore). The tag run is
553// titled with the workflow name (filter-*).
554func TestCITagEventOfACommitABranchHas(t *testing.T) {
555 c := newCIHarness(t, nil)
556 runner := c.registerRunner("alpha", "ubuntu-latest")
557 // Two commits on main, pushed with the workflow; both are on a branch.
558 c.push("main", map[string]string{".github/workflows/rel.yml": "name: Release\non:\n push:\n tags: ['v*']\njobs:\n j:\n runs-on: ubuntu-latest\n steps:\n - run: echo hi\n"}, "")
559 older := strings.TrimSpace(c.mustGit(c.wt, "rev-parse", "HEAD"))
560 c.writeFile(c.wt, "x.txt", []byte("x\n"))
561 c.mustGit(c.wt, "push", "-q", c.urlAuth(adminToken, "/proj.git"), "main")
562 newer := strings.TrimSpace(c.mustGit(c.wt, "rev-parse", "HEAD"))
563
564 ev, title := c.tagEvent(runner, "v1", 1)
565 assert.Equal(t, []string{"add x.txt"}, eventMessages(ev), "the tagged commit alone, not the push range")
566 assert.Equal(t, newer, ev["head_commit"].(map[string]any)["id"])
567 assert.Equal(t, publicOrigin+"/owner/proj/commit/"+newer[:12], ev["compare"])
568 assert.Equal(t, "Release", title, "a tag run is titled with the workflow name, not the commit message")
569
570 // A tag on an older commit of the branch: that commit alone.
571 c.mustGit(c.wt, "tag", "v0", older)
572 c.mustGit(c.wt, "push", "-q", c.urlAuth(adminToken, "/proj.git"), "v0")
573 job := runner.await()
574 var ctxs map[string]map[string]any
575 require.NoError(t, json.Unmarshal([]byte(job.GetContexts()), &ctxs))
576 runner.runJob(job, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
577 ev = ctxs["github"]["event"].(map[string]any)
578 assert.Equal(t, older, ev["after"])
579 assert.Equal(t, older, ev["head_commit"].(map[string]any)["id"])
580 assert.Equal(t, publicOrigin+"/owner/proj/commit/"+older[:12], ev["compare"])
581}
582
583// A tag on a new commit on top of the branch (recorded: push-tag) carries that
584// commit alone: the commit of the branch under it is not part of the event.
585func TestCITagEventOfANewCommitOnTopOfABranch(t *testing.T) {
586 c := newCIHarness(t, nil)
587 runner := c.registerRunner("alpha", "ubuntu-latest")
588 c.push("main", map[string]string{".github/workflows/ci.yml": "on:\n push:\n branches: [main]\n tags: ['v*']\njobs:\n j:\n runs-on: ubuntu-latest\n steps:\n - run: echo hi\n"}, "branch commit")
589 first := runner.await()
590 runner.runJob(first, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
591 c.waitFor("the branch run ends", func() bool { return len(c.runs()) == 1 && c.runs()[0].GetStatus() == pb.CiStatus_CI_STATUS_COMPLETED })
592 assert.Equal(t, "branch commit", c.runs()[0].GetTitle(), "a branch run keeps the commit message as title")
593
594 c.writeFile(c.wt, "t.txt", []byte("t\n")) // "add t.txt", only the tag is pushed
595 tagged := strings.TrimSpace(c.mustGit(c.wt, "rev-parse", "HEAD"))
596 ev, title := c.tagEvent(runner, "v1", 2)
597 assert.Equal(t, []string{"add t.txt"}, eventMessages(ev))
598 assert.Equal(t, publicOrigin+"/owner/proj/commit/"+tagged[:12], ev["compare"])
599 assert.Equal(t, ".github/workflows/ci.yml", title, "no name:, so the workflow name is the path of the file (assumed, not recorded)")
600 assert.Equal(t, true, ev["created"])
601 assert.Equal(t, domain.ZeroOID, ev["before"])
602}
603
604// Several tags in one push start one run each, with the same event: the
605// commit alone (recorded: push-tags-limit), titled with the workflow name.
606func TestCIThreeTagsInOnePush(t *testing.T) {
607 c := newCIHarness(t, nil)
608 runner := c.registerRunner("alpha", "ubuntu-latest")
609 c.push("main", map[string]string{".github/workflows/t.yml": "name: Tags\non:\n push:\n tags: ['v*']\njobs:\n j:\n runs-on: ubuntu-latest\n steps:\n - run: echo hi\n"}, "")
610 c.writeFile(c.wt, "t.txt", []byte("t\n"))
611 sha := strings.TrimSpace(c.mustGit(c.wt, "rev-parse", "HEAD"))
612 for _, tag := range []string{"v1", "v2", "v3"} {
613 c.mustGit(c.wt, "tag", tag)
614 }
615 c.mustGit(c.wt, "push", "-q", c.urlAuth(adminToken, "/proj.git"), "v1", "v2", "v3")
616 for range 3 {
617 job := runner.await()
618 var ctxs map[string]map[string]any
619 require.NoError(t, json.Unmarshal([]byte(job.GetContexts()), &ctxs))
620 ev := ctxs["github"]["event"].(map[string]any)
621 assert.Equal(t, []string{"add t.txt"}, eventMessages(ev))
622 assert.Equal(t, publicOrigin+"/owner/proj/commit/"+sha[:12], ev["compare"])
623 runner.runJob(job, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
624 }
625 c.waitFor("three runs", func() bool { return len(c.runs()) == 3 })
626 for _, r := range c.runs() {
627 assert.Equal(t, "Tags", r.GetTitle())
628 }
629}
630
631// A tag run without run-name is titled with the name of the workflow registered
632// for its path: the "name:" of the file on the default branch, which can differ
633// from the file at the tag (recorded: push-tag, push-tags-limit, push-delete-tag
634// and skip-directive-tag, whose ci.yml is "push-branch-create" on the default
635// branch of the recording repository and has the scenario's own name at the
636// tag; and the files with a unique name, where both agree). Its name stays the
637// workflow name of the tagged file.
638func TestCITagTitleIsTheNameRegisteredOnTheDefaultBranch(t *testing.T) {
639 c := newCIHarness(t, nil)
640 runner := c.registerRunner("alpha", "ubuntu-latest")
641 wf := func(name string) map[string]string {
642 return map[string]string{".github/workflows/ci.yml": "name: " + name + "\non:\n push:\n tags: ['v*']\njobs:\n j:\n runs-on: ubuntu-latest\n steps:\n - run: echo hi\n"}
643 }
644 c.push("main", wf("registered"), "")
645 // The tagged commit is off every branch and renames the workflow.
646 c.mustGit(c.wt, "checkout", "-q", "--detach")
647 c.writeFile(c.wt, ".github/workflows/ci.yml", []byte(wf("renamed")[".github/workflows/ci.yml"]))
648 _, title := c.tagEvent(runner, "v1", 1)
649 assert.Equal(t, "registered", title, "the name of ci.yml on the default branch")
650 r := c.runs()[0]
651 assert.Equal(t, "renamed", r.GetWorkflowName(), "the workflow of the run is the file at the tag (github.workflow)")
652 assert.Equal(t, "renamed", r.GetName(), "and so is the name of the run, which has no run-name")
653
654 // A file the default branch does not have: the run's own workflow name.
655 c.writeFile(c.wt, ".github/workflows/other.yml", []byte("name: other\non:\n push:\n tags: ['w*']\njobs:\n j:\n runs-on: ubuntu-latest\n steps:\n - run: echo hi\n"))
656 _, title = c.tagEvent(runner, "w1", 2)
657 assert.Equal(t, "other", title)
658}
659
660// A tag run with a run-name keeps the evaluated run-name as its title (the
661// run-name of a tag push is not recorded; branch pushes are, ctx-github-names).
662func TestCITagRunKeepsItsRunName(t *testing.T) {
663 c := newCIHarness(t, nil)
664 runner := c.registerRunner("alpha", "ubuntu-latest")
665 c.push("main", map[string]string{".github/workflows/t.yml": "name: Tags\nrun-name: \"release ${{ github.ref_name }}\"\non:\n push:\n tags: ['v*']\njobs:\n j:\n runs-on: ubuntu-latest\n steps:\n - run: echo hi\n"}, "")
666 _, title := c.tagEvent(runner, "v1", 1)
667 assert.Equal(t, "release v1", title)
668}
669
670// An exclude entry with a key that is no key of the matrix makes the workflow
671// invalid, and the run is named after the file (recorded: matrix-edge-values,
672// a-exclude-unknown-key.yml: name = path, no job), while an over-256 matrix
673// keeps the name of the workflow (matrix-over-256).
674func TestCIInvalidStrategyRunNames(t *testing.T) {
675 c := newCIHarness(t, nil)
676 c.registerRunner("alpha", "ubuntu-latest")
677 var a, b []string
678 for i := 1; i <= 17; i++ {
679 a = append(a, itoa(i))
680 }
681 for i := 1; i <= 16; i++ {
682 b = append(b, itoa(i))
683 }
684 c.push("main", map[string]string{
685 ".github/workflows/over.yml": `name: matrix-over-256
686on: push
687jobs:
688 too-many:
689 runs-on: ubuntu-latest
690 strategy:
691 matrix:
692 a: [` + strings.Join(a, ", ") + `]
693 b: [` + strings.Join(b, ", ") + `]
694 steps:
695 - run: echo
696`,
697 ".github/workflows/exclude.yml": `name: a-exclude-unknown-key
698on: push
699jobs:
700 main:
701 runs-on: ubuntu-latest
702 strategy:
703 matrix:
704 os: [one, two]
705 exclude:
706 - nokey: x
707 - os: one
708 nokey: y
709 steps:
710 - run: echo "os=${{ matrix.os }}"
711`,
712 ".github/workflows/noname.yml": `on: push
713jobs:
714 main:
715 runs-on: ubuntu-latest
716 strategy:
717 matrix:
718 os: [one]
719 exclude:
720 - nokey: x
721 steps:
722 - run: echo
723`}, "")
724 c.waitFor("three failed runs", func() bool {
725 rs := c.runs()
726 if len(rs) != 3 {
727 return false
728 }
729 for _, r := range rs {
730 if r.GetStatus() != pb.CiStatus_CI_STATUS_COMPLETED {
731 return false
732 }
733 }
734 return true
735 })
736 names := map[string]string{}
737 for _, r := range c.runs() {
738 names[r.GetWorkflowPath()] = r.GetWorkflowName()
739 assert.Equal(t, pb.CiConclusion_CI_CONCLUSION_FAILURE, r.GetConclusion(), r.GetWorkflowPath())
740 assert.Empty(t, c.getRun(r.GetId()).GetJobs(), "no job: %s", r.GetWorkflowPath())
741 if r.GetWorkflowPath() == ".github/workflows/exclude.yml" {
742 one := ".github/workflows/exclude.yml: Matrix exclude key 'nokey' does not match any key within the matrix"
743 assert.Equal(t, one+"\n"+one, r.GetError(), "once per offending entry")
744 }
745 }
746 assert.Equal(t, map[string]string{
747 ".github/workflows/over.yml": "matrix-over-256",
748 ".github/workflows/exclude.yml": ".github/workflows/exclude.yml",
749 ".github/workflows/noname.yml": ".github/workflows/noname.yml",
750 }, names)
751 // The check of a failed run carries the same workflow name.
752 sha := strings.TrimSpace(c.mustGit(c.wt, "rev-parse", "HEAD"))
753 checkNames := map[string]string{}
754 for _, ck := range c.checks(sha) {
755 checkNames[ck.GetName()] = ck.GetWorkflowName()
756 }
757 assert.Equal(t, "matrix-over-256", checkNames[".github/workflows/over.yml"])
758 assert.Equal(t, ".github/workflows/exclude.yml", checkNames[".github/workflows/exclude.yml"])
759}
760
761// The default success() of a job looks at the whole chain of its needs, not
762// only at the direct ones (recorded: needs-failed-dependency): chain-after-always
763// needs with-always, which ran through always() after broken failed, and is
764// skipped. with-always and with-failure still run.
765func TestCIDefaultSuccessLooksAtTheWholeChain(t *testing.T) {
766 c := newCIHarness(t, nil)
767 runner := c.registerRunner("alpha", "ubuntu-latest")
768 c.push("main", map[string]string{".github/workflows/n.yml": `name: needs-failed-dependency
769on: push
770jobs:
771 broken:
772 runs-on: ubuntu-latest
773 steps:
774 - run: exit 1
775 plain:
776 needs: broken
777 runs-on: ubuntu-latest
778 steps:
779 - run: echo SHOULD-NOT-RUN
780 with-always:
781 needs: broken
782 if: always()
783 runs-on: ubuntu-latest
784 steps:
785 - run: echo "result=${{ needs.broken.result }}"
786 with-failure:
787 needs: broken
788 if: failure()
789 runs-on: ubuntu-latest
790 steps:
791 - run: echo "result=${{ needs.broken.result }}"
792 chain-after-always:
793 needs: with-always
794 runs-on: ubuntu-latest
795 steps:
796 - run: echo chain-after-always-runs-because-with-always-succeeded
797 chain-with-failure:
798 needs: with-always
799 if: failure()
800 runs-on: ubuntu-latest
801 steps:
802 - run: echo "failure() is true when an ancestor failed"
803 chain-always:
804 needs: with-always
805 if: always()
806 runs-on: ubuntu-latest
807 steps:
808 - run: echo always
809`}, "")
810 broken := runner.await()
811 assert.Equal(t, "broken", broken.GetJobKey())
812 runner.runJob(broken, pb.CiConclusion_CI_CONCLUSION_FAILURE, nil, "x")
813 ran := map[string]bool{}
814 for range 4 {
815 j := runner.await()
816 ran[j.GetJobKey()] = true
817 runner.runJob(j, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
818 }
819 assert.Equal(t, map[string]bool{"with-always": true, "with-failure": true, "chain-with-failure": true, "chain-always": true}, ran)
820 c.waitFor("the run ends", func() bool { return c.runs()[0].GetStatus() == pb.CiStatus_CI_STATUS_COMPLETED })
821 got := map[string]pb.CiConclusion{}
822 for _, j := range c.getRun(c.runs()[0].GetId()).GetJobs() {
823 got[j.GetName()] = j.GetConclusion()
824 }
825 skipped, ok := pb.CiConclusion_CI_CONCLUSION_SKIPPED, pb.CiConclusion_CI_CONCLUSION_SUCCESS
826 assert.Equal(t, map[string]pb.CiConclusion{
827 "broken": pb.CiConclusion_CI_CONCLUSION_FAILURE, "plain": skipped, "with-always": ok, "with-failure": ok,
828 "chain-after-always": skipped, "chain-with-failure": ok, "chain-always": ok,
829 }, got)
830}