1// SPDX-License-Identifier: AGPL-3.0-or-later
2
3package app
4
5import (
6 "encoding/json"
7 "net/http"
8 "os"
9 "slices"
10 "strings"
11 "testing"
12
13 "connectrpc.com/connect"
14 "github.com/stretchr/testify/assert"
15 "github.com/stretchr/testify/require"
16
17 pb "bnema.dev/ermite-api/gen/ermite"
18 "bnema.dev/ermite/internal/domain"
19)
20
21// conformance is testdata/conformance/ctx-job-runner.contexts.json: the
22// contexts github.com gave to the ctx-job-runner scenario of ermite-conformance
23// (see its _source).
24type conformance struct {
25 GithubKeyOrder []string `json:"_github_key_order"`
26 JobKeyOrder []string `json:"_job_key_order"`
27 RunnerOwned []string `json:"_runner_owned_github_keys"`
28 Github map[string]any `json:"github"`
29 Job map[string]any `json:"job"`
30 Matrix any `json:"matrix"`
31 Strategy map[string]any `json:"strategy"`
32}
33
34func loadConformance(t *testing.T) conformance {
35 t.Helper()
36 b, err := os.ReadFile("testdata/conformance/ctx-job-runner.contexts.json")
37 require.NoError(t, err)
38 var c conformance
39 require.NoError(t, json.Unmarshal(b, &c))
40 return c
41}
42
43// numericPlaceholders are the placeholders of the normaliser of
44// ermite-conformance for values that are numbers (or times) on github.com: ids,
45// times and the repository size. In the raw bundle of the same scenario id,
46// created_at, pushed_at and size are JSON numbers; the fixture only has the
47// placeholder, a string. They are matched by "number or string".
48var numericPlaceholders = []string{"<id:", "<time>", "<size>", "<actor_id:", "<repository_id:", "<repository_owner_id:", "<check_run_id:"}
49
50// shape is the type of each value of an object. A placeholder of a number
51// has the type "scalar".
52func shape(m map[string]any) map[string]string {
53 out := map[string]string{}
54 for k, v := range m {
55 if s, ok := v.(string); ok && slices.ContainsFunc(numericPlaceholders, func(p string) bool { return strings.HasPrefix(s, p) }) {
56 out[k] = "scalar"
57 continue
58 }
59 switch v.(type) {
60 case nil:
61 out[k] = "null"
62 case string:
63 out[k] = "string"
64 case float64:
65 out[k] = "number"
66 case bool:
67 out[k] = "bool"
68 case []any:
69 out[k] = "array"
70 case map[string]any:
71 out[k] = "object"
72 }
73 }
74 return out
75}
76
77// orderedKeys returns the keys of a JSON object in document order.
78func orderedKeys(t *testing.T, raw json.RawMessage) []string {
79 t.Helper()
80 dec := json.NewDecoder(strings.NewReader(string(raw)))
81 tok, err := dec.Token()
82 require.NoError(t, err)
83 require.Equal(t, json.Delim('{'), tok)
84 var keys []string
85 for dec.More() {
86 k, err := dec.Token()
87 require.NoError(t, err)
88 keys = append(keys, k.(string))
89 var skip json.RawMessage
90 require.NoError(t, dec.Decode(&skip))
91 }
92 return keys
93}
94
95// sameShape compares the shape of a recorded object with ours: a "scalar"
96// (numeric placeholder) accepts a number or a string. The description is a
97// property of the recorded repository, which ermite's test repository lacks.
98func sameShape(t *testing.T, want, got map[string]any, what string) {
99 t.Helper()
100 ws, gs := shape(want), shape(got)
101 for k, wt := range ws {
102 if k == "description" {
103 continue
104 }
105 if wt == "scalar" {
106 assert.Contains(t, []string{"number", "string"}, gs[k], "%s.%s", what, k)
107 continue
108 }
109 assert.Equal(t, wt, gs[k], "%s.%s", what, k)
110 }
111}
112
113// The github and job contexts of a push have the keys, the key order and the
114// value types of the ones github.com recorded for the same scenario shape
115// (ermite-conformance, ctx-job-runner), apart from the keys the runner adds.
116func TestCIContextsMatchTheGitHubRecording(t *testing.T) {
117 want := loadConformance(t)
118 c := newCIHarness(t, nil)
119 runner := c.registerRunner("alpha", "ubuntu-latest")
120 c.push("main", map[string]string{".github/workflows/ci.yml": workflowWith("branches: [main]")}, "")
121 job := runner.await()
122 var raw map[string]json.RawMessage
123 require.NoError(t, json.Unmarshal([]byte(job.GetContexts()), &raw))
124 var ctxs map[string]map[string]any
125 require.NoError(t, json.Unmarshal([]byte(job.GetContexts()), &ctxs))
126 runner.runJob(job, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
127
128 t.Run("github", func(t *testing.T) {
129 gh := ctxs["github"]
130 // The runner owns some keys and adds job and token; the server sends the rest.
131 serverKeys := slices.DeleteFunc(slices.Clone(want.GithubKeyOrder), func(k string) bool { return slices.Contains(want.RunnerOwned, k) })
132 assert.Equal(t, serverKeys, orderedKeys(t, raw["github"]), "key set and order")
133 serverGithub := map[string]any{}
134 for k, v := range want.Github {
135 if !slices.Contains(want.RunnerOwned, k) {
136 serverGithub[k] = v
137 }
138 }
139 sameShape(t, serverGithub, gh, "github")
140 assert.Equal(t, "Actions", gh["secret_source"])
141 assert.Equal(t, "10", gh["artifact_cache_size_limit"])
142 assert.Equal(t, "private", gh["repository_visibility"], "from the repository")
143 assert.Equal(t, "owner", gh["actor"])
144 assert.Equal(t, "owner", gh["repository_owner"])
145 assert.Regexp(t, `^[1-9][0-9]*$`, gh["actor_id"])
146 assert.Regexp(t, `^[1-9][0-9]*$`, gh["repository_id"])
147 assert.Regexp(t, `^[1-9][0-9]*$`, gh["repository_owner_id"])
148 assert.Equal(t, gh["actor_id"], gh["repository_owner_id"], "the owner pushed")
149 assert.Equal(t, "git://example.test/owner/proj.git", gh["repositoryUrl"], "git://<host>/<owner>/<repo>.git, as GitHub's")
150 })
151
152 t.Run("event", func(t *testing.T) {
153 ev := ctxs["github"]["event"].(map[string]any)
154 wev := want.Github["event"].(map[string]any)
155 assert.ElementsMatch(t, keysSlice(wev), keysSlice(ev), "event keys")
156 sameShape(t, wev, ev, "event")
157 repo, wrepo := ev["repository"].(map[string]any), wev["repository"].(map[string]any)
158 assert.Len(t, wrepo, 83, "the recording has 83 repository keys")
159 assert.ElementsMatch(t, keysSlice(wrepo), keysSlice(repo), "repository keys")
160 sameShape(t, wrepo, repo, "event.repository")
161 assert.ElementsMatch(t, keysSlice(wrepo["owner"].(map[string]any)), keysSlice(repo["owner"].(map[string]any)), "repository.owner keys")
162 sender, wsender := ev["sender"].(map[string]any), wev["sender"].(map[string]any)
163 assert.Len(t, wsender, 19)
164 assert.ElementsMatch(t, keysSlice(wsender), keysSlice(sender), "sender keys")
165 sameShape(t, wsender, sender, "event.sender")
166 assert.ElementsMatch(t, keysSlice(wev["pusher"].(map[string]any)), keysSlice(ev["pusher"].(map[string]any)))
167 for _, got := range []map[string]any{ev["head_commit"].(map[string]any), ev["commits"].([]any)[0].(map[string]any)} {
168 wc := wev["head_commit"].(map[string]any)
169 assert.ElementsMatch(t, keysSlice(wc), keysSlice(got), "commit keys: no added/removed/modified/username")
170 assert.ElementsMatch(t, keysSlice(wc["author"].(map[string]any)), keysSlice(got["author"].(map[string]any)))
171 assert.ElementsMatch(t, keysSlice(wc["committer"].(map[string]any)), keysSlice(got["committer"].(map[string]any)))
172 assert.True(t, strings.HasPrefix(got["url"].(string), publicOrigin+"/owner/proj/commit/"), got["url"])
173 }
174 assert.Equal(t, "proj", repo["name"])
175 assert.Equal(t, "owner/proj", repo["full_name"])
176 assert.Equal(t, true, repo["private"])
177 assert.Equal(t, "private", repo["visibility"])
178 assert.Equal(t, "main", repo["default_branch"])
179 assert.Equal(t, ctxs["github"]["repository_id"], jsonNumber(repo["id"]), "repository_id is repository.id")
180 assert.Equal(t, ctxs["github"]["actor_id"], jsonNumber(sender["id"]))
181 assert.Equal(t, ctxs["github"]["repository_owner_id"], jsonNumber(repo["owner"].(map[string]any)["id"]))
182 })
183
184 t.Run("job", func(t *testing.T) {
185 j := ctxs["job"]
186 // The runner adds status first; the server sends the rest, in GitHub's order.
187 serverKeys := slices.DeleteFunc(slices.Clone(want.JobKeyOrder), func(k string) bool { return k == "status" })
188 assert.Equal(t, serverKeys, orderedKeys(t, raw["job"]))
189 serverJob := map[string]any{}
190 for k, v := range want.Job {
191 if k != "status" {
192 serverJob[k] = v
193 }
194 }
195 sameShape(t, serverJob, j, "job")
196 assert.Equal(t, ".github/workflows/ci.yml", j["workflow_file_path"])
197 assert.Equal(t, "owner/proj", j["workflow_repository"])
198 assert.Equal(t, "owner/proj/.github/workflows/ci.yml@refs/heads/main", j["workflow_ref"])
199 assert.Equal(t, ctxs["github"]["sha"], j["workflow_sha"])
200 assert.Equal(t, ctxs["github"]["workflow_ref"], j["workflow_ref"])
201 })
202
203 t.Run("matrix is null without a matrix", func(t *testing.T) {
204 assert.Equal(t, "null", string(raw["matrix"]))
205 assert.Nil(t, want.Matrix)
206 assert.ElementsMatch(t, keysSlice(want.Strategy), keysSlice(ctxs["strategy"]))
207 })
208}
209
210func keysSlice(m map[string]any) []string {
211 out := make([]string, 0, len(m))
212 for k := range m {
213 out = append(out, k)
214 }
215 return out
216}
217
218func jsonNumber(v any) string {
219 b, _ := json.Marshal(v)
220 return string(b)
221}
222
223// The run title is the evaluated run-name, else the head commit message. The
224// name of the run is the evaluated run-name too (recorded: ctx-github-names,
225// name = title = the run-name), else the workflow name; github.workflow stays
226// the workflow name.
227func TestCIRunTitle(t *testing.T) {
228 c := newCIHarness(t, nil)
229 c.registerRunner("alpha", "ubuntu-latest")
230 c.push("main", map[string]string{".github/workflows/named.yml": `name: Named
231run-name: "run ${{ github.event_name }} on ${{ github.ref_type }} by ${{ github.actor == '' && 'nobody' || 'someone' }}"
232on: push
233jobs:
234 j:
235 runs-on: ubuntu-latest
236 steps:
237 - run: echo hi
238`, ".github/workflows/plain.yml": workflowWith("branches: [main]")}, "the head commit\n\nbody")
239 c.waitFor("two runs", func() bool { return len(c.runs()) == 2 })
240 titles, names, workflows := map[string]string{}, map[string]string{}, map[string]string{}
241 for _, r := range c.runs() {
242 titles[r.GetWorkflowPath()] = r.GetTitle()
243 names[r.GetWorkflowPath()] = r.GetName()
244 workflows[r.GetWorkflowPath()] = r.GetWorkflowName()
245 }
246 assert.Equal(t, map[string]string{
247 ".github/workflows/named.yml": "run push on branch by someone",
248 ".github/workflows/plain.yml": "the head commit",
249 }, titles)
250 assert.Equal(t, map[string]string{
251 ".github/workflows/named.yml": "run push on branch by someone",
252 ".github/workflows/plain.yml": ".github/workflows/plain.yml",
253 }, names, "the run-name, else the workflow name")
254 assert.Equal(t, map[string]string{
255 ".github/workflows/named.yml": "Named",
256 ".github/workflows/plain.yml": ".github/workflows/plain.yml",
257 }, workflows, "workflow_name is unchanged by a run-name")
258}
259
260// An over-256 matrix is an invalid workflow, like on github.com
261// (matrix-over-256): a failed run with no job.
262func TestCIMatrixOver256IsAnInvalidWorkflow(t *testing.T) {
263 c := newCIHarness(t, nil)
264 c.registerRunner("alpha", "ubuntu-latest")
265 var a, b []string
266 for i := 1; i <= 17; i++ {
267 a = append(a, itoa(i))
268 }
269 for i := 1; i <= 16; i++ {
270 b = append(b, itoa(i))
271 }
272 c.push("main", map[string]string{".github/workflows/m.yml": `on: push
273jobs:
274 too-many:
275 runs-on: ubuntu-latest
276 strategy:
277 matrix:
278 a: [` + strings.Join(a, ", ") + `]
279 b: [` + strings.Join(b, ", ") + `]
280 steps:
281 - run: echo
282`}, "")
283 c.waitFor("the run fails", func() bool { return len(c.runs()) == 1 && c.runs()[0].GetStatus() == pb.CiStatus_CI_STATUS_COMPLETED })
284 run := c.getRun(c.runs()[0].GetId())
285 assert.Equal(t, pb.CiConclusion_CI_CONCLUSION_FAILURE, run.GetRun().GetConclusion())
286 assert.Empty(t, run.GetJobs(), "no job")
287 assert.Equal(t, ".github/workflows/m.yml", run.GetRun().GetWorkflowName(), "no name: in this file: the path")
288 assert.Contains(t, run.GetRun().GetError(), "Strategy for job 'too-many' produced 272 configurations which exceeds the maximum of 256 configurations")
289}
290
291func itoa(i int) string { b, _ := json.Marshal(i); return string(b) }
292
293// A step with continue-on-error is no longer refused; the runner runs it.
294func TestCIStepContinueOnErrorIsPlanned(t *testing.T) {
295 c := newCIHarness(t, nil)
296 runner := c.registerRunner("alpha", "ubuntu-latest")
297 c.push("main", map[string]string{".github/workflows/s.yml": `on: push
298jobs:
299 j:
300 runs-on: ubuntu-latest
301 steps:
302 - continue-on-error: ${{ github.event_name == 'push' }}
303 run: exit 1
304`}, "")
305 job := runner.await()
306 assert.Equal(t, "j", job.GetJobKey())
307 runner.runJob(job, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
308}
309
310// CI-2: planning runs what CI-1 refused (uses, container, services, a step
311// timeout) and CI-3 environments, and still refuses job-level uses and
312// concurrency.
313func TestCIPlanningAcceptsCI2Features(t *testing.T) {
314 c := newCIHarness(t, nil)
315 runner := c.registerRunner("alpha", "ubuntu-latest")
316 c.push("main", map[string]string{".github/workflows/s.yml": `on: push
317jobs:
318 j:
319 runs-on: ubuntu-latest
320 container: alpine:3
321 services:
322 db:
323 image: postgres:16
324 steps:
325 - uses: actions/checkout@v4
326 - timeout-minutes: 1
327 run: echo
328`}, "")
329 job := runner.await()
330 assert.Equal(t, "j", job.GetJobKey())
331
332 for _, tc := range []struct{ name, job, want string }{
333 {"reusable", "uses: ./.github/workflows/x.yml", "not supported yet: reusable workflows (uses at the job level)"},
334 {"concurrency", "runs-on: ubuntu-latest\n concurrency: g\n steps:\n - run: echo", "not supported yet: concurrency"},
335 } {
336 t.Run(tc.name, func(t *testing.T) {
337 body := "on: push\njobs:\n r:\n " + tc.job + "\n"
338 c.push("main", map[string]string{".github/workflows/" + tc.name + ".yml": body}, "")
339 var got string
340 c.waitFor(tc.name+" fails", func() bool {
341 for _, r := range c.runs() {
342 if r.GetWorkflowPath() == ".github/workflows/"+tc.name+".yml" && r.GetStatus() == pb.CiStatus_CI_STATUS_COMPLETED {
343 if jobs := c.getRun(r.GetId()).GetJobs(); len(jobs) == 1 {
344 got = jobs[0].GetError()
345 return true
346 }
347 }
348 }
349 return false
350 })
351 assert.Equal(t, tc.want, got)
352 })
353 }
354}
355
356// A failed job with continue-on-error does not fail the run, keeps its own
357// failure conclusion, and shows as success to the jobs that need it (recorded:
358// if-job-level-fail-effects, where the dependent job printed result=success).
359func TestCIJobContinueOnError(t *testing.T) {
360 c := newCIHarness(t, nil)
361 runner := c.registerRunner("alpha", "ubuntu-latest")
362 c.push("main", map[string]string{".github/workflows/c.yml": `on: push
363jobs:
364 soft:
365 runs-on: ubuntu-latest
366 continue-on-error: ${{ github.event_name == 'push' }}
367 steps:
368 - run: exit 5
369 after:
370 needs: soft
371 runs-on: ubuntu-latest
372 steps:
373 - run: echo "result=${{ needs.soft.result }}"
374`}, "")
375 soft := runner.await()
376 assert.Equal(t, "soft", soft.GetJobKey())
377 runner.runJob(soft, pb.CiConclusion_CI_CONCLUSION_FAILURE, nil, "x")
378 after := runner.await()
379 assert.Equal(t, "after", after.GetJobKey(), "a needed job that failed softly does not skip its dependents")
380 assert.Contains(t, after.GetContexts(), `"result":"success"`)
381 runner.runJob(after, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
382 c.waitFor("the run ends", func() bool { return c.runs()[0].GetStatus() == pb.CiStatus_CI_STATUS_COMPLETED })
383 assert.Equal(t, pb.CiConclusion_CI_CONCLUSION_SUCCESS, c.runs()[0].GetConclusion(), "the run does not fail")
384 byKey := map[string]pb.CiConclusion{}
385 for _, j := range c.getRun(c.runs()[0].GetId()).GetJobs() {
386 byKey[j.GetName()] = j.GetConclusion()
387 }
388 assert.Equal(t, pb.CiConclusion_CI_CONCLUSION_FAILURE, byKey["soft"], "the job itself failed")
389}
390
391// A matrix leg with continue-on-error that fails does not cancel its siblings
392// (fail-fast), and does not fail the run; a leg without it still does.
393func TestCIJobContinueOnErrorLegDoesNotTriggerFailFast(t *testing.T) {
394 c := newCIHarness(t, nil)
395 runner := c.registerRunner("alpha", "ubuntu-latest")
396 c.push("main", map[string]string{".github/workflows/m.yml": `on: push
397jobs:
398 build:
399 runs-on: ubuntu-latest
400 continue-on-error: ${{ matrix.experimental }}
401 strategy:
402 matrix:
403 experimental: [true, false]
404 steps:
405 - run: echo ${{ matrix.experimental }}
406`}, "")
407 a, b := runner.await(), runner.await()
408 soft, hard := a, b
409 if strings.Contains(b.GetMatrix(), "true") {
410 soft, hard = b, a
411 }
412 apiHard := runner.jobAPI(hard)
413 _, err := apiHard.runner.Heartbeat(c.ctx, connect.NewRequest(&pb.HeartbeatRequest{JobId: hard.GetJobId()}))
414 require.NoError(t, err)
415 runner.runJob(soft, pb.CiConclusion_CI_CONCLUSION_FAILURE, nil, "x")
416 // The sibling must not have been asked to stop.
417 watch, err := apiHard.runner.WatchJob(c.ctx, connect.NewRequest(&pb.WatchJobRequest{JobId: hard.GetJobId()}))
418 require.NoError(t, err)
419 done := make(chan *pb.CancelReason, 1)
420 go func() {
421 for watch.Receive() {
422 if m := watch.Msg().GetCancel(); m != nil {
423 r := m.GetReason()
424 done <- &r
425 return
426 }
427 }
428 done <- nil
429 }()
430 runner.runJob(hard, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
431 c.waitFor("the run ends", func() bool { return c.runs()[0].GetStatus() == pb.CiStatus_CI_STATUS_COMPLETED })
432 assert.Equal(t, pb.CiConclusion_CI_CONCLUSION_SUCCESS, c.runs()[0].GetConclusion())
433 if r := <-done; r != nil {
434 t.Fatalf("the sibling of a continue-on-error leg was cancelled: %v", *r)
435 }
436}
437
438// GitHub-shaped URLs, /<owner>/<repo>/..., redirect to ermite's pages, only for
439// the owner of the instance and only where there is no page of its own.
440func TestWebRedirectsOwnerURLs(t *testing.T) {
441 c := newCIHarness(t, nil)
442 b := browser{c.harness}
443 c.push("main", map[string]string{"README": "x\n"}, "")
444 sha := strings.TrimSpace(c.mustGit(c.wt, "rev-parse", "HEAD"))
445 for path, to := range map[string]string{
446 "/owner/proj": "/proj",
447 "/owner/proj/commit/" + sha: "/proj/commit/" + sha,
448 "/owner/proj/compare/a...b?x=1": "/proj/compare/a...b?x=1",
449 } {
450 res := b.get(path, nil)
451 assert.Equal(t, http.StatusPermanentRedirect, res.status, path)
452 assert.Equal(t, to, res.hdr.Get("Location"), path)
453 }
454 // The git URLs of the owner are served by git over HTTP, not redirected
455 // (checkout clones <server>/<owner>/<repo>): the private repository asks
456 // for credentials, a dumb request has no route.
457 for path, want := range map[string]int{
458 "/owner/proj.git/info/refs?service=git-upload-pack": http.StatusUnauthorized,
459 "/owner/proj/info/refs?service=git-upload-pack": http.StatusUnauthorized,
460 "/other/proj/info/refs?service=git-upload-pack": http.StatusNotFound,
461 "/owner/proj.git/info/refs": http.StatusNotFound,
462 } {
463 assert.Equal(t, want, b.get(path, nil).status, path)
464 }
465 assert.Equal(t, http.StatusNotFound, b.get("/other/proj/commit/"+sha, nil).status, "not the owner")
466 assert.Equal(t, http.StatusNotFound, b.get("/owner", nil).status, "a bare owner is no page")
467 assert.Equal(t, http.StatusNotFound, b.get("/owner/Bad Name", nil).status)
468 post := b.do("POST", "/owner/proj", map[string]string{"Origin": publicOrigin}, "")
469 assert.Equal(t, http.StatusForbidden, post.status, "a write is stopped by the gate, never redirected")
470 assert.Empty(t, post.hdr.Get("Location"))
471}
472
473// github.job is empty while a job is evaluated, so a name that reads it loses
474// it (recorded: ctx-github-names), but the job gets its key in the contexts.
475func TestCIJobNameReadsAnEmptyGithubJob(t *testing.T) {
476 c := newCIHarness(t, nil)
477 runner := c.registerRunner("alpha", "ubuntu-latest")
478 c.push("main", map[string]string{".github/workflows/n.yml": `on: push
479jobs:
480 first-job:
481 name: Display name of ${{ github.job }}
482 runs-on: ubuntu-latest
483 steps:
484 - run: echo hi
485`}, "")
486 job := runner.await()
487 assert.Contains(t, job.GetContexts(), `"job":"first-job"`)
488 runner.runJob(job, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
489 c.waitFor("the run ends", func() bool { return c.runs()[0].GetStatus() == pb.CiStatus_CI_STATUS_COMPLETED })
490 assert.Equal(t, "Display name of", c.getRun(c.runs()[0].GetId()).GetJobs()[0].GetName())
491}
492
493// A tag push carries the commits that no branch has, newest as head_commit, and
494// the compare URL of several commits is "<first>^...<last>" (recorded:
495// skip-directive-tag, where no branch exists).
496func TestCITagEventCarriesTheNewCommits(t *testing.T) {
497 c := newCIHarness(t, nil)
498 runner := c.registerRunner("alpha", "ubuntu-latest")
499 c.push("main", map[string]string{".github/workflows/rel.yml": workflowWith("tags: ['v*']")}, "")
500 c.mustGit(c.wt, "checkout", "-q", "-b", "rel")
501 c.writeFile(c.wt, "a.txt", []byte("a\n")) // commits "add a.txt"
502 first := strings.TrimSpace(c.mustGit(c.wt, "rev-parse", "HEAD"))
503 c.writeFile(c.wt, "b.txt", []byte("b\n")) // commits "add b.txt"
504 last := strings.TrimSpace(c.mustGit(c.wt, "rev-parse", "HEAD"))
505 c.mustGit(c.wt, "tag", "v1")
506 c.mustGit(c.wt, "push", "-q", c.urlAuth(adminToken, "/proj.git"), "v1")
507
508 job := runner.await()
509 var ctxs map[string]map[string]any
510 require.NoError(t, json.Unmarshal([]byte(job.GetContexts()), &ctxs))
511 runner.runJob(job, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
512 ev := ctxs["github"]["event"].(map[string]any)
513 var msgs []string
514 for _, cm := range ev["commits"].([]any) {
515 msgs = append(msgs, cm.(map[string]any)["message"].(string))
516 }
517 assert.Equal(t, []string{"add a.txt", "add b.txt"}, msgs)
518 assert.Equal(t, "add b.txt", ev["head_commit"].(map[string]any)["message"])
519 assert.Equal(t, publicOrigin+"/owner/proj/compare/"+first[:12]+"^..."+last[:12], ev["compare"])
520 assert.Equal(t, "tag", ctxs["github"]["ref_type"])
521 assert.Equal(t, true, ev["created"])
522}
523
524// tagEvent pushes the tag and returns the push event and the title of the run
525// it starts, after `want` runs exist.
526func (c *ciHarness) tagEvent(runner *fakeRunner, tag string, want int) (map[string]any, string) {
527 c.t.Helper()
528 c.mustGit(c.wt, "tag", tag)
529 c.mustGit(c.wt, "push", "-q", c.urlAuth(adminToken, "/proj.git"), tag)
530 job := runner.await()
531 var ctxs map[string]map[string]any
532 require.NoError(c.t, json.Unmarshal([]byte(job.GetContexts()), &ctxs))
533 runner.runJob(job, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
534 c.waitFor("the run exists", func() bool { return len(c.runs()) == want })
535 var title string
536 for _, r := range c.runs() {
537 if r.GetRef() == "refs/tags/"+tag {
538 title = r.GetTitle()
539 }
540 }
541 return ctxs["github"]["event"].(map[string]any), title
542}
543
544func eventMessages(ev map[string]any) []string {
545 var msgs []string
546 for _, cm := range ev["commits"].([]any) {
547 msgs = append(msgs, cm.(map[string]any)["message"].(string))
548 }
549 return msgs
550}
551
552// A tag on a commit that a branch has carries that commit alone, and the
553// compare URL is its page, not a range from the previous push (recorded:
554// filter-tags, filter-paths-tag, filter-branches-ignore). The tag run is
555// titled with the workflow name (filter-*).
556func TestCITagEventOfACommitABranchHas(t *testing.T) {
557 c := newCIHarness(t, nil)
558 runner := c.registerRunner("alpha", "ubuntu-latest")
559 // Two commits on main, pushed with the workflow; both are on a branch.
560 c.push("main", map[string]string{".github/workflows/rel.yml": "name: Release\non:\n push:\n tags: ['v*']\njobs:\n j:\n runs-on: ubuntu-latest\n steps:\n - run: echo hi\n"}, "")
561 older := strings.TrimSpace(c.mustGit(c.wt, "rev-parse", "HEAD"))
562 c.writeFile(c.wt, "x.txt", []byte("x\n"))
563 c.mustGit(c.wt, "push", "-q", c.urlAuth(adminToken, "/proj.git"), "main")
564 newer := strings.TrimSpace(c.mustGit(c.wt, "rev-parse", "HEAD"))
565
566 ev, title := c.tagEvent(runner, "v1", 1)
567 assert.Equal(t, []string{"add x.txt"}, eventMessages(ev), "the tagged commit alone, not the push range")
568 assert.Equal(t, newer, ev["head_commit"].(map[string]any)["id"])
569 assert.Equal(t, publicOrigin+"/owner/proj/commit/"+newer[:12], ev["compare"])
570 assert.Equal(t, "Release", title, "a tag run is titled with the workflow name, not the commit message")
571
572 // A tag on an older commit of the branch: that commit alone.
573 c.mustGit(c.wt, "tag", "v0", older)
574 c.mustGit(c.wt, "push", "-q", c.urlAuth(adminToken, "/proj.git"), "v0")
575 job := runner.await()
576 var ctxs map[string]map[string]any
577 require.NoError(t, json.Unmarshal([]byte(job.GetContexts()), &ctxs))
578 runner.runJob(job, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
579 ev = ctxs["github"]["event"].(map[string]any)
580 assert.Equal(t, older, ev["after"])
581 assert.Equal(t, older, ev["head_commit"].(map[string]any)["id"])
582 assert.Equal(t, publicOrigin+"/owner/proj/commit/"+older[:12], ev["compare"])
583}
584
585// A tag on a new commit on top of the branch (recorded: push-tag) carries that
586// commit alone: the commit of the branch under it is not part of the event.
587func TestCITagEventOfANewCommitOnTopOfABranch(t *testing.T) {
588 c := newCIHarness(t, nil)
589 runner := c.registerRunner("alpha", "ubuntu-latest")
590 c.push("main", map[string]string{".github/workflows/ci.yml": "on:\n push:\n branches: [main]\n tags: ['v*']\njobs:\n j:\n runs-on: ubuntu-latest\n steps:\n - run: echo hi\n"}, "branch commit")
591 first := runner.await()
592 runner.runJob(first, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
593 c.waitFor("the branch run ends", func() bool { return len(c.runs()) == 1 && c.runs()[0].GetStatus() == pb.CiStatus_CI_STATUS_COMPLETED })
594 assert.Equal(t, "branch commit", c.runs()[0].GetTitle(), "a branch run keeps the commit message as title")
595
596 c.writeFile(c.wt, "t.txt", []byte("t\n")) // "add t.txt", only the tag is pushed
597 tagged := strings.TrimSpace(c.mustGit(c.wt, "rev-parse", "HEAD"))
598 ev, title := c.tagEvent(runner, "v1", 2)
599 assert.Equal(t, []string{"add t.txt"}, eventMessages(ev))
600 assert.Equal(t, publicOrigin+"/owner/proj/commit/"+tagged[:12], ev["compare"])
601 assert.Equal(t, ".github/workflows/ci.yml", title, "no name:, so the workflow name is the path of the file (assumed, not recorded)")
602 assert.Equal(t, true, ev["created"])
603 assert.Equal(t, domain.ZeroOID, ev["before"])
604}
605
606// Several tags in one push start one run each, with the same event: the
607// commit alone (recorded: push-tags-limit), titled with the workflow name.
608func TestCIThreeTagsInOnePush(t *testing.T) {
609 c := newCIHarness(t, nil)
610 runner := c.registerRunner("alpha", "ubuntu-latest")
611 c.push("main", map[string]string{".github/workflows/t.yml": "name: Tags\non:\n push:\n tags: ['v*']\njobs:\n j:\n runs-on: ubuntu-latest\n steps:\n - run: echo hi\n"}, "")
612 c.writeFile(c.wt, "t.txt", []byte("t\n"))
613 sha := strings.TrimSpace(c.mustGit(c.wt, "rev-parse", "HEAD"))
614 for _, tag := range []string{"v1", "v2", "v3"} {
615 c.mustGit(c.wt, "tag", tag)
616 }
617 c.mustGit(c.wt, "push", "-q", c.urlAuth(adminToken, "/proj.git"), "v1", "v2", "v3")
618 for range 3 {
619 job := runner.await()
620 var ctxs map[string]map[string]any
621 require.NoError(t, json.Unmarshal([]byte(job.GetContexts()), &ctxs))
622 ev := ctxs["github"]["event"].(map[string]any)
623 assert.Equal(t, []string{"add t.txt"}, eventMessages(ev))
624 assert.Equal(t, publicOrigin+"/owner/proj/commit/"+sha[:12], ev["compare"])
625 runner.runJob(job, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
626 }
627 c.waitFor("three runs", func() bool { return len(c.runs()) == 3 })
628 for _, r := range c.runs() {
629 assert.Equal(t, "Tags", r.GetTitle())
630 }
631}
632
633// A tag run without run-name is titled with the name of the workflow registered
634// for its path: the "name:" of the file on the default branch, which can differ
635// from the file at the tag (recorded: push-tag, push-tags-limit, push-delete-tag
636// and skip-directive-tag, whose ci.yml is "push-branch-create" on the default
637// branch of the recording repository and has the scenario's own name at the
638// tag; and the files with a unique name, where both agree). Its name stays the
639// workflow name of the tagged file.
640func TestCITagTitleIsTheNameRegisteredOnTheDefaultBranch(t *testing.T) {
641 c := newCIHarness(t, nil)
642 runner := c.registerRunner("alpha", "ubuntu-latest")
643 wf := func(name string) map[string]string {
644 return map[string]string{".github/workflows/ci.yml": "name: " + name + "\non:\n push:\n tags: ['v*']\njobs:\n j:\n runs-on: ubuntu-latest\n steps:\n - run: echo hi\n"}
645 }
646 c.push("main", wf("registered"), "")
647 // The tagged commit is off every branch and renames the workflow.
648 c.mustGit(c.wt, "checkout", "-q", "--detach")
649 c.writeFile(c.wt, ".github/workflows/ci.yml", []byte(wf("renamed")[".github/workflows/ci.yml"]))
650 _, title := c.tagEvent(runner, "v1", 1)
651 assert.Equal(t, "registered", title, "the name of ci.yml on the default branch")
652 r := c.runs()[0]
653 assert.Equal(t, "renamed", r.GetWorkflowName(), "the workflow of the run is the file at the tag (github.workflow)")
654 assert.Equal(t, "renamed", r.GetName(), "and so is the name of the run, which has no run-name")
655
656 // A file the default branch does not have: the run's own workflow name.
657 c.writeFile(c.wt, ".github/workflows/other.yml", []byte("name: other\non:\n push:\n tags: ['w*']\njobs:\n j:\n runs-on: ubuntu-latest\n steps:\n - run: echo hi\n"))
658 _, title = c.tagEvent(runner, "w1", 2)
659 assert.Equal(t, "other", title)
660}
661
662// A tag run with a run-name keeps the evaluated run-name as its title (the
663// run-name of a tag push is not recorded; branch pushes are, ctx-github-names).
664func TestCITagRunKeepsItsRunName(t *testing.T) {
665 c := newCIHarness(t, nil)
666 runner := c.registerRunner("alpha", "ubuntu-latest")
667 c.push("main", map[string]string{".github/workflows/t.yml": "name: Tags\nrun-name: \"release ${{ github.ref_name }}\"\non:\n push:\n tags: ['v*']\njobs:\n j:\n runs-on: ubuntu-latest\n steps:\n - run: echo hi\n"}, "")
668 _, title := c.tagEvent(runner, "v1", 1)
669 assert.Equal(t, "release v1", title)
670}
671
672// An exclude entry with a key that is no key of the matrix makes the workflow
673// invalid, and the run is named after the file (recorded: matrix-edge-values,
674// a-exclude-unknown-key.yml: name = path, no job), while an over-256 matrix
675// keeps the name of the workflow (matrix-over-256).
676func TestCIInvalidStrategyRunNames(t *testing.T) {
677 c := newCIHarness(t, nil)
678 c.registerRunner("alpha", "ubuntu-latest")
679 var a, b []string
680 for i := 1; i <= 17; i++ {
681 a = append(a, itoa(i))
682 }
683 for i := 1; i <= 16; i++ {
684 b = append(b, itoa(i))
685 }
686 c.push("main", map[string]string{
687 ".github/workflows/over.yml": `name: matrix-over-256
688on: push
689jobs:
690 too-many:
691 runs-on: ubuntu-latest
692 strategy:
693 matrix:
694 a: [` + strings.Join(a, ", ") + `]
695 b: [` + strings.Join(b, ", ") + `]
696 steps:
697 - run: echo
698`,
699 ".github/workflows/exclude.yml": `name: a-exclude-unknown-key
700on: push
701jobs:
702 main:
703 runs-on: ubuntu-latest
704 strategy:
705 matrix:
706 os: [one, two]
707 exclude:
708 - nokey: x
709 - os: one
710 nokey: y
711 steps:
712 - run: echo "os=${{ matrix.os }}"
713`,
714 ".github/workflows/noname.yml": `on: push
715jobs:
716 main:
717 runs-on: ubuntu-latest
718 strategy:
719 matrix:
720 os: [one]
721 exclude:
722 - nokey: x
723 steps:
724 - run: echo
725`}, "")
726 c.waitFor("three failed runs", func() bool {
727 rs := c.runs()
728 if len(rs) != 3 {
729 return false
730 }
731 for _, r := range rs {
732 if r.GetStatus() != pb.CiStatus_CI_STATUS_COMPLETED {
733 return false
734 }
735 }
736 return true
737 })
738 names := map[string]string{}
739 for _, r := range c.runs() {
740 names[r.GetWorkflowPath()] = r.GetWorkflowName()
741 assert.Equal(t, pb.CiConclusion_CI_CONCLUSION_FAILURE, r.GetConclusion(), r.GetWorkflowPath())
742 assert.Empty(t, c.getRun(r.GetId()).GetJobs(), "no job: %s", r.GetWorkflowPath())
743 if r.GetWorkflowPath() == ".github/workflows/exclude.yml" {
744 one := ".github/workflows/exclude.yml: Matrix exclude key 'nokey' does not match any key within the matrix"
745 assert.Equal(t, one+"\n"+one, r.GetError(), "once per offending entry")
746 }
747 }
748 assert.Equal(t, map[string]string{
749 ".github/workflows/over.yml": "matrix-over-256",
750 ".github/workflows/exclude.yml": ".github/workflows/exclude.yml",
751 ".github/workflows/noname.yml": ".github/workflows/noname.yml",
752 }, names)
753 // The check of a failed run carries the same workflow name.
754 sha := strings.TrimSpace(c.mustGit(c.wt, "rev-parse", "HEAD"))
755 checkNames := map[string]string{}
756 for _, ck := range c.checks(sha) {
757 checkNames[ck.GetName()] = ck.GetWorkflowName()
758 }
759 assert.Equal(t, "matrix-over-256", checkNames[".github/workflows/over.yml"])
760 assert.Equal(t, ".github/workflows/exclude.yml", checkNames[".github/workflows/exclude.yml"])
761}
762
763// The default success() of a job looks at the whole chain of its needs, not
764// only at the direct ones (recorded: needs-failed-dependency): chain-after-always
765// needs with-always, which ran through always() after broken failed, and is
766// skipped. with-always and with-failure still run.
767func TestCIDefaultSuccessLooksAtTheWholeChain(t *testing.T) {
768 c := newCIHarness(t, nil)
769 runner := c.registerRunner("alpha", "ubuntu-latest")
770 c.push("main", map[string]string{".github/workflows/n.yml": `name: needs-failed-dependency
771on: push
772jobs:
773 broken:
774 runs-on: ubuntu-latest
775 steps:
776 - run: exit 1
777 plain:
778 needs: broken
779 runs-on: ubuntu-latest
780 steps:
781 - run: echo SHOULD-NOT-RUN
782 with-always:
783 needs: broken
784 if: always()
785 runs-on: ubuntu-latest
786 steps:
787 - run: echo "result=${{ needs.broken.result }}"
788 with-failure:
789 needs: broken
790 if: failure()
791 runs-on: ubuntu-latest
792 steps:
793 - run: echo "result=${{ needs.broken.result }}"
794 chain-after-always:
795 needs: with-always
796 runs-on: ubuntu-latest
797 steps:
798 - run: echo chain-after-always-runs-because-with-always-succeeded
799 chain-with-failure:
800 needs: with-always
801 if: failure()
802 runs-on: ubuntu-latest
803 steps:
804 - run: echo "failure() is true when an ancestor failed"
805 chain-always:
806 needs: with-always
807 if: always()
808 runs-on: ubuntu-latest
809 steps:
810 - run: echo always
811`}, "")
812 broken := runner.await()
813 assert.Equal(t, "broken", broken.GetJobKey())
814 runner.runJob(broken, pb.CiConclusion_CI_CONCLUSION_FAILURE, nil, "x")
815 ran := map[string]bool{}
816 for range 4 {
817 j := runner.await()
818 ran[j.GetJobKey()] = true
819 runner.runJob(j, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
820 }
821 assert.Equal(t, map[string]bool{"with-always": true, "with-failure": true, "chain-with-failure": true, "chain-always": true}, ran)
822 c.waitFor("the run ends", func() bool { return c.runs()[0].GetStatus() == pb.CiStatus_CI_STATUS_COMPLETED })
823 got := map[string]pb.CiConclusion{}
824 for _, j := range c.getRun(c.runs()[0].GetId()).GetJobs() {
825 got[j.GetName()] = j.GetConclusion()
826 }
827 skipped, ok := pb.CiConclusion_CI_CONCLUSION_SKIPPED, pb.CiConclusion_CI_CONCLUSION_SUCCESS
828 assert.Equal(t, map[string]pb.CiConclusion{
829 "broken": pb.CiConclusion_CI_CONCLUSION_FAILURE, "plain": skipped, "with-always": ok, "with-failure": ok,
830 "chain-after-always": skipped, "chain-with-failure": ok, "chain-always": ok,
831 }, got)
832}