1// SPDX-License-Identifier: AGPL-3.0-or-later
2
3package app
4
5import (
6 "encoding/json"
7 "net/http"
8 "os"
9 "slices"
10 "strings"
11 "testing"
12
13 "connectrpc.com/connect"
14 "github.com/stretchr/testify/assert"
15 "github.com/stretchr/testify/require"
16
17 pb "bnema.dev/ermite-api/gen/ermite"
18 "bnema.dev/ermite/internal/domain"
19)
20
21// conformance is testdata/conformance/ctx-job-runner.contexts.json: the
22// contexts github.com gave to the ctx-job-runner scenario of ermite-conformance
23// (see its _source).
24type conformance struct {
25 GithubKeyOrder []string `json:"_github_key_order"`
26 JobKeyOrder []string `json:"_job_key_order"`
27 RunnerOwned []string `json:"_runner_owned_github_keys"`
28 Github map[string]any `json:"github"`
29 Job map[string]any `json:"job"`
30 Matrix any `json:"matrix"`
31 Strategy map[string]any `json:"strategy"`
32}
33
34func loadConformance(t *testing.T) conformance {
35 t.Helper()
36 b, err := os.ReadFile("testdata/conformance/ctx-job-runner.contexts.json")
37 require.NoError(t, err)
38 var c conformance
39 require.NoError(t, json.Unmarshal(b, &c))
40 return c
41}
42
43// numericPlaceholders are the placeholders of the normaliser of
44// ermite-conformance for values that are numbers (or times) on github.com: ids,
45// times and the repository size. In the raw bundle of the same scenario id,
46// created_at, pushed_at and size are JSON numbers; the fixture only has the
47// placeholder, a string. They are matched by "number or string".
48var numericPlaceholders = []string{"<id:", "<time>", "<size>", "<actor_id:", "<repository_id:", "<repository_owner_id:", "<check_run_id:"}
49
50// shape is the type of each value of an object. A placeholder of a number
51// has the type "scalar".
52func shape(m map[string]any) map[string]string {
53 out := map[string]string{}
54 for k, v := range m {
55 if s, ok := v.(string); ok && slices.ContainsFunc(numericPlaceholders, func(p string) bool { return strings.HasPrefix(s, p) }) {
56 out[k] = "scalar"
57 continue
58 }
59 switch v.(type) {
60 case nil:
61 out[k] = "null"
62 case string:
63 out[k] = "string"
64 case float64:
65 out[k] = "number"
66 case bool:
67 out[k] = "bool"
68 case []any:
69 out[k] = "array"
70 case map[string]any:
71 out[k] = "object"
72 }
73 }
74 return out
75}
76
77// orderedKeys returns the keys of a JSON object in document order.
78func orderedKeys(t *testing.T, raw json.RawMessage) []string {
79 t.Helper()
80 dec := json.NewDecoder(strings.NewReader(string(raw)))
81 tok, err := dec.Token()
82 require.NoError(t, err)
83 require.Equal(t, json.Delim('{'), tok)
84 var keys []string
85 for dec.More() {
86 k, err := dec.Token()
87 require.NoError(t, err)
88 keys = append(keys, k.(string))
89 var skip json.RawMessage
90 require.NoError(t, dec.Decode(&skip))
91 }
92 return keys
93}
94
95// sameShape compares the shape of a recorded object with ours: a "scalar"
96// (numeric placeholder) accepts a number or a string. The description is a
97// property of the recorded repository, which ermite's test repository lacks.
98func sameShape(t *testing.T, want, got map[string]any, what string) {
99 t.Helper()
100 ws, gs := shape(want), shape(got)
101 for k, wt := range ws {
102 if k == "description" {
103 continue
104 }
105 if wt == "scalar" {
106 assert.Contains(t, []string{"number", "string"}, gs[k], "%s.%s", what, k)
107 continue
108 }
109 assert.Equal(t, wt, gs[k], "%s.%s", what, k)
110 }
111}
112
113// The github and job contexts of a push have the keys, the key order and the
114// value types of the ones github.com recorded for the same scenario shape
115// (ermite-conformance, ctx-job-runner), apart from the keys the runner adds.
116func TestCIContextsMatchTheGitHubRecording(t *testing.T) {
117 want := loadConformance(t)
118 c := newCIHarness(t, nil)
119 runner := c.registerRunner("alpha", "ubuntu-latest")
120 c.push("main", map[string]string{".github/workflows/ci.yml": workflowWith("branches: [main]")}, "")
121 job := runner.await()
122 var raw map[string]json.RawMessage
123 require.NoError(t, json.Unmarshal([]byte(job.GetContexts()), &raw))
124 var ctxs map[string]map[string]any
125 require.NoError(t, json.Unmarshal([]byte(job.GetContexts()), &ctxs))
126 runner.runJob(job, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
127
128 t.Run("github", func(t *testing.T) {
129 gh := ctxs["github"]
130 // The runner owns some keys and adds job and token; the server sends the rest.
131 serverKeys := slices.DeleteFunc(slices.Clone(want.GithubKeyOrder), func(k string) bool { return slices.Contains(want.RunnerOwned, k) })
132 assert.Equal(t, serverKeys, orderedKeys(t, raw["github"]), "key set and order")
133 serverGithub := map[string]any{}
134 for k, v := range want.Github {
135 if !slices.Contains(want.RunnerOwned, k) {
136 serverGithub[k] = v
137 }
138 }
139 sameShape(t, serverGithub, gh, "github")
140 assert.Equal(t, "Actions", gh["secret_source"])
141 assert.Equal(t, "10", gh["artifact_cache_size_limit"])
142 assert.Equal(t, "private", gh["repository_visibility"], "from the repository")
143 assert.Equal(t, "owner", gh["actor"])
144 assert.Equal(t, "owner", gh["repository_owner"])
145 assert.Regexp(t, `^[1-9][0-9]*$`, gh["actor_id"])
146 assert.Regexp(t, `^[1-9][0-9]*$`, gh["repository_id"])
147 assert.Regexp(t, `^[1-9][0-9]*$`, gh["repository_owner_id"])
148 assert.Equal(t, gh["actor_id"], gh["repository_owner_id"], "the owner pushed")
149 assert.Equal(t, "git://example.test/owner/proj.git", gh["repositoryUrl"], "git://<host>/<owner>/<repo>.git, as GitHub's")
150 })
151
152 t.Run("event", func(t *testing.T) {
153 ev := ctxs["github"]["event"].(map[string]any)
154 wev := want.Github["event"].(map[string]any)
155 assert.ElementsMatch(t, keysSlice(wev), keysSlice(ev), "event keys")
156 sameShape(t, wev, ev, "event")
157 repo, wrepo := ev["repository"].(map[string]any), wev["repository"].(map[string]any)
158 assert.Len(t, wrepo, 83, "the recording has 83 repository keys")
159 assert.ElementsMatch(t, keysSlice(wrepo), keysSlice(repo), "repository keys")
160 sameShape(t, wrepo, repo, "event.repository")
161 assert.ElementsMatch(t, keysSlice(wrepo["owner"].(map[string]any)), keysSlice(repo["owner"].(map[string]any)), "repository.owner keys")
162 sender, wsender := ev["sender"].(map[string]any), wev["sender"].(map[string]any)
163 assert.Len(t, wsender, 19)
164 assert.ElementsMatch(t, keysSlice(wsender), keysSlice(sender), "sender keys")
165 sameShape(t, wsender, sender, "event.sender")
166 assert.ElementsMatch(t, keysSlice(wev["pusher"].(map[string]any)), keysSlice(ev["pusher"].(map[string]any)))
167 for _, got := range []map[string]any{ev["head_commit"].(map[string]any), ev["commits"].([]any)[0].(map[string]any)} {
168 wc := wev["head_commit"].(map[string]any)
169 assert.ElementsMatch(t, keysSlice(wc), keysSlice(got), "commit keys: no added/removed/modified/username")
170 assert.ElementsMatch(t, keysSlice(wc["author"].(map[string]any)), keysSlice(got["author"].(map[string]any)))
171 assert.ElementsMatch(t, keysSlice(wc["committer"].(map[string]any)), keysSlice(got["committer"].(map[string]any)))
172 assert.True(t, strings.HasPrefix(got["url"].(string), publicOrigin+"/owner/proj/commit/"), got["url"])
173 }
174 assert.Equal(t, "proj", repo["name"])
175 assert.Equal(t, "owner/proj", repo["full_name"])
176 assert.Equal(t, true, repo["private"])
177 assert.Equal(t, "private", repo["visibility"])
178 assert.Equal(t, "main", repo["default_branch"])
179 assert.Equal(t, ctxs["github"]["repository_id"], jsonNumber(repo["id"]), "repository_id is repository.id")
180 assert.Equal(t, ctxs["github"]["actor_id"], jsonNumber(sender["id"]))
181 assert.Equal(t, ctxs["github"]["repository_owner_id"], jsonNumber(repo["owner"].(map[string]any)["id"]))
182 })
183
184 t.Run("job", func(t *testing.T) {
185 j := ctxs["job"]
186 // The runner adds status first; the server sends the rest, in GitHub's order.
187 serverKeys := slices.DeleteFunc(slices.Clone(want.JobKeyOrder), func(k string) bool { return k == "status" })
188 assert.Equal(t, serverKeys, orderedKeys(t, raw["job"]))
189 serverJob := map[string]any{}
190 for k, v := range want.Job {
191 if k != "status" {
192 serverJob[k] = v
193 }
194 }
195 sameShape(t, serverJob, j, "job")
196 assert.Equal(t, ".github/workflows/ci.yml", j["workflow_file_path"])
197 assert.Equal(t, "owner/proj", j["workflow_repository"])
198 assert.Equal(t, "owner/proj/.github/workflows/ci.yml@refs/heads/main", j["workflow_ref"])
199 assert.Equal(t, ctxs["github"]["sha"], j["workflow_sha"])
200 assert.Equal(t, ctxs["github"]["workflow_ref"], j["workflow_ref"])
201 })
202
203 t.Run("matrix is null without a matrix", func(t *testing.T) {
204 assert.Equal(t, "null", string(raw["matrix"]))
205 assert.Nil(t, want.Matrix)
206 assert.ElementsMatch(t, keysSlice(want.Strategy), keysSlice(ctxs["strategy"]))
207 })
208}
209
210func keysSlice(m map[string]any) []string {
211 out := make([]string, 0, len(m))
212 for k := range m {
213 out = append(out, k)
214 }
215 return out
216}
217
218func jsonNumber(v any) string {
219 b, _ := json.Marshal(v)
220 return string(b)
221}
222
223// The run title is the evaluated run-name, else the head commit message. The
224// name of the run is the evaluated run-name too (recorded: ctx-github-names,
225// name = title = the run-name), else the workflow name; github.workflow stays
226// the workflow name.
227func TestCIRunTitle(t *testing.T) {
228 c := newCIHarness(t, nil)
229 c.registerRunner("alpha", "ubuntu-latest")
230 c.push("main", map[string]string{".github/workflows/named.yml": `name: Named
231run-name: "run ${{ github.event_name }} on ${{ github.ref_type }} by ${{ github.actor == '' && 'nobody' || 'someone' }}"
232on: push
233jobs:
234 j:
235 runs-on: ubuntu-latest
236 steps:
237 - run: echo hi
238`, ".github/workflows/plain.yml": workflowWith("branches: [main]")}, "the head commit\n\nbody")
239 c.waitFor("two runs", func() bool { return len(c.runs()) == 2 })
240 titles, names, workflows := map[string]string{}, map[string]string{}, map[string]string{}
241 for _, r := range c.runs() {
242 titles[r.GetWorkflowPath()] = r.GetTitle()
243 names[r.GetWorkflowPath()] = r.GetName()
244 workflows[r.GetWorkflowPath()] = r.GetWorkflowName()
245 }
246 assert.Equal(t, map[string]string{
247 ".github/workflows/named.yml": "run push on branch by someone",
248 ".github/workflows/plain.yml": "the head commit",
249 }, titles)
250 assert.Equal(t, map[string]string{
251 ".github/workflows/named.yml": "run push on branch by someone",
252 ".github/workflows/plain.yml": ".github/workflows/plain.yml",
253 }, names, "the run-name, else the workflow name")
254 assert.Equal(t, map[string]string{
255 ".github/workflows/named.yml": "Named",
256 ".github/workflows/plain.yml": ".github/workflows/plain.yml",
257 }, workflows, "workflow_name is unchanged by a run-name")
258}
259
260// An over-256 matrix is an invalid workflow, like on github.com
261// (matrix-over-256): a failed run with no job.
262func TestCIMatrixOver256IsAnInvalidWorkflow(t *testing.T) {
263 c := newCIHarness(t, nil)
264 c.registerRunner("alpha", "ubuntu-latest")
265 var a, b []string
266 for i := 1; i <= 17; i++ {
267 a = append(a, itoa(i))
268 }
269 for i := 1; i <= 16; i++ {
270 b = append(b, itoa(i))
271 }
272 c.push("main", map[string]string{".github/workflows/m.yml": `on: push
273jobs:
274 too-many:
275 runs-on: ubuntu-latest
276 strategy:
277 matrix:
278 a: [` + strings.Join(a, ", ") + `]
279 b: [` + strings.Join(b, ", ") + `]
280 steps:
281 - run: echo
282`}, "")
283 c.waitFor("the run fails", func() bool { return len(c.runs()) == 1 && c.runs()[0].GetStatus() == pb.CiStatus_CI_STATUS_COMPLETED })
284 run := c.getRun(c.runs()[0].GetId())
285 assert.Equal(t, pb.CiConclusion_CI_CONCLUSION_FAILURE, run.GetRun().GetConclusion())
286 assert.Empty(t, run.GetJobs(), "no job")
287 assert.Equal(t, ".github/workflows/m.yml", run.GetRun().GetWorkflowName(), "no name: in this file: the path")
288 assert.Contains(t, run.GetRun().GetError(), "Strategy for job 'too-many' produced 272 configurations which exceeds the maximum of 256 configurations")
289}
290
291func itoa(i int) string { b, _ := json.Marshal(i); return string(b) }
292
293// A step with continue-on-error is no longer refused; the runner runs it.
294func TestCIStepContinueOnErrorIsPlanned(t *testing.T) {
295 c := newCIHarness(t, nil)
296 runner := c.registerRunner("alpha", "ubuntu-latest")
297 c.push("main", map[string]string{".github/workflows/s.yml": `on: push
298jobs:
299 j:
300 runs-on: ubuntu-latest
301 steps:
302 - continue-on-error: ${{ github.event_name == 'push' }}
303 run: exit 1
304`}, "")
305 job := runner.await()
306 assert.Equal(t, "j", job.GetJobKey())
307 runner.runJob(job, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
308}
309
310// A step timeout stays refused as long as the runner has none.
311func TestCIStepTimeoutIsStillRefused(t *testing.T) {
312 c := newCIHarness(t, nil)
313 c.registerRunner("alpha", "ubuntu-latest")
314 c.push("main", map[string]string{".github/workflows/s.yml": `on: push
315jobs:
316 j:
317 runs-on: ubuntu-latest
318 steps:
319 - timeout-minutes: 1
320 run: echo
321`}, "")
322 c.waitFor("the run fails", func() bool { return len(c.runs()) == 1 && c.runs()[0].GetStatus() == pb.CiStatus_CI_STATUS_COMPLETED })
323 assert.Equal(t, "not supported yet: timeout-minutes on a step", c.getRun(c.runs()[0].GetId()).GetJobs()[0].GetError())
324}
325
326// A failed job with continue-on-error does not fail the run, keeps its own
327// failure conclusion, and shows as success to the jobs that need it (recorded:
328// if-job-level-fail-effects, where the dependent job printed result=success).
329func TestCIJobContinueOnError(t *testing.T) {
330 c := newCIHarness(t, nil)
331 runner := c.registerRunner("alpha", "ubuntu-latest")
332 c.push("main", map[string]string{".github/workflows/c.yml": `on: push
333jobs:
334 soft:
335 runs-on: ubuntu-latest
336 continue-on-error: ${{ github.event_name == 'push' }}
337 steps:
338 - run: exit 5
339 after:
340 needs: soft
341 runs-on: ubuntu-latest
342 steps:
343 - run: echo "result=${{ needs.soft.result }}"
344`}, "")
345 soft := runner.await()
346 assert.Equal(t, "soft", soft.GetJobKey())
347 runner.runJob(soft, pb.CiConclusion_CI_CONCLUSION_FAILURE, nil, "x")
348 after := runner.await()
349 assert.Equal(t, "after", after.GetJobKey(), "a needed job that failed softly does not skip its dependents")
350 assert.Contains(t, after.GetContexts(), `"result":"success"`)
351 runner.runJob(after, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
352 c.waitFor("the run ends", func() bool { return c.runs()[0].GetStatus() == pb.CiStatus_CI_STATUS_COMPLETED })
353 assert.Equal(t, pb.CiConclusion_CI_CONCLUSION_SUCCESS, c.runs()[0].GetConclusion(), "the run does not fail")
354 byKey := map[string]pb.CiConclusion{}
355 for _, j := range c.getRun(c.runs()[0].GetId()).GetJobs() {
356 byKey[j.GetName()] = j.GetConclusion()
357 }
358 assert.Equal(t, pb.CiConclusion_CI_CONCLUSION_FAILURE, byKey["soft"], "the job itself failed")
359}
360
361// A matrix leg with continue-on-error that fails does not cancel its siblings
362// (fail-fast), and does not fail the run; a leg without it still does.
363func TestCIJobContinueOnErrorLegDoesNotTriggerFailFast(t *testing.T) {
364 c := newCIHarness(t, nil)
365 runner := c.registerRunner("alpha", "ubuntu-latest")
366 c.push("main", map[string]string{".github/workflows/m.yml": `on: push
367jobs:
368 build:
369 runs-on: ubuntu-latest
370 continue-on-error: ${{ matrix.experimental }}
371 strategy:
372 matrix:
373 experimental: [true, false]
374 steps:
375 - run: echo ${{ matrix.experimental }}
376`}, "")
377 a, b := runner.await(), runner.await()
378 soft, hard := a, b
379 if strings.Contains(b.GetMatrix(), "true") {
380 soft, hard = b, a
381 }
382 apiHard := runner.jobAPI(hard)
383 _, err := apiHard.runner.Heartbeat(c.ctx, connect.NewRequest(&pb.HeartbeatRequest{JobId: hard.GetJobId()}))
384 require.NoError(t, err)
385 runner.runJob(soft, pb.CiConclusion_CI_CONCLUSION_FAILURE, nil, "x")
386 // The sibling must not have been asked to stop.
387 watch, err := apiHard.runner.WatchJob(c.ctx, connect.NewRequest(&pb.WatchJobRequest{JobId: hard.GetJobId()}))
388 require.NoError(t, err)
389 done := make(chan *pb.CancelReason, 1)
390 go func() {
391 for watch.Receive() {
392 if m := watch.Msg().GetCancel(); m != nil {
393 r := m.GetReason()
394 done <- &r
395 return
396 }
397 }
398 done <- nil
399 }()
400 runner.runJob(hard, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
401 c.waitFor("the run ends", func() bool { return c.runs()[0].GetStatus() == pb.CiStatus_CI_STATUS_COMPLETED })
402 assert.Equal(t, pb.CiConclusion_CI_CONCLUSION_SUCCESS, c.runs()[0].GetConclusion())
403 if r := <-done; r != nil {
404 t.Fatalf("the sibling of a continue-on-error leg was cancelled: %v", *r)
405 }
406}
407
408// GitHub-shaped URLs, /<owner>/<repo>/..., redirect to ermite's pages, only for
409// the owner of the instance and only where there is no page of its own.
410func TestWebRedirectsOwnerURLs(t *testing.T) {
411 c := newCIHarness(t, nil)
412 b := browser{c.harness}
413 c.push("main", map[string]string{"README": "x\n"}, "")
414 sha := strings.TrimSpace(c.mustGit(c.wt, "rev-parse", "HEAD"))
415 for path, to := range map[string]string{
416 "/owner/proj": "/proj",
417 "/owner/proj/commit/" + sha: "/proj/commit/" + sha,
418 "/owner/proj/compare/a...b?x=1": "/proj/compare/a...b?x=1",
419 "/owner/proj.git/info/refs": "/proj.git/info/refs",
420 } {
421 res := b.get(path, nil)
422 assert.Equal(t, http.StatusPermanentRedirect, res.status, path)
423 assert.Equal(t, to, res.hdr.Get("Location"), path)
424 }
425 assert.Equal(t, http.StatusNotFound, b.get("/other/proj/commit/"+sha, nil).status, "not the owner")
426 assert.Equal(t, http.StatusNotFound, b.get("/owner", nil).status, "a bare owner is no page")
427 assert.Equal(t, http.StatusNotFound, b.get("/owner/Bad Name", nil).status)
428 post := b.do("POST", "/owner/proj", map[string]string{"Origin": publicOrigin}, "")
429 assert.Equal(t, http.StatusForbidden, post.status, "a write is stopped by the gate, never redirected")
430 assert.Empty(t, post.hdr.Get("Location"))
431}
432
433// github.job is empty while a job is evaluated, so a name that reads it loses
434// it (recorded: ctx-github-names), but the job gets its key in the contexts.
435func TestCIJobNameReadsAnEmptyGithubJob(t *testing.T) {
436 c := newCIHarness(t, nil)
437 runner := c.registerRunner("alpha", "ubuntu-latest")
438 c.push("main", map[string]string{".github/workflows/n.yml": `on: push
439jobs:
440 first-job:
441 name: Display name of ${{ github.job }}
442 runs-on: ubuntu-latest
443 steps:
444 - run: echo hi
445`}, "")
446 job := runner.await()
447 assert.Contains(t, job.GetContexts(), `"job":"first-job"`)
448 runner.runJob(job, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
449 c.waitFor("the run ends", func() bool { return c.runs()[0].GetStatus() == pb.CiStatus_CI_STATUS_COMPLETED })
450 assert.Equal(t, "Display name of", c.getRun(c.runs()[0].GetId()).GetJobs()[0].GetName())
451}
452
453// A tag push carries the commits that no branch has, newest as head_commit, and
454// the compare URL of several commits is "<first>^...<last>" (recorded:
455// skip-directive-tag, where no branch exists).
456func TestCITagEventCarriesTheNewCommits(t *testing.T) {
457 c := newCIHarness(t, nil)
458 runner := c.registerRunner("alpha", "ubuntu-latest")
459 c.push("main", map[string]string{".github/workflows/rel.yml": workflowWith("tags: ['v*']")}, "")
460 c.mustGit(c.wt, "checkout", "-q", "-b", "rel")
461 c.writeFile(c.wt, "a.txt", []byte("a\n")) // commits "add a.txt"
462 first := strings.TrimSpace(c.mustGit(c.wt, "rev-parse", "HEAD"))
463 c.writeFile(c.wt, "b.txt", []byte("b\n")) // commits "add b.txt"
464 last := strings.TrimSpace(c.mustGit(c.wt, "rev-parse", "HEAD"))
465 c.mustGit(c.wt, "tag", "v1")
466 c.mustGit(c.wt, "push", "-q", c.urlAuth(adminToken, "/proj.git"), "v1")
467
468 job := runner.await()
469 var ctxs map[string]map[string]any
470 require.NoError(t, json.Unmarshal([]byte(job.GetContexts()), &ctxs))
471 runner.runJob(job, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
472 ev := ctxs["github"]["event"].(map[string]any)
473 var msgs []string
474 for _, cm := range ev["commits"].([]any) {
475 msgs = append(msgs, cm.(map[string]any)["message"].(string))
476 }
477 assert.Equal(t, []string{"add a.txt", "add b.txt"}, msgs)
478 assert.Equal(t, "add b.txt", ev["head_commit"].(map[string]any)["message"])
479 assert.Equal(t, publicOrigin+"/owner/proj/compare/"+first[:12]+"^..."+last[:12], ev["compare"])
480 assert.Equal(t, "tag", ctxs["github"]["ref_type"])
481 assert.Equal(t, true, ev["created"])
482}
483
484// tagEvent pushes the tag and returns the push event and the title of the run
485// it starts, after `want` runs exist.
486func (c *ciHarness) tagEvent(runner *fakeRunner, tag string, want int) (map[string]any, string) {
487 c.t.Helper()
488 c.mustGit(c.wt, "tag", tag)
489 c.mustGit(c.wt, "push", "-q", c.urlAuth(adminToken, "/proj.git"), tag)
490 job := runner.await()
491 var ctxs map[string]map[string]any
492 require.NoError(c.t, json.Unmarshal([]byte(job.GetContexts()), &ctxs))
493 runner.runJob(job, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
494 c.waitFor("the run exists", func() bool { return len(c.runs()) == want })
495 var title string
496 for _, r := range c.runs() {
497 if r.GetRef() == "refs/tags/"+tag {
498 title = r.GetTitle()
499 }
500 }
501 return ctxs["github"]["event"].(map[string]any), title
502}
503
504func eventMessages(ev map[string]any) []string {
505 var msgs []string
506 for _, cm := range ev["commits"].([]any) {
507 msgs = append(msgs, cm.(map[string]any)["message"].(string))
508 }
509 return msgs
510}
511
512// A tag on a commit that a branch has carries that commit alone, and the
513// compare URL is its page, not a range from the previous push (recorded:
514// filter-tags, filter-paths-tag, filter-branches-ignore). The tag run is
515// titled with the workflow name (filter-*).
516func TestCITagEventOfACommitABranchHas(t *testing.T) {
517 c := newCIHarness(t, nil)
518 runner := c.registerRunner("alpha", "ubuntu-latest")
519 // Two commits on main, pushed with the workflow; both are on a branch.
520 c.push("main", map[string]string{".github/workflows/rel.yml": "name: Release\non:\n push:\n tags: ['v*']\njobs:\n j:\n runs-on: ubuntu-latest\n steps:\n - run: echo hi\n"}, "")
521 older := strings.TrimSpace(c.mustGit(c.wt, "rev-parse", "HEAD"))
522 c.writeFile(c.wt, "x.txt", []byte("x\n"))
523 c.mustGit(c.wt, "push", "-q", c.urlAuth(adminToken, "/proj.git"), "main")
524 newer := strings.TrimSpace(c.mustGit(c.wt, "rev-parse", "HEAD"))
525
526 ev, title := c.tagEvent(runner, "v1", 1)
527 assert.Equal(t, []string{"add x.txt"}, eventMessages(ev), "the tagged commit alone, not the push range")
528 assert.Equal(t, newer, ev["head_commit"].(map[string]any)["id"])
529 assert.Equal(t, publicOrigin+"/owner/proj/commit/"+newer[:12], ev["compare"])
530 assert.Equal(t, "Release", title, "a tag run is titled with the workflow name, not the commit message")
531
532 // A tag on an older commit of the branch: that commit alone.
533 c.mustGit(c.wt, "tag", "v0", older)
534 c.mustGit(c.wt, "push", "-q", c.urlAuth(adminToken, "/proj.git"), "v0")
535 job := runner.await()
536 var ctxs map[string]map[string]any
537 require.NoError(t, json.Unmarshal([]byte(job.GetContexts()), &ctxs))
538 runner.runJob(job, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
539 ev = ctxs["github"]["event"].(map[string]any)
540 assert.Equal(t, older, ev["after"])
541 assert.Equal(t, older, ev["head_commit"].(map[string]any)["id"])
542 assert.Equal(t, publicOrigin+"/owner/proj/commit/"+older[:12], ev["compare"])
543}
544
545// A tag on a new commit on top of the branch (recorded: push-tag) carries that
546// commit alone: the commit of the branch under it is not part of the event.
547func TestCITagEventOfANewCommitOnTopOfABranch(t *testing.T) {
548 c := newCIHarness(t, nil)
549 runner := c.registerRunner("alpha", "ubuntu-latest")
550 c.push("main", map[string]string{".github/workflows/ci.yml": "on:\n push:\n branches: [main]\n tags: ['v*']\njobs:\n j:\n runs-on: ubuntu-latest\n steps:\n - run: echo hi\n"}, "branch commit")
551 first := runner.await()
552 runner.runJob(first, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
553 c.waitFor("the branch run ends", func() bool { return len(c.runs()) == 1 && c.runs()[0].GetStatus() == pb.CiStatus_CI_STATUS_COMPLETED })
554 assert.Equal(t, "branch commit", c.runs()[0].GetTitle(), "a branch run keeps the commit message as title")
555
556 c.writeFile(c.wt, "t.txt", []byte("t\n")) // "add t.txt", only the tag is pushed
557 tagged := strings.TrimSpace(c.mustGit(c.wt, "rev-parse", "HEAD"))
558 ev, title := c.tagEvent(runner, "v1", 2)
559 assert.Equal(t, []string{"add t.txt"}, eventMessages(ev))
560 assert.Equal(t, publicOrigin+"/owner/proj/commit/"+tagged[:12], ev["compare"])
561 assert.Equal(t, ".github/workflows/ci.yml", title, "no name:, so the workflow name is the path of the file (assumed, not recorded)")
562 assert.Equal(t, true, ev["created"])
563 assert.Equal(t, domain.ZeroOID, ev["before"])
564}
565
566// Several tags in one push start one run each, with the same event: the
567// commit alone (recorded: push-tags-limit), titled with the workflow name.
568func TestCIThreeTagsInOnePush(t *testing.T) {
569 c := newCIHarness(t, nil)
570 runner := c.registerRunner("alpha", "ubuntu-latest")
571 c.push("main", map[string]string{".github/workflows/t.yml": "name: Tags\non:\n push:\n tags: ['v*']\njobs:\n j:\n runs-on: ubuntu-latest\n steps:\n - run: echo hi\n"}, "")
572 c.writeFile(c.wt, "t.txt", []byte("t\n"))
573 sha := strings.TrimSpace(c.mustGit(c.wt, "rev-parse", "HEAD"))
574 for _, tag := range []string{"v1", "v2", "v3"} {
575 c.mustGit(c.wt, "tag", tag)
576 }
577 c.mustGit(c.wt, "push", "-q", c.urlAuth(adminToken, "/proj.git"), "v1", "v2", "v3")
578 for range 3 {
579 job := runner.await()
580 var ctxs map[string]map[string]any
581 require.NoError(t, json.Unmarshal([]byte(job.GetContexts()), &ctxs))
582 ev := ctxs["github"]["event"].(map[string]any)
583 assert.Equal(t, []string{"add t.txt"}, eventMessages(ev))
584 assert.Equal(t, publicOrigin+"/owner/proj/commit/"+sha[:12], ev["compare"])
585 runner.runJob(job, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
586 }
587 c.waitFor("three runs", func() bool { return len(c.runs()) == 3 })
588 for _, r := range c.runs() {
589 assert.Equal(t, "Tags", r.GetTitle())
590 }
591}
592
593// A tag run without run-name is titled with the name of the workflow registered
594// for its path: the "name:" of the file on the default branch, which can differ
595// from the file at the tag (recorded: push-tag, push-tags-limit, push-delete-tag
596// and skip-directive-tag, whose ci.yml is "push-branch-create" on the default
597// branch of the recording repository and has the scenario's own name at the
598// tag; and the files with a unique name, where both agree). Its name stays the
599// workflow name of the tagged file.
600func TestCITagTitleIsTheNameRegisteredOnTheDefaultBranch(t *testing.T) {
601 c := newCIHarness(t, nil)
602 runner := c.registerRunner("alpha", "ubuntu-latest")
603 wf := func(name string) map[string]string {
604 return map[string]string{".github/workflows/ci.yml": "name: " + name + "\non:\n push:\n tags: ['v*']\njobs:\n j:\n runs-on: ubuntu-latest\n steps:\n - run: echo hi\n"}
605 }
606 c.push("main", wf("registered"), "")
607 // The tagged commit is off every branch and renames the workflow.
608 c.mustGit(c.wt, "checkout", "-q", "--detach")
609 c.writeFile(c.wt, ".github/workflows/ci.yml", []byte(wf("renamed")[".github/workflows/ci.yml"]))
610 _, title := c.tagEvent(runner, "v1", 1)
611 assert.Equal(t, "registered", title, "the name of ci.yml on the default branch")
612 r := c.runs()[0]
613 assert.Equal(t, "renamed", r.GetWorkflowName(), "the workflow of the run is the file at the tag (github.workflow)")
614 assert.Equal(t, "renamed", r.GetName(), "and so is the name of the run, which has no run-name")
615
616 // A file the default branch does not have: the run's own workflow name.
617 c.writeFile(c.wt, ".github/workflows/other.yml", []byte("name: other\non:\n push:\n tags: ['w*']\njobs:\n j:\n runs-on: ubuntu-latest\n steps:\n - run: echo hi\n"))
618 _, title = c.tagEvent(runner, "w1", 2)
619 assert.Equal(t, "other", title)
620}
621
622// A tag run with a run-name keeps the evaluated run-name as its title (the
623// run-name of a tag push is not recorded; branch pushes are, ctx-github-names).
624func TestCITagRunKeepsItsRunName(t *testing.T) {
625 c := newCIHarness(t, nil)
626 runner := c.registerRunner("alpha", "ubuntu-latest")
627 c.push("main", map[string]string{".github/workflows/t.yml": "name: Tags\nrun-name: \"release ${{ github.ref_name }}\"\non:\n push:\n tags: ['v*']\njobs:\n j:\n runs-on: ubuntu-latest\n steps:\n - run: echo hi\n"}, "")
628 _, title := c.tagEvent(runner, "v1", 1)
629 assert.Equal(t, "release v1", title)
630}
631
632// An exclude entry with a key that is no key of the matrix makes the workflow
633// invalid, and the run is named after the file (recorded: matrix-edge-values,
634// a-exclude-unknown-key.yml: name = path, no job), while an over-256 matrix
635// keeps the name of the workflow (matrix-over-256).
636func TestCIInvalidStrategyRunNames(t *testing.T) {
637 c := newCIHarness(t, nil)
638 c.registerRunner("alpha", "ubuntu-latest")
639 var a, b []string
640 for i := 1; i <= 17; i++ {
641 a = append(a, itoa(i))
642 }
643 for i := 1; i <= 16; i++ {
644 b = append(b, itoa(i))
645 }
646 c.push("main", map[string]string{
647 ".github/workflows/over.yml": `name: matrix-over-256
648on: push
649jobs:
650 too-many:
651 runs-on: ubuntu-latest
652 strategy:
653 matrix:
654 a: [` + strings.Join(a, ", ") + `]
655 b: [` + strings.Join(b, ", ") + `]
656 steps:
657 - run: echo
658`,
659 ".github/workflows/exclude.yml": `name: a-exclude-unknown-key
660on: push
661jobs:
662 main:
663 runs-on: ubuntu-latest
664 strategy:
665 matrix:
666 os: [one, two]
667 exclude:
668 - nokey: x
669 - os: one
670 nokey: y
671 steps:
672 - run: echo "os=${{ matrix.os }}"
673`,
674 ".github/workflows/noname.yml": `on: push
675jobs:
676 main:
677 runs-on: ubuntu-latest
678 strategy:
679 matrix:
680 os: [one]
681 exclude:
682 - nokey: x
683 steps:
684 - run: echo
685`}, "")
686 c.waitFor("three failed runs", func() bool {
687 rs := c.runs()
688 if len(rs) != 3 {
689 return false
690 }
691 for _, r := range rs {
692 if r.GetStatus() != pb.CiStatus_CI_STATUS_COMPLETED {
693 return false
694 }
695 }
696 return true
697 })
698 names := map[string]string{}
699 for _, r := range c.runs() {
700 names[r.GetWorkflowPath()] = r.GetWorkflowName()
701 assert.Equal(t, pb.CiConclusion_CI_CONCLUSION_FAILURE, r.GetConclusion(), r.GetWorkflowPath())
702 assert.Empty(t, c.getRun(r.GetId()).GetJobs(), "no job: %s", r.GetWorkflowPath())
703 if r.GetWorkflowPath() == ".github/workflows/exclude.yml" {
704 one := ".github/workflows/exclude.yml: Matrix exclude key 'nokey' does not match any key within the matrix"
705 assert.Equal(t, one+"\n"+one, r.GetError(), "once per offending entry")
706 }
707 }
708 assert.Equal(t, map[string]string{
709 ".github/workflows/over.yml": "matrix-over-256",
710 ".github/workflows/exclude.yml": ".github/workflows/exclude.yml",
711 ".github/workflows/noname.yml": ".github/workflows/noname.yml",
712 }, names)
713 // The check of a failed run carries the same workflow name.
714 sha := strings.TrimSpace(c.mustGit(c.wt, "rev-parse", "HEAD"))
715 checkNames := map[string]string{}
716 for _, ck := range c.checks(sha) {
717 checkNames[ck.GetName()] = ck.GetWorkflowName()
718 }
719 assert.Equal(t, "matrix-over-256", checkNames[".github/workflows/over.yml"])
720 assert.Equal(t, ".github/workflows/exclude.yml", checkNames[".github/workflows/exclude.yml"])
721}
722
723// The default success() of a job looks at the whole chain of its needs, not
724// only at the direct ones (recorded: needs-failed-dependency): chain-after-always
725// needs with-always, which ran through always() after broken failed, and is
726// skipped. with-always and with-failure still run.
727func TestCIDefaultSuccessLooksAtTheWholeChain(t *testing.T) {
728 c := newCIHarness(t, nil)
729 runner := c.registerRunner("alpha", "ubuntu-latest")
730 c.push("main", map[string]string{".github/workflows/n.yml": `name: needs-failed-dependency
731on: push
732jobs:
733 broken:
734 runs-on: ubuntu-latest
735 steps:
736 - run: exit 1
737 plain:
738 needs: broken
739 runs-on: ubuntu-latest
740 steps:
741 - run: echo SHOULD-NOT-RUN
742 with-always:
743 needs: broken
744 if: always()
745 runs-on: ubuntu-latest
746 steps:
747 - run: echo "result=${{ needs.broken.result }}"
748 with-failure:
749 needs: broken
750 if: failure()
751 runs-on: ubuntu-latest
752 steps:
753 - run: echo "result=${{ needs.broken.result }}"
754 chain-after-always:
755 needs: with-always
756 runs-on: ubuntu-latest
757 steps:
758 - run: echo chain-after-always-runs-because-with-always-succeeded
759 chain-with-failure:
760 needs: with-always
761 if: failure()
762 runs-on: ubuntu-latest
763 steps:
764 - run: echo "failure() is true when an ancestor failed"
765 chain-always:
766 needs: with-always
767 if: always()
768 runs-on: ubuntu-latest
769 steps:
770 - run: echo always
771`}, "")
772 broken := runner.await()
773 assert.Equal(t, "broken", broken.GetJobKey())
774 runner.runJob(broken, pb.CiConclusion_CI_CONCLUSION_FAILURE, nil, "x")
775 ran := map[string]bool{}
776 for range 4 {
777 j := runner.await()
778 ran[j.GetJobKey()] = true
779 runner.runJob(j, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
780 }
781 assert.Equal(t, map[string]bool{"with-always": true, "with-failure": true, "chain-with-failure": true, "chain-always": true}, ran)
782 c.waitFor("the run ends", func() bool { return c.runs()[0].GetStatus() == pb.CiStatus_CI_STATUS_COMPLETED })
783 got := map[string]pb.CiConclusion{}
784 for _, j := range c.getRun(c.runs()[0].GetId()).GetJobs() {
785 got[j.GetName()] = j.GetConclusion()
786 }
787 skipped, ok := pb.CiConclusion_CI_CONCLUSION_SKIPPED, pb.CiConclusion_CI_CONCLUSION_SUCCESS
788 assert.Equal(t, map[string]pb.CiConclusion{
789 "broken": pb.CiConclusion_CI_CONCLUSION_FAILURE, "plain": skipped, "with-always": ok, "with-failure": ok,
790 "chain-after-always": skipped, "chain-with-failure": ok, "chain-always": ok,
791 }, got)
792}