1// SPDX-License-Identifier: AGPL-3.0-or-later
2
3package app
4
5import (
6 "encoding/json"
7 "net/http"
8 "os"
9 "slices"
10 "strings"
11 "testing"
12
13 "connectrpc.com/connect"
14 "github.com/stretchr/testify/assert"
15 "github.com/stretchr/testify/require"
16
17 pb "bnema.dev/ermite-api/gen/ermite"
18 "bnema.dev/ermite/internal/domain"
19)
20
21// conformance is testdata/conformance/ctx-job-runner.contexts.json: the
22// contexts github.com gave to the ctx-job-runner scenario of ermite-conformance
23// (see its _source).
24type conformance struct {
25 GithubKeyOrder []string `json:"_github_key_order"`
26 JobKeyOrder []string `json:"_job_key_order"`
27 RunnerOwned []string `json:"_runner_owned_github_keys"`
28 Github map[string]any `json:"github"`
29 Job map[string]any `json:"job"`
30 Matrix any `json:"matrix"`
31 Strategy map[string]any `json:"strategy"`
32}
33
34func loadConformance(t *testing.T) conformance {
35 t.Helper()
36 b, err := os.ReadFile("testdata/conformance/ctx-job-runner.contexts.json")
37 require.NoError(t, err)
38 var c conformance
39 require.NoError(t, json.Unmarshal(b, &c))
40 return c
41}
42
43// numericPlaceholders are the placeholders of the normaliser of
44// ermite-conformance for values that are numbers (or times) on github.com: ids,
45// times and the repository size. In the raw bundle of the same scenario id,
46// created_at, pushed_at and size are JSON numbers; the fixture only has the
47// placeholder, a string. They are matched by "number or string".
48var numericPlaceholders = []string{"<id:", "<time>", "<size>", "<actor_id:", "<repository_id:", "<repository_owner_id:", "<check_run_id:"}
49
50// shape is the type of each value of an object. A placeholder of a number
51// has the type "scalar".
52func shape(m map[string]any) map[string]string {
53 out := map[string]string{}
54 for k, v := range m {
55 if s, ok := v.(string); ok && slices.ContainsFunc(numericPlaceholders, func(p string) bool { return strings.HasPrefix(s, p) }) {
56 out[k] = "scalar"
57 continue
58 }
59 switch v.(type) {
60 case nil:
61 out[k] = "null"
62 case string:
63 out[k] = "string"
64 case float64:
65 out[k] = "number"
66 case bool:
67 out[k] = "bool"
68 case []any:
69 out[k] = "array"
70 case map[string]any:
71 out[k] = "object"
72 }
73 }
74 return out
75}
76
77// orderedKeys returns the keys of a JSON object in document order.
78func orderedKeys(t *testing.T, raw json.RawMessage) []string {
79 t.Helper()
80 dec := json.NewDecoder(strings.NewReader(string(raw)))
81 tok, err := dec.Token()
82 require.NoError(t, err)
83 require.Equal(t, json.Delim('{'), tok)
84 var keys []string
85 for dec.More() {
86 k, err := dec.Token()
87 require.NoError(t, err)
88 keys = append(keys, k.(string))
89 var skip json.RawMessage
90 require.NoError(t, dec.Decode(&skip))
91 }
92 return keys
93}
94
95// sameShape compares the shape of a recorded object with ours: a "scalar"
96// (numeric placeholder) accepts a number or a string. The description is a
97// property of the recorded repository, which ermite's test repository lacks.
98func sameShape(t *testing.T, want, got map[string]any, what string) {
99 t.Helper()
100 ws, gs := shape(want), shape(got)
101 for k, wt := range ws {
102 if k == "description" {
103 continue
104 }
105 if wt == "scalar" {
106 assert.Contains(t, []string{"number", "string"}, gs[k], "%s.%s", what, k)
107 continue
108 }
109 assert.Equal(t, wt, gs[k], "%s.%s", what, k)
110 }
111}
112
113// The github and job contexts of a push have the keys, the key order and the
114// value types of the ones github.com recorded for the same scenario shape
115// (ermite-conformance, ctx-job-runner), apart from the keys the runner adds.
116func TestCIContextsMatchTheGitHubRecording(t *testing.T) {
117 want := loadConformance(t)
118 c := newCIHarness(t, nil)
119 runner := c.registerRunner("alpha", "ubuntu-latest")
120 c.push("main", map[string]string{".github/workflows/ci.yml": workflowWith("branches: [main]")}, "")
121 job := runner.await()
122 var raw map[string]json.RawMessage
123 require.NoError(t, json.Unmarshal([]byte(job.GetContexts()), &raw))
124 var ctxs map[string]map[string]any
125 require.NoError(t, json.Unmarshal([]byte(job.GetContexts()), &ctxs))
126 runner.runJob(job, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
127
128 t.Run("github", func(t *testing.T) {
129 gh := ctxs["github"]
130 // The runner owns some keys and adds job and token; the server sends the rest.
131 serverKeys := slices.DeleteFunc(slices.Clone(want.GithubKeyOrder), func(k string) bool { return slices.Contains(want.RunnerOwned, k) })
132 assert.Equal(t, serverKeys, orderedKeys(t, raw["github"]), "key set and order")
133 serverGithub := map[string]any{}
134 for k, v := range want.Github {
135 if !slices.Contains(want.RunnerOwned, k) {
136 serverGithub[k] = v
137 }
138 }
139 sameShape(t, serverGithub, gh, "github")
140 assert.Equal(t, "Actions", gh["secret_source"])
141 assert.Equal(t, "10", gh["artifact_cache_size_limit"])
142 assert.Equal(t, "private", gh["repository_visibility"], "from the repository")
143 assert.Equal(t, "owner", gh["actor"])
144 assert.Equal(t, "owner", gh["repository_owner"])
145 assert.Regexp(t, `^[1-9][0-9]*$`, gh["actor_id"])
146 assert.Regexp(t, `^[1-9][0-9]*$`, gh["repository_id"])
147 assert.Regexp(t, `^[1-9][0-9]*$`, gh["repository_owner_id"])
148 assert.Equal(t, gh["actor_id"], gh["repository_owner_id"], "the owner pushed")
149 assert.Equal(t, "git://example.test/owner/proj.git", gh["repositoryUrl"], "git://<host>/<owner>/<repo>.git, as GitHub's")
150 })
151
152 t.Run("event", func(t *testing.T) {
153 ev := ctxs["github"]["event"].(map[string]any)
154 wev := want.Github["event"].(map[string]any)
155 assert.ElementsMatch(t, keysSlice(wev), keysSlice(ev), "event keys")
156 sameShape(t, wev, ev, "event")
157 repo, wrepo := ev["repository"].(map[string]any), wev["repository"].(map[string]any)
158 assert.Len(t, wrepo, 83, "the recording has 83 repository keys")
159 assert.ElementsMatch(t, keysSlice(wrepo), keysSlice(repo), "repository keys")
160 sameShape(t, wrepo, repo, "event.repository")
161 assert.ElementsMatch(t, keysSlice(wrepo["owner"].(map[string]any)), keysSlice(repo["owner"].(map[string]any)), "repository.owner keys")
162 sender, wsender := ev["sender"].(map[string]any), wev["sender"].(map[string]any)
163 assert.Len(t, wsender, 19)
164 assert.ElementsMatch(t, keysSlice(wsender), keysSlice(sender), "sender keys")
165 sameShape(t, wsender, sender, "event.sender")
166 assert.ElementsMatch(t, keysSlice(wev["pusher"].(map[string]any)), keysSlice(ev["pusher"].(map[string]any)))
167 for _, got := range []map[string]any{ev["head_commit"].(map[string]any), ev["commits"].([]any)[0].(map[string]any)} {
168 wc := wev["head_commit"].(map[string]any)
169 assert.ElementsMatch(t, keysSlice(wc), keysSlice(got), "commit keys: no added/removed/modified/username")
170 assert.ElementsMatch(t, keysSlice(wc["author"].(map[string]any)), keysSlice(got["author"].(map[string]any)))
171 assert.ElementsMatch(t, keysSlice(wc["committer"].(map[string]any)), keysSlice(got["committer"].(map[string]any)))
172 assert.True(t, strings.HasPrefix(got["url"].(string), publicOrigin+"/owner/proj/commit/"), got["url"])
173 }
174 assert.Equal(t, "proj", repo["name"])
175 assert.Equal(t, "owner/proj", repo["full_name"])
176 assert.Equal(t, true, repo["private"])
177 assert.Equal(t, "private", repo["visibility"])
178 assert.Equal(t, "main", repo["default_branch"])
179 assert.Equal(t, ctxs["github"]["repository_id"], jsonNumber(repo["id"]), "repository_id is repository.id")
180 assert.Equal(t, ctxs["github"]["actor_id"], jsonNumber(sender["id"]))
181 assert.Equal(t, ctxs["github"]["repository_owner_id"], jsonNumber(repo["owner"].(map[string]any)["id"]))
182 })
183
184 t.Run("job", func(t *testing.T) {
185 j := ctxs["job"]
186 // The runner adds status first; the server sends the rest, in GitHub's order.
187 serverKeys := slices.DeleteFunc(slices.Clone(want.JobKeyOrder), func(k string) bool { return k == "status" })
188 assert.Equal(t, serverKeys, orderedKeys(t, raw["job"]))
189 serverJob := map[string]any{}
190 for k, v := range want.Job {
191 if k != "status" {
192 serverJob[k] = v
193 }
194 }
195 sameShape(t, serverJob, j, "job")
196 assert.Equal(t, ".github/workflows/ci.yml", j["workflow_file_path"])
197 assert.Equal(t, "owner/proj", j["workflow_repository"])
198 assert.Equal(t, "owner/proj/.github/workflows/ci.yml@refs/heads/main", j["workflow_ref"])
199 assert.Equal(t, ctxs["github"]["sha"], j["workflow_sha"])
200 assert.Equal(t, ctxs["github"]["workflow_ref"], j["workflow_ref"])
201 })
202
203 t.Run("matrix is null without a matrix", func(t *testing.T) {
204 assert.Equal(t, "null", string(raw["matrix"]))
205 assert.Nil(t, want.Matrix)
206 assert.ElementsMatch(t, keysSlice(want.Strategy), keysSlice(ctxs["strategy"]))
207 })
208}
209
210func keysSlice(m map[string]any) []string {
211 out := make([]string, 0, len(m))
212 for k := range m {
213 out = append(out, k)
214 }
215 return out
216}
217
218func jsonNumber(v any) string {
219 b, _ := json.Marshal(v)
220 return string(b)
221}
222
223// The run title is the evaluated run-name, else the head commit message. The
224// name of the run is the evaluated run-name too (recorded: ctx-github-names,
225// name = title = the run-name), else the workflow name; github.workflow stays
226// the workflow name.
227func TestCIRunTitle(t *testing.T) {
228 c := newCIHarness(t, nil)
229 c.registerRunner("alpha", "ubuntu-latest")
230 c.push("main", map[string]string{".github/workflows/named.yml": `name: Named
231run-name: "run ${{ github.event_name }} on ${{ github.ref_type }} by ${{ github.actor == '' && 'nobody' || 'someone' }}"
232on: push
233jobs:
234 j:
235 runs-on: ubuntu-latest
236 steps:
237 - run: echo hi
238`, ".github/workflows/plain.yml": workflowWith("branches: [main]")}, "the head commit\n\nbody")
239 c.waitFor("two runs", func() bool { return len(c.runs()) == 2 })
240 titles, names, workflows := map[string]string{}, map[string]string{}, map[string]string{}
241 for _, r := range c.runs() {
242 titles[r.GetWorkflowPath()] = r.GetTitle()
243 names[r.GetWorkflowPath()] = r.GetName()
244 workflows[r.GetWorkflowPath()] = r.GetWorkflowName()
245 }
246 assert.Equal(t, map[string]string{
247 ".github/workflows/named.yml": "run push on branch by someone",
248 ".github/workflows/plain.yml": "the head commit",
249 }, titles)
250 assert.Equal(t, map[string]string{
251 ".github/workflows/named.yml": "run push on branch by someone",
252 ".github/workflows/plain.yml": ".github/workflows/plain.yml",
253 }, names, "the run-name, else the workflow name")
254 assert.Equal(t, map[string]string{
255 ".github/workflows/named.yml": "Named",
256 ".github/workflows/plain.yml": ".github/workflows/plain.yml",
257 }, workflows, "workflow_name is unchanged by a run-name")
258}
259
260// An over-256 matrix is an invalid workflow, like on github.com
261// (matrix-over-256): a failed run with no job.
262func TestCIMatrixOver256IsAnInvalidWorkflow(t *testing.T) {
263 c := newCIHarness(t, nil)
264 c.registerRunner("alpha", "ubuntu-latest")
265 var a, b []string
266 for i := 1; i <= 17; i++ {
267 a = append(a, itoa(i))
268 }
269 for i := 1; i <= 16; i++ {
270 b = append(b, itoa(i))
271 }
272 c.push("main", map[string]string{".github/workflows/m.yml": `on: push
273jobs:
274 too-many:
275 runs-on: ubuntu-latest
276 strategy:
277 matrix:
278 a: [` + strings.Join(a, ", ") + `]
279 b: [` + strings.Join(b, ", ") + `]
280 steps:
281 - run: echo
282`}, "")
283 c.waitFor("the run fails", func() bool { return len(c.runs()) == 1 && c.runs()[0].GetStatus() == pb.CiStatus_CI_STATUS_COMPLETED })
284 run := c.getRun(c.runs()[0].GetId())
285 assert.Equal(t, pb.CiConclusion_CI_CONCLUSION_FAILURE, run.GetRun().GetConclusion())
286 assert.Empty(t, run.GetJobs(), "no job")
287 assert.Equal(t, ".github/workflows/m.yml", run.GetRun().GetWorkflowName(), "no name: in this file: the path")
288 assert.Contains(t, run.GetRun().GetError(), "Strategy for job 'too-many' produced 272 configurations which exceeds the maximum of 256 configurations")
289}
290
291func itoa(i int) string { b, _ := json.Marshal(i); return string(b) }
292
293// A step with continue-on-error is no longer refused; the runner runs it.
294func TestCIStepContinueOnErrorIsPlanned(t *testing.T) {
295 c := newCIHarness(t, nil)
296 runner := c.registerRunner("alpha", "ubuntu-latest")
297 c.push("main", map[string]string{".github/workflows/s.yml": `on: push
298jobs:
299 j:
300 runs-on: ubuntu-latest
301 steps:
302 - continue-on-error: ${{ github.event_name == 'push' }}
303 run: exit 1
304`}, "")
305 job := runner.await()
306 assert.Equal(t, "j", job.GetJobKey())
307 runner.runJob(job, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
308}
309
310// CI-2: planning runs what CI-1 refused (uses, container, services, a step
311// timeout).
312func TestCIPlanningAcceptsCI2Features(t *testing.T) {
313 c := newCIHarness(t, nil)
314 runner := c.registerRunner("alpha", "ubuntu-latest")
315 c.push("main", map[string]string{".github/workflows/s.yml": `on: push
316jobs:
317 j:
318 runs-on: ubuntu-latest
319 container: alpine:3
320 services:
321 db:
322 image: postgres:16
323 steps:
324 - uses: actions/checkout@v4
325 - timeout-minutes: 1
326 run: echo
327`}, "")
328 job := runner.await()
329 assert.Equal(t, "j", job.GetJobKey())
330}
331
332// A failed job with continue-on-error does not fail the run, keeps its own
333// failure conclusion, and shows as success to the jobs that need it (recorded:
334// if-job-level-fail-effects, where the dependent job printed result=success).
335func TestCIJobContinueOnError(t *testing.T) {
336 c := newCIHarness(t, nil)
337 runner := c.registerRunner("alpha", "ubuntu-latest")
338 c.push("main", map[string]string{".github/workflows/c.yml": `on: push
339jobs:
340 soft:
341 runs-on: ubuntu-latest
342 continue-on-error: ${{ github.event_name == 'push' }}
343 steps:
344 - run: exit 5
345 after:
346 needs: soft
347 runs-on: ubuntu-latest
348 steps:
349 - run: echo "result=${{ needs.soft.result }}"
350`}, "")
351 soft := runner.await()
352 assert.Equal(t, "soft", soft.GetJobKey())
353 runner.runJob(soft, pb.CiConclusion_CI_CONCLUSION_FAILURE, nil, "x")
354 after := runner.await()
355 assert.Equal(t, "after", after.GetJobKey(), "a needed job that failed softly does not skip its dependents")
356 assert.Contains(t, after.GetContexts(), `"result":"success"`)
357 runner.runJob(after, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
358 c.waitFor("the run ends", func() bool { return c.runs()[0].GetStatus() == pb.CiStatus_CI_STATUS_COMPLETED })
359 assert.Equal(t, pb.CiConclusion_CI_CONCLUSION_SUCCESS, c.runs()[0].GetConclusion(), "the run does not fail")
360 byKey := map[string]pb.CiConclusion{}
361 for _, j := range c.getRun(c.runs()[0].GetId()).GetJobs() {
362 byKey[j.GetName()] = j.GetConclusion()
363 }
364 assert.Equal(t, pb.CiConclusion_CI_CONCLUSION_FAILURE, byKey["soft"], "the job itself failed")
365}
366
367// A matrix leg with continue-on-error that fails does not cancel its siblings
368// (fail-fast), and does not fail the run; a leg without it still does.
369func TestCIJobContinueOnErrorLegDoesNotTriggerFailFast(t *testing.T) {
370 c := newCIHarness(t, nil)
371 runner := c.registerRunner("alpha", "ubuntu-latest")
372 c.push("main", map[string]string{".github/workflows/m.yml": `on: push
373jobs:
374 build:
375 runs-on: ubuntu-latest
376 continue-on-error: ${{ matrix.experimental }}
377 strategy:
378 matrix:
379 experimental: [true, false]
380 steps:
381 - run: echo ${{ matrix.experimental }}
382`}, "")
383 a, b := runner.await(), runner.await()
384 soft, hard := a, b
385 if strings.Contains(b.GetMatrix(), "true") {
386 soft, hard = b, a
387 }
388 apiHard := runner.jobAPI(hard)
389 _, err := apiHard.runner.Heartbeat(c.ctx, connect.NewRequest(&pb.HeartbeatRequest{JobId: hard.GetJobId()}))
390 require.NoError(t, err)
391 runner.runJob(soft, pb.CiConclusion_CI_CONCLUSION_FAILURE, nil, "x")
392 // The sibling must not have been asked to stop.
393 watch, err := apiHard.runner.WatchJob(c.ctx, connect.NewRequest(&pb.WatchJobRequest{JobId: hard.GetJobId()}))
394 require.NoError(t, err)
395 done := make(chan *pb.CancelReason, 1)
396 go func() {
397 for watch.Receive() {
398 if m := watch.Msg().GetCancel(); m != nil {
399 r := m.GetReason()
400 done <- &r
401 return
402 }
403 }
404 done <- nil
405 }()
406 runner.runJob(hard, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
407 c.waitFor("the run ends", func() bool { return c.runs()[0].GetStatus() == pb.CiStatus_CI_STATUS_COMPLETED })
408 assert.Equal(t, pb.CiConclusion_CI_CONCLUSION_SUCCESS, c.runs()[0].GetConclusion())
409 if r := <-done; r != nil {
410 t.Fatalf("the sibling of a continue-on-error leg was cancelled: %v", *r)
411 }
412}
413
414// GitHub-shaped URLs, /<owner>/<repo>/..., redirect to ermite's pages, only for
415// the owner of the instance and only where there is no page of its own.
416func TestWebRedirectsOwnerURLs(t *testing.T) {
417 c := newCIHarness(t, nil)
418 b := browser{c.harness}
419 c.push("main", map[string]string{"README": "x\n"}, "")
420 sha := strings.TrimSpace(c.mustGit(c.wt, "rev-parse", "HEAD"))
421 for path, to := range map[string]string{
422 "/owner/proj": "/proj",
423 "/owner/proj/commit/" + sha: "/proj/commit/" + sha,
424 "/owner/proj/compare/a...b?x=1": "/proj/compare/a...b?x=1",
425 } {
426 res := b.get(path, nil)
427 assert.Equal(t, http.StatusPermanentRedirect, res.status, path)
428 assert.Equal(t, to, res.hdr.Get("Location"), path)
429 }
430 // The git URLs of the owner are served by git over HTTP, not redirected
431 // (checkout clones <server>/<owner>/<repo>): the private repository asks
432 // for credentials, a dumb request has no route.
433 for path, want := range map[string]int{
434 "/owner/proj.git/info/refs?service=git-upload-pack": http.StatusUnauthorized,
435 "/owner/proj/info/refs?service=git-upload-pack": http.StatusUnauthorized,
436 "/other/proj/info/refs?service=git-upload-pack": http.StatusNotFound,
437 "/owner/proj.git/info/refs": http.StatusNotFound,
438 } {
439 assert.Equal(t, want, b.get(path, nil).status, path)
440 }
441 assert.Equal(t, http.StatusNotFound, b.get("/other/proj/commit/"+sha, nil).status, "not the owner")
442 assert.Equal(t, http.StatusNotFound, b.get("/owner", nil).status, "a bare owner is no page")
443 assert.Equal(t, http.StatusNotFound, b.get("/owner/Bad Name", nil).status)
444 post := b.do("POST", "/owner/proj", map[string]string{"Origin": publicOrigin}, "")
445 assert.Equal(t, http.StatusForbidden, post.status, "a write is stopped by the gate, never redirected")
446 assert.Empty(t, post.hdr.Get("Location"))
447}
448
449// github.job is empty while a job is evaluated, so a name that reads it loses
450// it (recorded: ctx-github-names), but the job gets its key in the contexts.
451func TestCIJobNameReadsAnEmptyGithubJob(t *testing.T) {
452 c := newCIHarness(t, nil)
453 runner := c.registerRunner("alpha", "ubuntu-latest")
454 c.push("main", map[string]string{".github/workflows/n.yml": `on: push
455jobs:
456 first-job:
457 name: Display name of ${{ github.job }}
458 runs-on: ubuntu-latest
459 steps:
460 - run: echo hi
461`}, "")
462 job := runner.await()
463 assert.Contains(t, job.GetContexts(), `"job":"first-job"`)
464 runner.runJob(job, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
465 c.waitFor("the run ends", func() bool { return c.runs()[0].GetStatus() == pb.CiStatus_CI_STATUS_COMPLETED })
466 assert.Equal(t, "Display name of", c.getRun(c.runs()[0].GetId()).GetJobs()[0].GetName())
467}
468
469// A tag push carries the commits that no branch has, newest as head_commit, and
470// the compare URL of several commits is "<first>^...<last>" (recorded:
471// skip-directive-tag, where no branch exists).
472func TestCITagEventCarriesTheNewCommits(t *testing.T) {
473 c := newCIHarness(t, nil)
474 runner := c.registerRunner("alpha", "ubuntu-latest")
475 c.push("main", map[string]string{".github/workflows/rel.yml": workflowWith("tags: ['v*']")}, "")
476 c.mustGit(c.wt, "checkout", "-q", "-b", "rel")
477 c.writeFile(c.wt, "a.txt", []byte("a\n")) // commits "add a.txt"
478 first := strings.TrimSpace(c.mustGit(c.wt, "rev-parse", "HEAD"))
479 c.writeFile(c.wt, "b.txt", []byte("b\n")) // commits "add b.txt"
480 last := strings.TrimSpace(c.mustGit(c.wt, "rev-parse", "HEAD"))
481 c.mustGit(c.wt, "tag", "v1")
482 c.mustGit(c.wt, "push", "-q", c.urlAuth(adminToken, "/proj.git"), "v1")
483
484 job := runner.await()
485 var ctxs map[string]map[string]any
486 require.NoError(t, json.Unmarshal([]byte(job.GetContexts()), &ctxs))
487 runner.runJob(job, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
488 ev := ctxs["github"]["event"].(map[string]any)
489 var msgs []string
490 for _, cm := range ev["commits"].([]any) {
491 msgs = append(msgs, cm.(map[string]any)["message"].(string))
492 }
493 assert.Equal(t, []string{"add a.txt", "add b.txt"}, msgs)
494 assert.Equal(t, "add b.txt", ev["head_commit"].(map[string]any)["message"])
495 assert.Equal(t, publicOrigin+"/owner/proj/compare/"+first[:12]+"^..."+last[:12], ev["compare"])
496 assert.Equal(t, "tag", ctxs["github"]["ref_type"])
497 assert.Equal(t, true, ev["created"])
498}
499
500// tagEvent pushes the tag and returns the push event and the title of the run
501// it starts, after `want` runs exist.
502func (c *ciHarness) tagEvent(runner *fakeRunner, tag string, want int) (map[string]any, string) {
503 c.t.Helper()
504 c.mustGit(c.wt, "tag", tag)
505 c.mustGit(c.wt, "push", "-q", c.urlAuth(adminToken, "/proj.git"), tag)
506 job := runner.await()
507 var ctxs map[string]map[string]any
508 require.NoError(c.t, json.Unmarshal([]byte(job.GetContexts()), &ctxs))
509 runner.runJob(job, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
510 c.waitFor("the run exists", func() bool { return len(c.runs()) == want })
511 var title string
512 for _, r := range c.runs() {
513 if r.GetRef() == "refs/tags/"+tag {
514 title = r.GetTitle()
515 }
516 }
517 return ctxs["github"]["event"].(map[string]any), title
518}
519
520func eventMessages(ev map[string]any) []string {
521 var msgs []string
522 for _, cm := range ev["commits"].([]any) {
523 msgs = append(msgs, cm.(map[string]any)["message"].(string))
524 }
525 return msgs
526}
527
528// A tag on a commit that a branch has carries that commit alone, and the
529// compare URL is its page, not a range from the previous push (recorded:
530// filter-tags, filter-paths-tag, filter-branches-ignore). The tag run is
531// titled with the workflow name (filter-*).
532func TestCITagEventOfACommitABranchHas(t *testing.T) {
533 c := newCIHarness(t, nil)
534 runner := c.registerRunner("alpha", "ubuntu-latest")
535 // Two commits on main, pushed with the workflow; both are on a branch.
536 c.push("main", map[string]string{".github/workflows/rel.yml": "name: Release\non:\n push:\n tags: ['v*']\njobs:\n j:\n runs-on: ubuntu-latest\n steps:\n - run: echo hi\n"}, "")
537 older := strings.TrimSpace(c.mustGit(c.wt, "rev-parse", "HEAD"))
538 c.writeFile(c.wt, "x.txt", []byte("x\n"))
539 c.mustGit(c.wt, "push", "-q", c.urlAuth(adminToken, "/proj.git"), "main")
540 newer := strings.TrimSpace(c.mustGit(c.wt, "rev-parse", "HEAD"))
541
542 ev, title := c.tagEvent(runner, "v1", 1)
543 assert.Equal(t, []string{"add x.txt"}, eventMessages(ev), "the tagged commit alone, not the push range")
544 assert.Equal(t, newer, ev["head_commit"].(map[string]any)["id"])
545 assert.Equal(t, publicOrigin+"/owner/proj/commit/"+newer[:12], ev["compare"])
546 assert.Equal(t, "Release", title, "a tag run is titled with the workflow name, not the commit message")
547
548 // A tag on an older commit of the branch: that commit alone.
549 c.mustGit(c.wt, "tag", "v0", older)
550 c.mustGit(c.wt, "push", "-q", c.urlAuth(adminToken, "/proj.git"), "v0")
551 job := runner.await()
552 var ctxs map[string]map[string]any
553 require.NoError(t, json.Unmarshal([]byte(job.GetContexts()), &ctxs))
554 runner.runJob(job, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
555 ev = ctxs["github"]["event"].(map[string]any)
556 assert.Equal(t, older, ev["after"])
557 assert.Equal(t, older, ev["head_commit"].(map[string]any)["id"])
558 assert.Equal(t, publicOrigin+"/owner/proj/commit/"+older[:12], ev["compare"])
559}
560
561// A tag on a new commit on top of the branch (recorded: push-tag) carries that
562// commit alone: the commit of the branch under it is not part of the event.
563func TestCITagEventOfANewCommitOnTopOfABranch(t *testing.T) {
564 c := newCIHarness(t, nil)
565 runner := c.registerRunner("alpha", "ubuntu-latest")
566 c.push("main", map[string]string{".github/workflows/ci.yml": "on:\n push:\n branches: [main]\n tags: ['v*']\njobs:\n j:\n runs-on: ubuntu-latest\n steps:\n - run: echo hi\n"}, "branch commit")
567 first := runner.await()
568 runner.runJob(first, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
569 c.waitFor("the branch run ends", func() bool { return len(c.runs()) == 1 && c.runs()[0].GetStatus() == pb.CiStatus_CI_STATUS_COMPLETED })
570 assert.Equal(t, "branch commit", c.runs()[0].GetTitle(), "a branch run keeps the commit message as title")
571
572 c.writeFile(c.wt, "t.txt", []byte("t\n")) // "add t.txt", only the tag is pushed
573 tagged := strings.TrimSpace(c.mustGit(c.wt, "rev-parse", "HEAD"))
574 ev, title := c.tagEvent(runner, "v1", 2)
575 assert.Equal(t, []string{"add t.txt"}, eventMessages(ev))
576 assert.Equal(t, publicOrigin+"/owner/proj/commit/"+tagged[:12], ev["compare"])
577 assert.Equal(t, ".github/workflows/ci.yml", title, "no name:, so the workflow name is the path of the file (assumed, not recorded)")
578 assert.Equal(t, true, ev["created"])
579 assert.Equal(t, domain.ZeroOID, ev["before"])
580}
581
582// Several tags in one push start one run each, with the same event: the
583// commit alone (recorded: push-tags-limit), titled with the workflow name.
584func TestCIThreeTagsInOnePush(t *testing.T) {
585 c := newCIHarness(t, nil)
586 runner := c.registerRunner("alpha", "ubuntu-latest")
587 c.push("main", map[string]string{".github/workflows/t.yml": "name: Tags\non:\n push:\n tags: ['v*']\njobs:\n j:\n runs-on: ubuntu-latest\n steps:\n - run: echo hi\n"}, "")
588 c.writeFile(c.wt, "t.txt", []byte("t\n"))
589 sha := strings.TrimSpace(c.mustGit(c.wt, "rev-parse", "HEAD"))
590 for _, tag := range []string{"v1", "v2", "v3"} {
591 c.mustGit(c.wt, "tag", tag)
592 }
593 c.mustGit(c.wt, "push", "-q", c.urlAuth(adminToken, "/proj.git"), "v1", "v2", "v3")
594 for range 3 {
595 job := runner.await()
596 var ctxs map[string]map[string]any
597 require.NoError(t, json.Unmarshal([]byte(job.GetContexts()), &ctxs))
598 ev := ctxs["github"]["event"].(map[string]any)
599 assert.Equal(t, []string{"add t.txt"}, eventMessages(ev))
600 assert.Equal(t, publicOrigin+"/owner/proj/commit/"+sha[:12], ev["compare"])
601 runner.runJob(job, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
602 }
603 c.waitFor("three runs", func() bool { return len(c.runs()) == 3 })
604 for _, r := range c.runs() {
605 assert.Equal(t, "Tags", r.GetTitle())
606 }
607}
608
609// A tag run without run-name is titled with the name of the workflow registered
610// for its path: the "name:" of the file on the default branch, which can differ
611// from the file at the tag (recorded: push-tag, push-tags-limit, push-delete-tag
612// and skip-directive-tag, whose ci.yml is "push-branch-create" on the default
613// branch of the recording repository and has the scenario's own name at the
614// tag; and the files with a unique name, where both agree). Its name stays the
615// workflow name of the tagged file.
616func TestCITagTitleIsTheNameRegisteredOnTheDefaultBranch(t *testing.T) {
617 c := newCIHarness(t, nil)
618 runner := c.registerRunner("alpha", "ubuntu-latest")
619 wf := func(name string) map[string]string {
620 return map[string]string{".github/workflows/ci.yml": "name: " + name + "\non:\n push:\n tags: ['v*']\njobs:\n j:\n runs-on: ubuntu-latest\n steps:\n - run: echo hi\n"}
621 }
622 c.push("main", wf("registered"), "")
623 // The tagged commit is off every branch and renames the workflow.
624 c.mustGit(c.wt, "checkout", "-q", "--detach")
625 c.writeFile(c.wt, ".github/workflows/ci.yml", []byte(wf("renamed")[".github/workflows/ci.yml"]))
626 _, title := c.tagEvent(runner, "v1", 1)
627 assert.Equal(t, "registered", title, "the name of ci.yml on the default branch")
628 r := c.runs()[0]
629 assert.Equal(t, "renamed", r.GetWorkflowName(), "the workflow of the run is the file at the tag (github.workflow)")
630 assert.Equal(t, "renamed", r.GetName(), "and so is the name of the run, which has no run-name")
631
632 // A file the default branch does not have: the run's own workflow name.
633 c.writeFile(c.wt, ".github/workflows/other.yml", []byte("name: other\non:\n push:\n tags: ['w*']\njobs:\n j:\n runs-on: ubuntu-latest\n steps:\n - run: echo hi\n"))
634 _, title = c.tagEvent(runner, "w1", 2)
635 assert.Equal(t, "other", title)
636}
637
638// A tag run with a run-name keeps the evaluated run-name as its title (the
639// run-name of a tag push is not recorded; branch pushes are, ctx-github-names).
640func TestCITagRunKeepsItsRunName(t *testing.T) {
641 c := newCIHarness(t, nil)
642 runner := c.registerRunner("alpha", "ubuntu-latest")
643 c.push("main", map[string]string{".github/workflows/t.yml": "name: Tags\nrun-name: \"release ${{ github.ref_name }}\"\non:\n push:\n tags: ['v*']\njobs:\n j:\n runs-on: ubuntu-latest\n steps:\n - run: echo hi\n"}, "")
644 _, title := c.tagEvent(runner, "v1", 1)
645 assert.Equal(t, "release v1", title)
646}
647
648// An exclude entry with a key that is no key of the matrix makes the workflow
649// invalid, and the run is named after the file (recorded: matrix-edge-values,
650// a-exclude-unknown-key.yml: name = path, no job), while an over-256 matrix
651// keeps the name of the workflow (matrix-over-256).
652func TestCIInvalidStrategyRunNames(t *testing.T) {
653 c := newCIHarness(t, nil)
654 c.registerRunner("alpha", "ubuntu-latest")
655 var a, b []string
656 for i := 1; i <= 17; i++ {
657 a = append(a, itoa(i))
658 }
659 for i := 1; i <= 16; i++ {
660 b = append(b, itoa(i))
661 }
662 c.push("main", map[string]string{
663 ".github/workflows/over.yml": `name: matrix-over-256
664on: push
665jobs:
666 too-many:
667 runs-on: ubuntu-latest
668 strategy:
669 matrix:
670 a: [` + strings.Join(a, ", ") + `]
671 b: [` + strings.Join(b, ", ") + `]
672 steps:
673 - run: echo
674`,
675 ".github/workflows/exclude.yml": `name: a-exclude-unknown-key
676on: push
677jobs:
678 main:
679 runs-on: ubuntu-latest
680 strategy:
681 matrix:
682 os: [one, two]
683 exclude:
684 - nokey: x
685 - os: one
686 nokey: y
687 steps:
688 - run: echo "os=${{ matrix.os }}"
689`,
690 ".github/workflows/noname.yml": `on: push
691jobs:
692 main:
693 runs-on: ubuntu-latest
694 strategy:
695 matrix:
696 os: [one]
697 exclude:
698 - nokey: x
699 steps:
700 - run: echo
701`}, "")
702 c.waitFor("three failed runs", func() bool {
703 rs := c.runs()
704 if len(rs) != 3 {
705 return false
706 }
707 for _, r := range rs {
708 if r.GetStatus() != pb.CiStatus_CI_STATUS_COMPLETED {
709 return false
710 }
711 }
712 return true
713 })
714 names := map[string]string{}
715 for _, r := range c.runs() {
716 names[r.GetWorkflowPath()] = r.GetWorkflowName()
717 assert.Equal(t, pb.CiConclusion_CI_CONCLUSION_FAILURE, r.GetConclusion(), r.GetWorkflowPath())
718 assert.Empty(t, c.getRun(r.GetId()).GetJobs(), "no job: %s", r.GetWorkflowPath())
719 if r.GetWorkflowPath() == ".github/workflows/exclude.yml" {
720 one := ".github/workflows/exclude.yml: Matrix exclude key 'nokey' does not match any key within the matrix"
721 assert.Equal(t, one+"\n"+one, r.GetError(), "once per offending entry")
722 }
723 }
724 assert.Equal(t, map[string]string{
725 ".github/workflows/over.yml": "matrix-over-256",
726 ".github/workflows/exclude.yml": ".github/workflows/exclude.yml",
727 ".github/workflows/noname.yml": ".github/workflows/noname.yml",
728 }, names)
729 // The check of a failed run carries the same workflow name.
730 sha := strings.TrimSpace(c.mustGit(c.wt, "rev-parse", "HEAD"))
731 checkNames := map[string]string{}
732 for _, ck := range c.checks(sha) {
733 checkNames[ck.GetName()] = ck.GetWorkflowName()
734 }
735 assert.Equal(t, "matrix-over-256", checkNames[".github/workflows/over.yml"])
736 assert.Equal(t, ".github/workflows/exclude.yml", checkNames[".github/workflows/exclude.yml"])
737}
738
739// The default success() of a job looks at the whole chain of its needs, not
740// only at the direct ones (recorded: needs-failed-dependency): chain-after-always
741// needs with-always, which ran through always() after broken failed, and is
742// skipped. with-always and with-failure still run.
743func TestCIDefaultSuccessLooksAtTheWholeChain(t *testing.T) {
744 c := newCIHarness(t, nil)
745 runner := c.registerRunner("alpha", "ubuntu-latest")
746 c.push("main", map[string]string{".github/workflows/n.yml": `name: needs-failed-dependency
747on: push
748jobs:
749 broken:
750 runs-on: ubuntu-latest
751 steps:
752 - run: exit 1
753 plain:
754 needs: broken
755 runs-on: ubuntu-latest
756 steps:
757 - run: echo SHOULD-NOT-RUN
758 with-always:
759 needs: broken
760 if: always()
761 runs-on: ubuntu-latest
762 steps:
763 - run: echo "result=${{ needs.broken.result }}"
764 with-failure:
765 needs: broken
766 if: failure()
767 runs-on: ubuntu-latest
768 steps:
769 - run: echo "result=${{ needs.broken.result }}"
770 chain-after-always:
771 needs: with-always
772 runs-on: ubuntu-latest
773 steps:
774 - run: echo chain-after-always-runs-because-with-always-succeeded
775 chain-with-failure:
776 needs: with-always
777 if: failure()
778 runs-on: ubuntu-latest
779 steps:
780 - run: echo "failure() is true when an ancestor failed"
781 chain-always:
782 needs: with-always
783 if: always()
784 runs-on: ubuntu-latest
785 steps:
786 - run: echo always
787`}, "")
788 broken := runner.await()
789 assert.Equal(t, "broken", broken.GetJobKey())
790 runner.runJob(broken, pb.CiConclusion_CI_CONCLUSION_FAILURE, nil, "x")
791 ran := map[string]bool{}
792 for range 4 {
793 j := runner.await()
794 ran[j.GetJobKey()] = true
795 runner.runJob(j, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
796 }
797 assert.Equal(t, map[string]bool{"with-always": true, "with-failure": true, "chain-with-failure": true, "chain-always": true}, ran)
798 c.waitFor("the run ends", func() bool { return c.runs()[0].GetStatus() == pb.CiStatus_CI_STATUS_COMPLETED })
799 got := map[string]pb.CiConclusion{}
800 for _, j := range c.getRun(c.runs()[0].GetId()).GetJobs() {
801 got[j.GetName()] = j.GetConclusion()
802 }
803 skipped, ok := pb.CiConclusion_CI_CONCLUSION_SKIPPED, pb.CiConclusion_CI_CONCLUSION_SUCCESS
804 assert.Equal(t, map[string]pb.CiConclusion{
805 "broken": pb.CiConclusion_CI_CONCLUSION_FAILURE, "plain": skipped, "with-always": ok, "with-failure": ok,
806 "chain-after-always": skipped, "chain-with-failure": ok, "chain-always": ok,
807 }, got)
808}