1// SPDX-License-Identifier: AGPL-3.0-or-later
2
3package app
4
5import (
6 "encoding/json"
7 "net/http"
8 "os"
9 "slices"
10 "strings"
11 "testing"
12
13 "connectrpc.com/connect"
14 "github.com/stretchr/testify/assert"
15 "github.com/stretchr/testify/require"
16
17 pb "bnema.dev/ermite-api/gen/ermite"
18)
19
20// conformance is testdata/conformance/ctx-job-runner.contexts.json: the
21// contexts github.com gave to the ctx-job-runner scenario of ermite-conformance
22// (see its _source).
23type conformance struct {
24 GithubKeyOrder []string `json:"_github_key_order"`
25 JobKeyOrder []string `json:"_job_key_order"`
26 RunnerOwned []string `json:"_runner_owned_github_keys"`
27 Github map[string]any `json:"github"`
28 Job map[string]any `json:"job"`
29 Matrix any `json:"matrix"`
30 Strategy map[string]any `json:"strategy"`
31}
32
33func loadConformance(t *testing.T) conformance {
34 t.Helper()
35 b, err := os.ReadFile("testdata/conformance/ctx-job-runner.contexts.json")
36 require.NoError(t, err)
37 var c conformance
38 require.NoError(t, json.Unmarshal(b, &c))
39 return c
40}
41
42// numericPlaceholders are the placeholders of the normaliser of
43// ermite-conformance for values that are numbers (or times) on github.com: ids,
44// times and the repository size. In the raw bundle of the same scenario id,
45// created_at, pushed_at and size are JSON numbers; the fixture only has the
46// placeholder, a string. They are matched by "number or string".
47var numericPlaceholders = []string{"<id:", "<time>", "<size>", "<actor_id:", "<repository_id:", "<repository_owner_id:", "<check_run_id:"}
48
49// shape is the type of each value of an object. A placeholder of a number
50// has the type "scalar".
51func shape(m map[string]any) map[string]string {
52 out := map[string]string{}
53 for k, v := range m {
54 if s, ok := v.(string); ok && slices.ContainsFunc(numericPlaceholders, func(p string) bool { return strings.HasPrefix(s, p) }) {
55 out[k] = "scalar"
56 continue
57 }
58 switch v.(type) {
59 case nil:
60 out[k] = "null"
61 case string:
62 out[k] = "string"
63 case float64:
64 out[k] = "number"
65 case bool:
66 out[k] = "bool"
67 case []any:
68 out[k] = "array"
69 case map[string]any:
70 out[k] = "object"
71 }
72 }
73 return out
74}
75
76// orderedKeys returns the keys of a JSON object in document order.
77func orderedKeys(t *testing.T, raw json.RawMessage) []string {
78 t.Helper()
79 dec := json.NewDecoder(strings.NewReader(string(raw)))
80 tok, err := dec.Token()
81 require.NoError(t, err)
82 require.Equal(t, json.Delim('{'), tok)
83 var keys []string
84 for dec.More() {
85 k, err := dec.Token()
86 require.NoError(t, err)
87 keys = append(keys, k.(string))
88 var skip json.RawMessage
89 require.NoError(t, dec.Decode(&skip))
90 }
91 return keys
92}
93
94// sameShape compares the shape of a recorded object with ours: a "scalar"
95// (numeric placeholder) accepts a number or a string. The description is a
96// property of the recorded repository, which ermite's test repository lacks.
97func sameShape(t *testing.T, want, got map[string]any, what string) {
98 t.Helper()
99 ws, gs := shape(want), shape(got)
100 for k, wt := range ws {
101 if k == "description" {
102 continue
103 }
104 if wt == "scalar" {
105 assert.Contains(t, []string{"number", "string"}, gs[k], "%s.%s", what, k)
106 continue
107 }
108 assert.Equal(t, wt, gs[k], "%s.%s", what, k)
109 }
110}
111
112// The github and job contexts of a push have the keys, the key order and the
113// value types of the ones github.com recorded for the same scenario shape
114// (ermite-conformance, ctx-job-runner), apart from the keys the runner adds.
115func TestCIContextsMatchTheGitHubRecording(t *testing.T) {
116 want := loadConformance(t)
117 c := newCIHarness(t, nil)
118 runner := c.registerRunner("alpha", "ubuntu-latest")
119 c.push("main", map[string]string{".github/workflows/ci.yml": workflowWith("branches: [main]")}, "")
120 job := runner.await()
121 var raw map[string]json.RawMessage
122 require.NoError(t, json.Unmarshal([]byte(job.GetContexts()), &raw))
123 var ctxs map[string]map[string]any
124 require.NoError(t, json.Unmarshal([]byte(job.GetContexts()), &ctxs))
125 runner.runJob(job, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
126
127 t.Run("github", func(t *testing.T) {
128 gh := ctxs["github"]
129 // The runner owns some keys and adds job and token; the server sends the rest.
130 serverKeys := slices.DeleteFunc(slices.Clone(want.GithubKeyOrder), func(k string) bool { return slices.Contains(want.RunnerOwned, k) })
131 assert.Equal(t, serverKeys, orderedKeys(t, raw["github"]), "key set and order")
132 serverGithub := map[string]any{}
133 for k, v := range want.Github {
134 if !slices.Contains(want.RunnerOwned, k) {
135 serverGithub[k] = v
136 }
137 }
138 sameShape(t, serverGithub, gh, "github")
139 assert.Equal(t, "Actions", gh["secret_source"])
140 assert.Equal(t, "10", gh["artifact_cache_size_limit"])
141 assert.Equal(t, "private", gh["repository_visibility"], "from the repository")
142 assert.Equal(t, "owner", gh["actor"])
143 assert.Equal(t, "owner", gh["repository_owner"])
144 assert.Regexp(t, `^[1-9][0-9]*$`, gh["actor_id"])
145 assert.Regexp(t, `^[1-9][0-9]*$`, gh["repository_id"])
146 assert.Regexp(t, `^[1-9][0-9]*$`, gh["repository_owner_id"])
147 assert.Equal(t, gh["actor_id"], gh["repository_owner_id"], "the owner pushed")
148 assert.Equal(t, "git://example.test/owner/proj.git", gh["repositoryUrl"], "git://<host>/<owner>/<repo>.git, as GitHub's")
149 })
150
151 t.Run("event", func(t *testing.T) {
152 ev := ctxs["github"]["event"].(map[string]any)
153 wev := want.Github["event"].(map[string]any)
154 assert.ElementsMatch(t, keysSlice(wev), keysSlice(ev), "event keys")
155 sameShape(t, wev, ev, "event")
156 repo, wrepo := ev["repository"].(map[string]any), wev["repository"].(map[string]any)
157 assert.Len(t, wrepo, 83, "the recording has 83 repository keys")
158 assert.ElementsMatch(t, keysSlice(wrepo), keysSlice(repo), "repository keys")
159 sameShape(t, wrepo, repo, "event.repository")
160 assert.ElementsMatch(t, keysSlice(wrepo["owner"].(map[string]any)), keysSlice(repo["owner"].(map[string]any)), "repository.owner keys")
161 sender, wsender := ev["sender"].(map[string]any), wev["sender"].(map[string]any)
162 assert.Len(t, wsender, 19)
163 assert.ElementsMatch(t, keysSlice(wsender), keysSlice(sender), "sender keys")
164 sameShape(t, wsender, sender, "event.sender")
165 assert.ElementsMatch(t, keysSlice(wev["pusher"].(map[string]any)), keysSlice(ev["pusher"].(map[string]any)))
166 for _, got := range []map[string]any{ev["head_commit"].(map[string]any), ev["commits"].([]any)[0].(map[string]any)} {
167 wc := wev["head_commit"].(map[string]any)
168 assert.ElementsMatch(t, keysSlice(wc), keysSlice(got), "commit keys: no added/removed/modified/username")
169 assert.ElementsMatch(t, keysSlice(wc["author"].(map[string]any)), keysSlice(got["author"].(map[string]any)))
170 assert.ElementsMatch(t, keysSlice(wc["committer"].(map[string]any)), keysSlice(got["committer"].(map[string]any)))
171 assert.True(t, strings.HasPrefix(got["url"].(string), publicOrigin+"/owner/proj/commit/"), got["url"])
172 }
173 assert.Equal(t, "proj", repo["name"])
174 assert.Equal(t, "owner/proj", repo["full_name"])
175 assert.Equal(t, true, repo["private"])
176 assert.Equal(t, "private", repo["visibility"])
177 assert.Equal(t, "main", repo["default_branch"])
178 assert.Equal(t, ctxs["github"]["repository_id"], jsonNumber(repo["id"]), "repository_id is repository.id")
179 assert.Equal(t, ctxs["github"]["actor_id"], jsonNumber(sender["id"]))
180 assert.Equal(t, ctxs["github"]["repository_owner_id"], jsonNumber(repo["owner"].(map[string]any)["id"]))
181 })
182
183 t.Run("job", func(t *testing.T) {
184 j := ctxs["job"]
185 // The runner adds status first; the server sends the rest, in GitHub's order.
186 serverKeys := slices.DeleteFunc(slices.Clone(want.JobKeyOrder), func(k string) bool { return k == "status" })
187 assert.Equal(t, serverKeys, orderedKeys(t, raw["job"]))
188 serverJob := map[string]any{}
189 for k, v := range want.Job {
190 if k != "status" {
191 serverJob[k] = v
192 }
193 }
194 sameShape(t, serverJob, j, "job")
195 assert.Equal(t, ".github/workflows/ci.yml", j["workflow_file_path"])
196 assert.Equal(t, "owner/proj", j["workflow_repository"])
197 assert.Equal(t, "owner/proj/.github/workflows/ci.yml@refs/heads/main", j["workflow_ref"])
198 assert.Equal(t, ctxs["github"]["sha"], j["workflow_sha"])
199 assert.Equal(t, ctxs["github"]["workflow_ref"], j["workflow_ref"])
200 })
201
202 t.Run("matrix is null without a matrix", func(t *testing.T) {
203 assert.Equal(t, "null", string(raw["matrix"]))
204 assert.Nil(t, want.Matrix)
205 assert.ElementsMatch(t, keysSlice(want.Strategy), keysSlice(ctxs["strategy"]))
206 })
207}
208
209func keysSlice(m map[string]any) []string {
210 out := make([]string, 0, len(m))
211 for k := range m {
212 out = append(out, k)
213 }
214 return out
215}
216
217func jsonNumber(v any) string {
218 b, _ := json.Marshal(v)
219 return string(b)
220}
221
222// The run title is the evaluated run-name, else the head commit message.
223func TestCIRunTitle(t *testing.T) {
224 c := newCIHarness(t, nil)
225 c.registerRunner("alpha", "ubuntu-latest")
226 c.push("main", map[string]string{".github/workflows/named.yml": `name: Named
227run-name: "run ${{ github.event_name }} on ${{ github.ref_type }} by ${{ github.actor == '' && 'nobody' || 'someone' }}"
228on: push
229jobs:
230 j:
231 runs-on: ubuntu-latest
232 steps:
233 - run: echo hi
234`, ".github/workflows/plain.yml": workflowWith("branches: [main]")}, "the head commit\n\nbody")
235 c.waitFor("two runs", func() bool { return len(c.runs()) == 2 })
236 titles := map[string]string{}
237 for _, r := range c.runs() {
238 titles[r.GetWorkflowPath()] = r.GetTitle()
239 }
240 assert.Equal(t, map[string]string{
241 ".github/workflows/named.yml": "run push on branch by someone",
242 ".github/workflows/plain.yml": "the head commit",
243 }, titles)
244}
245
246// An over-256 matrix is an invalid workflow, like on github.com
247// (matrix-over-256): a failed run with no job.
248func TestCIMatrixOver256IsAnInvalidWorkflow(t *testing.T) {
249 c := newCIHarness(t, nil)
250 c.registerRunner("alpha", "ubuntu-latest")
251 var a, b []string
252 for i := 1; i <= 17; i++ {
253 a = append(a, itoa(i))
254 }
255 for i := 1; i <= 16; i++ {
256 b = append(b, itoa(i))
257 }
258 c.push("main", map[string]string{".github/workflows/m.yml": `on: push
259jobs:
260 too-many:
261 runs-on: ubuntu-latest
262 strategy:
263 matrix:
264 a: [` + strings.Join(a, ", ") + `]
265 b: [` + strings.Join(b, ", ") + `]
266 steps:
267 - run: echo
268`}, "")
269 c.waitFor("the run fails", func() bool { return len(c.runs()) == 1 && c.runs()[0].GetStatus() == pb.CiStatus_CI_STATUS_COMPLETED })
270 run := c.getRun(c.runs()[0].GetId())
271 assert.Equal(t, pb.CiConclusion_CI_CONCLUSION_FAILURE, run.GetRun().GetConclusion())
272 assert.Empty(t, run.GetJobs(), "no job")
273 assert.Contains(t, run.GetRun().GetError(), "Strategy for job 'too-many' produced 272 configurations which exceeds the maximum of 256 configurations")
274}
275
276func itoa(i int) string { b, _ := json.Marshal(i); return string(b) }
277
278// A step with continue-on-error is no longer refused; the runner runs it.
279func TestCIStepContinueOnErrorIsPlanned(t *testing.T) {
280 c := newCIHarness(t, nil)
281 runner := c.registerRunner("alpha", "ubuntu-latest")
282 c.push("main", map[string]string{".github/workflows/s.yml": `on: push
283jobs:
284 j:
285 runs-on: ubuntu-latest
286 steps:
287 - continue-on-error: ${{ github.event_name == 'push' }}
288 run: exit 1
289`}, "")
290 job := runner.await()
291 assert.Equal(t, "j", job.GetJobKey())
292 runner.runJob(job, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
293}
294
295// A step timeout stays refused as long as the runner has none.
296func TestCIStepTimeoutIsStillRefused(t *testing.T) {
297 c := newCIHarness(t, nil)
298 c.registerRunner("alpha", "ubuntu-latest")
299 c.push("main", map[string]string{".github/workflows/s.yml": `on: push
300jobs:
301 j:
302 runs-on: ubuntu-latest
303 steps:
304 - timeout-minutes: 1
305 run: echo
306`}, "")
307 c.waitFor("the run fails", func() bool { return len(c.runs()) == 1 && c.runs()[0].GetStatus() == pb.CiStatus_CI_STATUS_COMPLETED })
308 assert.Equal(t, "not supported yet: timeout-minutes on a step", c.getRun(c.runs()[0].GetId()).GetJobs()[0].GetError())
309}
310
311// A failed job with continue-on-error does not fail the run, keeps its own
312// failure conclusion, and shows as success to the jobs that need it (recorded:
313// if-job-level-fail-effects, where the dependent job printed result=success).
314func TestCIJobContinueOnError(t *testing.T) {
315 c := newCIHarness(t, nil)
316 runner := c.registerRunner("alpha", "ubuntu-latest")
317 c.push("main", map[string]string{".github/workflows/c.yml": `on: push
318jobs:
319 soft:
320 runs-on: ubuntu-latest
321 continue-on-error: ${{ github.event_name == 'push' }}
322 steps:
323 - run: exit 5
324 after:
325 needs: soft
326 runs-on: ubuntu-latest
327 steps:
328 - run: echo "result=${{ needs.soft.result }}"
329`}, "")
330 soft := runner.await()
331 assert.Equal(t, "soft", soft.GetJobKey())
332 runner.runJob(soft, pb.CiConclusion_CI_CONCLUSION_FAILURE, nil, "x")
333 after := runner.await()
334 assert.Equal(t, "after", after.GetJobKey(), "a needed job that failed softly does not skip its dependents")
335 assert.Contains(t, after.GetContexts(), `"result":"success"`)
336 runner.runJob(after, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
337 c.waitFor("the run ends", func() bool { return c.runs()[0].GetStatus() == pb.CiStatus_CI_STATUS_COMPLETED })
338 assert.Equal(t, pb.CiConclusion_CI_CONCLUSION_SUCCESS, c.runs()[0].GetConclusion(), "the run does not fail")
339 byKey := map[string]pb.CiConclusion{}
340 for _, j := range c.getRun(c.runs()[0].GetId()).GetJobs() {
341 byKey[j.GetName()] = j.GetConclusion()
342 }
343 assert.Equal(t, pb.CiConclusion_CI_CONCLUSION_FAILURE, byKey["soft"], "the job itself failed")
344}
345
346// A matrix leg with continue-on-error that fails does not cancel its siblings
347// (fail-fast), and does not fail the run; a leg without it still does.
348func TestCIJobContinueOnErrorLegDoesNotTriggerFailFast(t *testing.T) {
349 c := newCIHarness(t, nil)
350 runner := c.registerRunner("alpha", "ubuntu-latest")
351 c.push("main", map[string]string{".github/workflows/m.yml": `on: push
352jobs:
353 build:
354 runs-on: ubuntu-latest
355 continue-on-error: ${{ matrix.experimental }}
356 strategy:
357 matrix:
358 experimental: [true, false]
359 steps:
360 - run: echo ${{ matrix.experimental }}
361`}, "")
362 a, b := runner.await(), runner.await()
363 soft, hard := a, b
364 if strings.Contains(b.GetMatrix(), "true") {
365 soft, hard = b, a
366 }
367 apiHard := runner.jobAPI(hard)
368 _, err := apiHard.runner.Heartbeat(c.ctx, connect.NewRequest(&pb.HeartbeatRequest{JobId: hard.GetJobId()}))
369 require.NoError(t, err)
370 runner.runJob(soft, pb.CiConclusion_CI_CONCLUSION_FAILURE, nil, "x")
371 // The sibling must not have been asked to stop.
372 watch, err := apiHard.runner.WatchJob(c.ctx, connect.NewRequest(&pb.WatchJobRequest{JobId: hard.GetJobId()}))
373 require.NoError(t, err)
374 done := make(chan *pb.CancelReason, 1)
375 go func() {
376 for watch.Receive() {
377 if m := watch.Msg().GetCancel(); m != nil {
378 r := m.GetReason()
379 done <- &r
380 return
381 }
382 }
383 done <- nil
384 }()
385 runner.runJob(hard, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
386 c.waitFor("the run ends", func() bool { return c.runs()[0].GetStatus() == pb.CiStatus_CI_STATUS_COMPLETED })
387 assert.Equal(t, pb.CiConclusion_CI_CONCLUSION_SUCCESS, c.runs()[0].GetConclusion())
388 if r := <-done; r != nil {
389 t.Fatalf("the sibling of a continue-on-error leg was cancelled: %v", *r)
390 }
391}
392
393// GitHub-shaped URLs, /<owner>/<repo>/..., redirect to ermite's pages, only for
394// the owner of the instance and only where there is no page of its own.
395func TestWebRedirectsOwnerURLs(t *testing.T) {
396 c := newCIHarness(t, nil)
397 b := browser{c.harness}
398 c.push("main", map[string]string{"README": "x\n"}, "")
399 sha := strings.TrimSpace(c.mustGit(c.wt, "rev-parse", "HEAD"))
400 for path, to := range map[string]string{
401 "/owner/proj": "/proj",
402 "/owner/proj/commit/" + sha: "/proj/commit/" + sha,
403 "/owner/proj/compare/a...b?x=1": "/proj/compare/a...b?x=1",
404 "/owner/proj.git/info/refs": "/proj.git/info/refs",
405 } {
406 res := b.get(path, nil)
407 assert.Equal(t, http.StatusPermanentRedirect, res.status, path)
408 assert.Equal(t, to, res.hdr.Get("Location"), path)
409 }
410 assert.Equal(t, http.StatusNotFound, b.get("/other/proj/commit/"+sha, nil).status, "not the owner")
411 assert.Equal(t, http.StatusNotFound, b.get("/owner", nil).status, "a bare owner is no page")
412 assert.Equal(t, http.StatusNotFound, b.get("/owner/Bad Name", nil).status)
413 post := b.do("POST", "/owner/proj", map[string]string{"Origin": publicOrigin}, "")
414 assert.Equal(t, http.StatusForbidden, post.status, "a write is stopped by the gate, never redirected")
415 assert.Empty(t, post.hdr.Get("Location"))
416}
417
418// github.job is empty while a job is evaluated, so a name that reads it loses
419// it (recorded: ctx-github-names), but the job gets its key in the contexts.
420func TestCIJobNameReadsAnEmptyGithubJob(t *testing.T) {
421 c := newCIHarness(t, nil)
422 runner := c.registerRunner("alpha", "ubuntu-latest")
423 c.push("main", map[string]string{".github/workflows/n.yml": `on: push
424jobs:
425 first-job:
426 name: Display name of ${{ github.job }}
427 runs-on: ubuntu-latest
428 steps:
429 - run: echo hi
430`}, "")
431 job := runner.await()
432 assert.Contains(t, job.GetContexts(), `"job":"first-job"`)
433 runner.runJob(job, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
434 c.waitFor("the run ends", func() bool { return c.runs()[0].GetStatus() == pb.CiStatus_CI_STATUS_COMPLETED })
435 assert.Equal(t, "Display name of", c.getRun(c.runs()[0].GetId()).GetJobs()[0].GetName())
436}
437
438// A tag push carries, like a created branch, the commits the default branch
439// lacks, newest as head_commit, and the compare URL of several commits is
440// "<first>^...<last>" (recorded: push-tag, skip-directive-tag).
441func TestCITagEventCarriesTheNewCommits(t *testing.T) {
442 c := newCIHarness(t, nil)
443 runner := c.registerRunner("alpha", "ubuntu-latest")
444 c.push("main", map[string]string{".github/workflows/rel.yml": workflowWith("tags: ['v*']")}, "")
445 c.mustGit(c.wt, "checkout", "-q", "-b", "rel")
446 c.writeFile(c.wt, "a.txt", []byte("a\n")) // commits "add a.txt"
447 first := strings.TrimSpace(c.mustGit(c.wt, "rev-parse", "HEAD"))
448 c.writeFile(c.wt, "b.txt", []byte("b\n")) // commits "add b.txt"
449 last := strings.TrimSpace(c.mustGit(c.wt, "rev-parse", "HEAD"))
450 c.mustGit(c.wt, "tag", "v1")
451 c.mustGit(c.wt, "push", "-q", c.urlAuth(adminToken, "/proj.git"), "v1")
452
453 job := runner.await()
454 var ctxs map[string]map[string]any
455 require.NoError(t, json.Unmarshal([]byte(job.GetContexts()), &ctxs))
456 runner.runJob(job, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil, "x")
457 ev := ctxs["github"]["event"].(map[string]any)
458 var msgs []string
459 for _, cm := range ev["commits"].([]any) {
460 msgs = append(msgs, cm.(map[string]any)["message"].(string))
461 }
462 assert.Equal(t, []string{"add a.txt", "add b.txt"}, msgs)
463 assert.Equal(t, "add b.txt", ev["head_commit"].(map[string]any)["message"])
464 assert.Equal(t, publicOrigin+"/owner/proj/compare/"+first[:12]+"^..."+last[:12], ev["compare"])
465 assert.Equal(t, "tag", ctxs["github"]["ref_type"])
466 assert.Equal(t, true, ev["created"])
467}