1// SPDX-License-Identifier: AGPL-3.0-or-later
2
3package usecase
4
5import (
6 "context"
7 "errors"
8 "fmt"
9 "io"
10 "path"
11 "runtime/debug"
12 "slices"
13 "strconv"
14 "strings"
15 "time"
16
17 "github.com/bnema/zerowrap"
18
19 "bnema.dev/ermite-actions/expr"
20 "bnema.dev/ermite-actions/template"
21 "bnema.dev/ermite-actions/workflow"
22 "bnema.dev/ermite/internal/domain"
23 "bnema.dev/ermite/internal/ports"
24)
25
26const (
27 workflowDir = ".github/workflows"
28 // maxRunError bounds Run.error (8 KiB, run.proto).
29 maxRunError = 8 << 10
30 // maxPushCommits is the number of commits of the push event payload.
31 maxPushCommits = 20
32)
33
34// ciTrigger is a push waiting to be planned.
35type ciTrigger struct {
36 repo domain.RepoName
37 updates []domain.RefUpdate
38 actor string
39 at time.Time
40 log zerowrap.Logger
41}
42
43// OnPush queues the refs of a push or a merge for planning and returns at once;
44// a worker plans them. A full queue drops the trigger, logged at error: a push
45// is never slowed down or failed by CI.
46func (c *CI) OnPush(ctx context.Context, repo domain.RepoName, updates []domain.RefUpdate, pusher domain.Principal) {
47 var kept []domain.RefUpdate
48 for _, u := range updates {
49 if !u.IsDelete() && (strings.HasPrefix(u.Ref, "refs/heads/") || strings.HasPrefix(u.Ref, "refs/tags/")) {
50 kept = append(kept, u)
51 }
52 }
53 if len(kept) == 0 {
54 return
55 }
56 t := ciTrigger{repo: repo, updates: kept, actor: authorOf(pusher), at: c.now(), log: zerowrap.FromCtx(ctx)}
57 c.mu.Lock()
58 closed := c.closed
59 c.mu.Unlock()
60 if closed {
61 return
62 }
63 select {
64 case c.queue <- t:
65 default:
66 t.log.Error().Str("repo", repo.String()).Int("refs", len(kept)).Msg("ci trigger queue is full: the workflows of this push are not run")
67 }
68}
69
70func (c *CI) triggerLoop() {
71 defer c.wg.Done()
72 for {
73 select {
74 case <-c.ctx.Done():
75 return
76 case t := <-c.queue:
77 c.planPush(t)
78 }
79 }
80}
81
82// planPush plans one push; a panic or an error is logged and never escapes.
83func (c *CI) planPush(t ciTrigger) {
84 ctx := zerowrap.WithCtx(c.ctx, t.log)
85 defer func() {
86 if r := recover(); r != nil {
87 t.log.Error().Interface("panic", r).Bytes("stack", debug.Stack()).Str("repo", t.repo.String()).Msg("ci trigger panicked")
88 }
89 }()
90 if err := c.handlePush(ctx, t); err != nil && ctx.Err() == nil {
91 t.log.Error().Err(err).Str("repo", t.repo.String()).Msg("ci trigger failed")
92 }
93}
94
95// handlePush starts the workflows that match each ref of the push.
96func (c *CI) handlePush(ctx context.Context, t ciTrigger) error {
97 // CI is on once a runner is registered (plan, "Config"): without one a push
98 // makes no run and no check, so a forge that does not use CI shows nothing.
99 runners, err := c.cfg.Store.ListRunners(ctx)
100 if err != nil {
101 return fmt.Errorf("list the runners: %w", err)
102 }
103 if len(runners) == 0 {
104 return nil
105 }
106 tags := 0
107 for _, u := range t.updates {
108 if strings.HasPrefix(u.Ref, "refs/tags/") {
109 tags++
110 }
111 }
112 var errs []error
113 for _, u := range t.updates {
114 if err := c.handleRef(ctx, t, u, tags); err != nil {
115 errs = append(errs, fmt.Errorf("%s: %w", u.Ref, err))
116 }
117 }
118 return errors.Join(errs...)
119}
120
121// pushFacts is what is known of one pushed ref.
122type pushFacts struct {
123 update domain.RefUpdate
124 commit domain.ObjectID // the commit the ref points to, tags followed
125 head domain.Commit
126 commits []domain.Commit // oldest first, at most maxPushCommits
127 forced bool
128 created bool
129 files []string
130 // filesTruncated: the list of changed files is not available.
131 filesTruncated bool
132 tagCount int
133 isTag bool
134 // defaultBranch is the default branch of the repository, "" when unknown.
135 defaultBranch string
136 // payload is the commit part of the push event, shared by the workflows
137 // of the ref; handleRef builds it once, before the first one that matches.
138 payload *pushPayload
139}
140
141func (c *CI) handleRef(ctx context.Context, t ciTrigger, u domain.RefUpdate, tagCount int) error {
142 isTag := strings.HasPrefix(u.Ref, "refs/tags/")
143 if isTag && tagCount > workflow.MaxTagsPerPush {
144 return nil // GitHub creates no event for the tags of such a push
145 }
146 repo, err := c.cfg.Meta.Get(ctx, t.repo)
147 if err != nil {
148 return nil // deleted meanwhile
149 }
150 facts, err := c.collect(ctx, t.repo, u, isTag, tagCount)
151 if err != nil {
152 return err
153 }
154 files, err := c.workflowFiles(ctx, t.repo, facts.commit)
155 if err != nil || len(files) == 0 {
156 return err
157 }
158 skip := workflow.SkipCI(facts.head.Message)
159 var errs []error
160 for _, f := range files {
161 wf, perrs := workflow.Parse(f.path, f.content)
162 switch {
163 case f.problem != "":
164 errs = append(errs, c.startInvalid(ctx, repo, t, facts, f.path, f.problem))
165 case len(perrs) > 0:
166 var lines []string
167 for _, e := range perrs {
168 lines = append(lines, e.Error())
169 }
170 errs = append(errs, c.startInvalid(ctx, repo, t, facts, f.path, strings.Join(lines, "\n")))
171 case skip || !wf.MatchPush(workflow.PushInfo{
172 Ref: u.Ref, Created: facts.created, ChangedFiles: facts.files, FilesTruncated: facts.filesTruncated, TagCount: tagCount,
173 }):
174 default:
175 if facts.payload == nil {
176 facts.payload = c.pushPayload(ctx, repo, facts)
177 }
178 if problem := c.matrixProblem(ctx, repo, t, facts, f.path, wf); problem != "" {
179 errs = append(errs, c.startInvalid(ctx, repo, t, facts, f.path, problem))
180 continue
181 }
182 errs = append(errs, c.startRun(ctx, repo, t, facts, f.path, string(f.content), wf))
183 }
184 }
185 return errors.Join(errs...)
186}
187
188// collect reads what the payload and the filters need from Git.
189func (c *CI) collect(ctx context.Context, repo domain.RepoName, u domain.RefUpdate, isTag bool, tagCount int) (pushFacts, error) {
190 newID, err := domain.ParseObjectID(u.New)
191 if err != nil {
192 return pushFacts{}, err
193 }
194 f := pushFacts{update: u, created: u.IsCreate(), isTag: isTag, tagCount: tagCount}
195 if def, _, err := c.cfg.Repos.DefaultBranch(ctx, repo); err == nil {
196 f.defaultBranch = def
197 }
198 res, err := c.cfg.Reader.ResolveRevision(ctx, repo, domain.RevisionOf(newID))
199 if err != nil {
200 return f, fmt.Errorf("resolve %s: %w", u.New, err)
201 }
202 f.commit = res.CommitID
203 if isTag {
204 // Recorded (push-tag, skip-directive-tag): the event of a tag carries
205 // the commits that the default branch does not have, like a created
206 // branch, newest as head_commit. No file list: paths filters are not
207 // evaluated for tags.
208 base := c.defaultBase(ctx, repo, "", f.commit)
209 cs, _, err := c.cfg.Git.ListRange(ctx, repo, base, f.commit, ports.MaxCommitLimit)
210 if err != nil {
211 return f, fmt.Errorf("read the commit of the tag: %w", err)
212 }
213 if len(cs) == 0 {
214 cs, _, err = c.cfg.Git.ListRange(ctx, repo, domain.ObjectID{}, f.commit, 1)
215 if err != nil {
216 return f, fmt.Errorf("read the commit of the tag: %w", err)
217 }
218 if len(cs) == 0 {
219 return f, fmt.Errorf("read the commit of the tag: %w", domain.ErrNotFound)
220 }
221 f.head = cs[0]
222 return f, nil
223 }
224 f.head = cs[0]
225 f.commits = slices.Clone(cs[:min(len(cs), maxPushCommits)])
226 slices.Reverse(f.commits)
227 return f, nil
228 }
229
230 var exclude domain.ObjectID
231 if !f.created {
232 if old, err := domain.ParseObjectID(u.Old); err == nil {
233 exclude = old
234 if ok, err := c.cfg.Git.IsAncestor(ctx, repo, old, f.commit); err == nil && !ok {
235 f.forced = true
236 }
237 }
238 }
239 base := exclude
240 if f.created {
241 base = c.defaultBase(ctx, repo, strings.TrimPrefix(u.Ref, "refs/heads/"), f.commit)
242 exclude = base
243 }
244 cs, more, err := c.cfg.Git.ListRange(ctx, repo, exclude, f.commit, ports.MaxCommitLimit)
245 if err != nil {
246 return f, fmt.Errorf("read the commits of the push: %w", err)
247 }
248 if len(cs) == 0 {
249 // A new branch at a commit the default branch has, or a branch rewound
250 // to an ancestor: no new commit, no changed file. GitHub still runs
251 // the push workflows that have no paths filter, for the head commit.
252 head, _, err := c.cfg.Git.ListRange(ctx, repo, domain.ObjectID{}, f.commit, 1)
253 if err != nil {
254 return f, fmt.Errorf("read the head commit of the push: %w", err)
255 }
256 if len(head) == 0 {
257 return f, fmt.Errorf("read the head commit of the push: %w", domain.ErrNotFound)
258 }
259 f.head = head[0]
260 return f, nil
261 }
262 f.head = cs[0]
263 n := min(len(cs), maxPushCommits)
264 f.commits = slices.Clone(cs[:n])
265 slices.Reverse(f.commits)
266 if more || base == f.commit {
267 // More than 1,000 commits: the diff is not looked at (it runs).
268 f.filesTruncated = more
269 }
270 if !more && base != f.commit {
271 paths, _, err := c.cfg.Diff.ChangedPaths(ctx, repo, base, f.commit, workflow.MaxFilesEvaluated)
272 if err != nil {
273 f.filesTruncated = true // diff unavailable: runs
274 } else {
275 f.files = paths
276 }
277 }
278 return f, nil
279}
280
281// defaultBase is the merge base of a new branch with the default branch, the
282// zero id when there is none (every file of the branch counts).
283func (c *CI) defaultBase(ctx context.Context, repo domain.RepoName, branch string, head domain.ObjectID) domain.ObjectID {
284 def, exists, err := c.cfg.Repos.DefaultBranch(ctx, repo)
285 if err != nil || !exists || def == branch {
286 return domain.ObjectID{}
287 }
288 tip, err := c.cfg.Git.BranchHead(ctx, repo, def)
289 if err != nil {
290 return domain.ObjectID{}
291 }
292 base, err := c.cfg.Git.MergeBase(ctx, repo, tip, head)
293 if err != nil {
294 return domain.ObjectID{}
295 }
296 return base
297}
298
299type workflowFile struct {
300 path string
301 content []byte
302 // problem is set when the file cannot even be parsed (too large).
303 problem string
304}
305
306// workflowFiles reads .github/workflows/*.yml|*.yaml at the commit.
307func (c *CI) workflowFiles(ctx context.Context, repo domain.RepoName, commit domain.ObjectID) ([]workflowFile, error) {
308 dir, err := domain.ParseTreePath(workflowDir)
309 if err != nil {
310 return nil, err
311 }
312 tree, err := c.cfg.Reader.GetTree(ctx, repo, domain.RevisionOf(commit), dir, false)
313 if errors.Is(err, domain.ErrNotFound) || errors.Is(err, domain.ErrNotDirectory) {
314 return nil, nil
315 }
316 if err != nil {
317 return nil, fmt.Errorf("list %s: %w", workflowDir, err)
318 }
319 var out []workflowFile
320 for _, e := range tree.Entries {
321 ext := strings.ToLower(path.Ext(e.Name))
322 if (e.Kind != domain.EntryFile && e.Kind != domain.EntryExecutable) || (ext != ".yml" && ext != ".yaml") {
323 continue
324 }
325 if len(out) == maxCIWorkflowFiles {
326 break
327 }
328 wf := workflowFile{path: e.Path}
329 if wf.path == "" {
330 wf.path = workflowDir + "/" + e.Name
331 }
332 if e.Size > maxCIWorkflowBytes {
333 wf.problem = fmt.Sprintf("%s: the workflow file is larger than 1 MiB", wf.path)
334 out = append(out, wf)
335 continue
336 }
337 tp, err := domain.ParseTreePath(wf.path)
338 if err != nil {
339 continue
340 }
341 _, rc, err := c.cfg.Reader.OpenBlob(ctx, repo, domain.RevisionOf(commit), tp)
342 if err != nil {
343 return nil, fmt.Errorf("read %s: %w", wf.path, err)
344 }
345 wf.content, err = io.ReadAll(io.LimitReader(rc, maxCIWorkflowBytes+1))
346 _ = rc.Close()
347 if err != nil {
348 return nil, fmt.Errorf("read %s: %w", wf.path, err)
349 }
350 if len(wf.content) > maxCIWorkflowBytes {
351 wf.content, wf.problem = nil, fmt.Sprintf("%s: the workflow file is larger than 1 MiB", wf.path)
352 }
353 out = append(out, wf)
354 }
355 return out, nil
356}
357
358// --- the run ---
359
360func firstLine(s string) string {
361 line, _, _ := strings.Cut(s, "\n")
362 return truncateRunes(strings.TrimSpace(line), 256)
363}
364
365// startInvalid records the failed run of an invalid workflow file: no job, the
366// errors as Run.error, like GitHub's "Invalid workflow file".
367func (c *CI) startInvalid(ctx context.Context, repo domain.Repo, t ciTrigger, f pushFacts, wfPath, problem string) error {
368 now := c.now()
369 draft, err := c.cfg.Store.CreateRun(ctx, repo.Name, wfPath, func(id, number int64) ports.CIRunDraft {
370 run := domain.CIRun{
371 Repo: repo.Name, ID: id, Number: number, Attempt: 1, WorkflowPath: wfPath, WorkflowName: wfPath,
372 Title: firstLine(f.head.Message), Event: "push", Ref: f.update.Ref, Sha: f.commit.String(), Actor: t.actor,
373 Status: domain.CICompleted, Conclusion: domain.CIFailure, Error: cutText(problem, maxRunError),
374 CreatedAt: now, StartedAt: now, CompletedAt: now,
375 }
376 return ports.CIRunDraft{
377 Run: run, Source: domain.CIRunSource{Repo: repo.Name, WorkflowYAML: "", GithubJSON: "{}"},
378 Checks: []domain.CICheck{{
379 ID: "run-" + strconv.FormatInt(id, 10), Repo: repo.Name, Sha: run.Sha, Name: wfPath, Status: domain.CICompleted,
380 Conclusion: domain.CIFailure, RunID: id, Attempt: 1, WorkflowName: wfPath, CreatedAt: now, StartedAt: now, CompletedAt: now,
381 }},
382 }
383 })
384 if errors.Is(err, domain.ErrNotFound) {
385 return nil
386 }
387 if err != nil {
388 return err
389 }
390 c.publishRun(ctx, repo, draft.Run)
391 log := logRun(ctx, draft.Run)
392 log.Info().Str("workflow", wfPath).Msg("ci run failed: invalid workflow file")
393 return nil
394}
395
396func (c *CI) publishRun(ctx context.Context, repo domain.Repo, run domain.CIRun) {
397 c.cfg.Bus.Publish(ctx, domain.RunUpdated{Run: run, At: c.now(), Visibility: repo.Visibility, CreatedAt: repo.CreatedAt})
398}
399
400// startRun creates the run of a matching workflow with one waiting record per
401// job of the file, then plans the jobs that can start.
402func (c *CI) startRun(ctx context.Context, repo domain.Repo, t ciTrigger, f pushFacts, wfPath, yaml string, wf *workflow.Workflow) error {
403 now := c.now()
404 name := wf.Name
405 if name == "" {
406 name = wfPath
407 }
408 draft, err := c.cfg.Store.CreateRun(ctx, repo.Name, wfPath, func(id, number int64) ports.CIRunDraft {
409 run := domain.CIRun{
410 Repo: repo.Name, ID: id, Number: number, Attempt: 1, WorkflowPath: wfPath, WorkflowName: name,
411 Title: firstLine(f.head.Message), Event: "push", Ref: f.update.Ref, Sha: f.commit.String(), Actor: t.actor,
412 Status: domain.CIQueued, CreatedAt: now,
413 }
414 githubJSON := c.githubJSON(ctx, repo, run, f)
415 run.Title = c.runTitle(ctx, wf, githubJSON, run.Title)
416 d := ports.CIRunDraft{Run: run, Source: domain.CIRunSource{
417 Repo: repo.Name, WorkflowYAML: yaml, GithubJSON: githubJSON,
418 }}
419 for i, j := range wf.Jobs {
420 job := domain.CIJob{
421 ID: domain.NewJobID(id, i, 0), RunID: id, Attempt: 1, RunAttempt: 1, Repo: repo.Name, Key: j.ID, Name: j.ID,
422 Phase: domain.CIPhaseWaiting, Needs: slices.Clone(j.Needs), Index: i, CreatedAt: now,
423 }
424 d.Jobs = append(d.Jobs, job)
425 d.Checks = append(d.Checks, domain.CICheck{
426 ID: job.ID, Repo: repo.Name, Sha: run.Sha, Name: job.Name, Status: domain.CIQueued, RunID: id, JobID: job.ID,
427 Attempt: 1, WorkflowName: name, CreatedAt: now,
428 })
429 }
430 return d
431 })
432 if errors.Is(err, domain.ErrNotFound) {
433 return nil
434 }
435 if err != nil {
436 return err
437 }
438 c.publishRun(ctx, repo, draft.Run)
439 for _, j := range draft.Jobs {
440 c.cfg.Bus.Publish(ctx, domain.JobUpdated{Job: j, At: now, Visibility: repo.Visibility, CreatedAt: repo.CreatedAt})
441 }
442 log := logRun(ctx, draft.Run)
443 log.Info().Str("workflow", wfPath).Str("ref", f.update.Ref).Int("jobs", len(draft.Jobs)).Msg("ci run created")
444 return c.advanceRun(ctx, draft.Run.ID)
445}
446
447// runTitle is the title of a run, GitHub's display_title: the run-name
448// evaluated with the github, inputs and vars contexts (the contexts the
449// schema allows there), else the first line of the head commit message. A
450// run-name that does not evaluate, or that evaluates to nothing, falls back to
451// the commit message, and the failure is logged.
452func (c *CI) runTitle(ctx context.Context, wf *workflow.Workflow, githubJSON, commitTitle string) string {
453 if wf.RunName == nil {
454 return commitTitle
455 }
456 gh, err := expr.FromJSON([]byte(githubJSON))
457 if err != nil {
458 return commitTitle
459 }
460 env := template.Env{Contexts: map[string]expr.Value{
461 "github": gh, "inputs": expr.NewDictionary(), "vars": expr.NewDictionary(),
462 }}
463 name, err := template.EvaluateString(ctx, wf.RunName, env)
464 if err != nil {
465 log := zerowrap.FromCtx(ctx)
466 log.Warn().Err(err).Msg("ci run-name does not evaluate: the run is titled with the commit message")
467 return commitTitle
468 }
469 if name = firstLine(name); name == "" {
470 return commitTitle
471 }
472 return name
473}
474
475// matrixProblem expands the strategy of every job of the file, whatever its
476// "if:" says, with an empty needs context (a matrix built from a needs output does not
477// evaluate here, and that error is ignored),
478// and returns the error text of the first strategy that produces more
479// configurations than GitHub allows: GitHub then fails the run with no job,
480// like an invalid workflow file (recorded: matrix-over-256). Any other error
481// of a strategy is ignored here and stays a job failure, found when the job is
482// planned. Whether GitHub counts the matrix of a job that "if:" skips is not
483// recorded yet (needs an "if: false" scenario over 256). The github context
484// is that of a run with no id yet, on the commits shared by the ref.
485func (c *CI) matrixProblem(ctx context.Context, repo domain.Repo, t ciTrigger, f pushFacts, wfPath string, wf *workflow.Workflow) string {
486 if !slices.ContainsFunc(wf.Jobs, func(j *workflow.Job) bool { return j.Strategy != nil }) {
487 return "" // nothing to expand: no context to build
488 }
489 probe := domain.CIRun{Repo: repo.Name, Ref: f.update.Ref, Sha: f.commit.String(), Actor: t.actor, WorkflowPath: wfPath, WorkflowName: wf.Name, CreatedAt: c.now()}
490 gh, err := expr.FromJSON([]byte(c.githubJSON(ctx, repo, probe, f)))
491 if err != nil {
492 return ""
493 }
494 env := template.Env{Contexts: map[string]expr.Value{
495 "github": gh, "needs": expr.NewDictionary(), "inputs": expr.NewDictionary(), "vars": expr.NewDictionary(),
496 }}
497 for _, j := range wf.Jobs {
498 if _, err := j.EvaluateStrategy(ctx, env); errors.Is(err, workflow.ErrTooManyConfigurations) {
499 return wfPath + ": " + err.Error()
500 }
501 }
502 return ""
503}
504
505// --- the github context (see ci_github.go) ---
506
507// shortID is the 12-character form of an object id that GitHub puts in the
508// compare URL of a push.
509func shortID(id string) string {
510 if len(id) > 12 {
511 return id[:12]
512 }
513 return id
514}
515
516// retentionDays is the retention in days, as github.retention_days shows it
517// (at least 1).
518func (c *CI) retentionDays() int {
519 return max(int(c.cfg.Retention/(24*time.Hour)), 1)
520}