1// SPDX-License-Identifier: AGPL-3.0-or-later
2
3package usecase
4
5import (
6 "context"
7 "errors"
8 "fmt"
9 "slices"
10 "strings"
11 "sync"
12 "time"
13
14 "github.com/bnema/zerowrap"
15
16 "bnema.dev/ermite/internal/domain"
17 "bnema.dev/ermite/internal/ports"
18)
19
20// ResolveActions turns the "uses" of a job into where to download the actions
21// (ADR 0025, CI-2), as GitHub's action download info does: ermite resolves, the
22// runner downloads from the sandbox.
23//
24// - owner = ERMITE_OWNER and the repository exists on ermite: ermite wins (as
25// the actions of a GitHub Enterprise Server do). The ref is a tag, else a
26// branch, else a full commit id that exists; the archive is served by the
27// REST subset and downloaded with a read token for that repository only.
28// - anything else: github.com. The ref is resolved with `git ls-remote` (tag,
29// then branch, the peeled commit of an annotated tag); a full commit id is
30// returned as it is, without checking. The archive is codeload's, with no
31// token.
32//
33// The commit a ref resolves to is kept for the run attempt: every job and every
34// call of the attempt gets the same one, the first that was stored. A
35// reference that cannot be resolved is not an error of the call: its answer
36// carries GitHub's message.
37func (c *CI) ResolveActions(ctx context.Context, secret, jobID string, uses []string) ([]ports.ResolvedAction, error) {
38 out, err := c.resolveActions(ctx, secret, jobID, uses)
39 if err != nil && ctx.Err() == nil && errors.Is(err, context.DeadlineExceeded) {
40 // Our own deadline, not the caller's: github.com (or the disk) was too
41 // slow. The runner may retry, or fail the job.
42 return nil, fmt.Errorf("%w: resolving the actions took more than %s", domain.ErrUnavailable, c.cfg.ResolveTimeout)
43 }
44 return out, err
45}
46
47func (c *CI) resolveActions(ctx context.Context, secret, jobID string, uses []string) ([]ports.ResolvedAction, error) {
48 ctx, cancel := context.WithTimeout(ctx, c.cfg.ResolveTimeout)
49 defer cancel()
50 j, err := c.jobFor(ctx, secret, jobID)
51 if err != nil {
52 return nil, err
53 }
54 if j.Ended() {
55 return nil, endedError(j)
56 }
57 if len(uses) > maxActionRefs {
58 return nil, fmt.Errorf("%w: at most %d actions per call", domain.ErrInvalid, maxActionRefs)
59 }
60 parsed := make([]actionRef, len(uses))
61 for i, u := range uses {
62 if strings.HasPrefix(u, "./") || strings.HasPrefix(u, ".\\") {
63 return nil, fmt.Errorf("%w: %q is a local action: the runner reads it from the workspace", domain.ErrInvalid, u)
64 }
65 if strings.HasPrefix(u, "docker://") {
66 return nil, fmt.Errorf("%w: %q is a Docker image: the runner pulls it", domain.ErrInvalid, u)
67 }
68 parsed[i] = parseActionRef(u)
69 }
70
71 src, err := c.cfg.Store.GetRunSource(ctx, j.RunID)
72 if err != nil {
73 return nil, err
74 }
75 by := c.onBehalfOf(src)
76
77 // Resolve what the attempt has not resolved yet, outside the lock of the
78 // run: a ls-remote takes seconds and must not hold up the job's calls.
79 found := map[string]domain.CIActionResolution{}
80 fails := map[string]string{}
81 remote := c.newRemoteRefs(j.RunAttempt)
82 for _, p := range parsed {
83 if p.err != "" {
84 continue
85 }
86 if _, done := found[p.key()]; done {
87 continue
88 }
89 if _, bad := fails[p.key()]; bad {
90 continue
91 }
92 if res, err := c.cfg.Store.GetActionResolution(ctx, j.RunID, j.RunAttempt, p.key()); err == nil {
93 found[p.key()] = res
94 continue
95 } else if !errors.Is(err, domain.ErrNotFound) {
96 return nil, err
97 }
98 res, msg, err := c.resolveAction(ctx, j, p, remote, by)
99 if err != nil {
100 return nil, err
101 }
102 if msg != "" {
103 fails[p.key()] = msg
104 continue
105 }
106 found[p.key()] = res
107 }
108
109 out := make([]ports.ResolvedAction, len(parsed))
110 err = c.lockedJob(ctx, j, func(cur *domain.CIJob) error {
111 if cur.Ended() {
112 return endedError(*cur)
113 }
114 rs, err := c.loadRun(ctx, cur.RunID, cur.RunAttempt)
115 if err != nil {
116 return err
117 }
118 own := rs.find(cur.ID)
119 if own == nil {
120 return domain.ErrNotFound
121 }
122 for i, p := range parsed {
123 out[i] = ports.ResolvedAction{Ref: p.uses}
124 if p.err != "" {
125 out[i].Error = p.err
126 continue
127 }
128 if msg, bad := fails[p.key()]; bad {
129 out[i].Error = msg
130 continue
131 }
132 // The first resolution stored wins, ours or a concurrent job's.
133 res, ok := found[p.key()]
134 if cached, err := c.cfg.Store.GetActionResolution(ctx, cur.RunID, cur.RunAttempt, p.key()); err == nil {
135 res, ok = cached, true
136 } else if !errors.Is(err, domain.ErrNotFound) {
137 return err
138 } else if ok {
139 rs.resolutions = append(rs.resolutions, res)
140 found[p.key()] = res // two refs of the call with the same key
141 }
142 if !ok {
143 continue
144 }
145 ra, err := c.answer(ctx, rs, own, p, res, by)
146 if err != nil {
147 return err
148 }
149 out[i] = ra
150 }
151 if err := rs.flush(ctx); err != nil {
152 // The tokens were not stored: forget their secrets.
153 c.mu.Lock()
154 delete(c.archiveTokens, jobRef(*cur))
155 c.mu.Unlock()
156 return err
157 }
158 return nil
159 })
160 if err != nil {
161 return nil, err
162 }
163 return out, nil
164}
165
166// maxActionRefs is the most references of one ResolveActions call.
167const maxActionRefs = 100
168
169// answer builds the answer for a resolved reference. The archive token of an
170// ermite action is bound to the commit of the answer: the one the job already
171// holds for it, else a new one, written with the next flush. Whether the pusher
172// may read the repository is asked again at each answer, not only when the
173// commit was resolved: a token revoked since, or a repository made private,
174// stops the answers.
175func (c *CI) answer(ctx context.Context, rs *runState, j *domain.CIJob, p actionRef, res domain.CIActionResolution, by pusherAccess) (ports.ResolvedAction, error) {
176 ra := ports.ResolvedAction{
177 Ref: p.uses, NameWithOwner: res.NameWithOwner, Path: p.path, ResolvedSHA: res.ResolvedSHA, Source: res.Source,
178 }
179 notFound := ports.ResolvedAction{Ref: p.uses, Error: errUnresolvedRepo(p)}
180
181 // A repository of this server that the pusher cannot read is never served
182 // from anywhere else: not from github.com under the same name, whatever
183 // source an earlier resolution of the attempt stored. It answers as a
184 // repository that does not exist. (When it does not exist on this server, a
185 // stored ermite resolution has nothing to serve either.)
186 repo, onErmite, err := c.ermiteRepoOf(ctx, p)
187 if err != nil {
188 return ports.ResolvedAction{}, err
189 }
190 if onErmite {
191 if ok, err := c.pusherCanRead(ctx, by, rs.run.Repo, repo); err != nil {
192 return ports.ResolvedAction{}, err
193 } else if !ok {
194 return notFound, nil
195 }
196 }
197 switch res.Source {
198 case domain.CIActionGitHub:
199 // The case the workflow wrote: the stored resolution is shared by
200 // spellings that differ only by case, and github.com ignores it.
201 ra.NameWithOwner = p.owner + "/" + p.repo
202 ra.TarballURL = "https://codeload.github.com/" + ra.NameWithOwner + "/tar.gz/" + res.ResolvedSHA
203 case domain.CIActionErmite:
204 if !onErmite {
205 return notFound, nil
206 }
207 ra.TarballURL = c.cfg.PublicURL + apiPath + "/repos/" + res.NameWithOwner + "/tarball/" + res.ResolvedSHA
208 ra.Token = c.archiveToken(rs, j, repo.Name, res.ResolvedSHA)
209 }
210 return ra, nil
211}
212
213// archiveToken returns the secret of the token of the job for the archive of
214// the commit sha of repo, issuing it when the job has none. The token downloads
215// that archive and nothing else (domain.RepoTokenPrincipal).
216func (c *CI) archiveToken(rs *runState, j *domain.CIJob, repo domain.RepoName, sha string) string {
217 ref, key := jobRef(*j), repo.String()+"@"+sha
218 c.mu.Lock()
219 tok, ok := c.archiveTokens[ref][key]
220 c.mu.Unlock()
221 if ok {
222 return tok
223 }
224 tok = rs.issueRepoTokenFor(j, repo, sha)
225 c.mu.Lock()
226 if c.archiveTokens[ref] == nil {
227 c.archiveTokens[ref] = map[string]string{}
228 }
229 c.archiveTokens[ref][key] = tok
230 c.mu.Unlock()
231 return tok
232}
233
234// ermiteRepoOf finds the repository of this server that a reference names: the
235// owner is ERMITE_OWNER (without regard to case) and the repository exists. ok
236// is false for any other reference, which github.com serves. It is the one
237// place that decides the source of an action, for a first resolution and for the
238// answer from a stored one.
239func (c *CI) ermiteRepoOf(ctx context.Context, a actionRef) (repo domain.Repo, ok bool, err error) {
240 if !domain.OwnerMatches(c.cfg.Owner, a.owner) {
241 return domain.Repo{}, false, nil
242 }
243 name, perr := domain.ParseRepoName(strings.ToLower(a.repo))
244 if perr != nil {
245 return domain.Repo{}, false, nil // not a name ermite can hold
246 }
247 repo, err = c.cfg.Meta.Get(ctx, name)
248 switch {
249 case errors.Is(err, domain.ErrNotFound):
250 return domain.Repo{}, false, nil
251 case err != nil:
252 return domain.Repo{}, false, err
253 }
254 return repo, true, nil
255}
256
257// pusherAccess is who the actions of a run are resolved on behalf of: the
258// pusher recorded with the run.
259type pusherAccess struct{ pusher domain.CIPusher }
260
261func (c *CI) onBehalfOf(src domain.CIRunSource) pusherAccess { return pusherAccess{src.Pusher} }
262
263// pusherCanRead tells whether the pusher of a run may read repo, with the rule
264// of the forge (canRead) applied to the principal the pusher is now: the root
265// token and the owner's session read everything; a stored token is looked up
266// again, so a token revoked or expired since reads nothing, and its scopes and
267// repository list apply; a run that records no pusher reads what an anonymous
268// caller reads. The repository of the run itself needs no check: the job reads
269// it with its own token.
270func (c *CI) pusherCanRead(ctx context.Context, by pusherAccess, runRepo domain.RepoName, repo domain.Repo) (bool, error) {
271 if repo.Name == runRepo {
272 return true, nil
273 }
274 var p domain.Principal
275 switch by.pusher.Kind {
276 case domain.CIPusherRoot:
277 p = domain.Root()
278 case domain.CIPusherOwner:
279 p = domain.Owner()
280 case domain.CIPusherToken:
281 if c.cfg.Tokens == nil {
282 return false, nil
283 }
284 tok, err := c.cfg.Tokens.Get(ctx, by.pusher.TokenID)
285 if errors.Is(err, domain.ErrNotFound) {
286 return false, nil
287 }
288 if err != nil {
289 return false, err
290 }
291 if tok.Expired(c.now()) {
292 return false, nil
293 }
294 p = domain.TokenPrincipal(tok)
295 default:
296 p = domain.Anonymous()
297 }
298 return canRead(p, repo), nil
299}
300
301// actionRef is a parsed "uses": owner/repo[/path]@ref.
302type actionRef struct {
303 uses, owner, repo, path, ref string
304 // err is GitHub's message for a reference of the wrong shape.
305 err string
306}
307
308// key identifies the resolution: GitHub names are case-insensitive, refs are
309// not.
310func (a actionRef) key() string { return strings.ToLower(a.owner+"/"+a.repo) + "@" + a.ref }
311
312// parseActionRef splits a "uses" of a repository action. Only the shape
313// owner/repo[/path]@ref is valid; GitHub splits the path on '/' and '\' and
314// ignores empty segments (WorkflowTemplateConverter.cs), so do we.
315func parseActionRef(uses string) actionRef {
316 a := actionRef{uses: uses}
317 left, ref, ok := strings.Cut(uses, "@")
318 segs := strings.FieldsFunc(left, func(r rune) bool { return r == '/' || r == '\\' })
319 if !ok || strings.Contains(ref, "@") || len(segs) < 2 || ref == "" {
320 a.err = errActionFormat(uses)
321 return a
322 }
323 for _, s := range segs[2:] {
324 if s == "." || s == ".." {
325 a.err = errActionFormat(uses)
326 return a
327 }
328 }
329 if !validActionName(segs[0]) || !validActionName(segs[1]) {
330 a.err = errActionFormat(uses)
331 return a
332 }
333 a.owner, a.repo, a.ref = segs[0], segs[1], ref
334 a.path = strings.Join(segs[2:], "/")
335 return a
336}
337
338// validActionName reports whether s can be an owner or a repository name on
339// GitHub: letters, digits, '.', '_' and '-', 1 to 100 characters, not made of
340// dots only ("." and ".." are path elements). Every source takes the same names, so that nothing but a name reaches a
341// URL, a path or a command.
342func validActionName(s string) bool {
343 if s == "" || len(s) > 100 || strings.Trim(s, ".") == "" {
344 return false
345 }
346 for _, r := range s {
347 switch {
348 case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '.', r == '_', r == '-':
349 default:
350 return false
351 }
352 }
353 return true
354}
355
356// --- GitHub's texts ---
357//
358// All the messages a reference can fail with are here. The first is the
359// runner's own parser text; the two others are the ones GitHub gives for an
360// unknown version and an unknown repository, to be confirmed by recording
361// against github.com (ci2-execution plan, phase A).
362
363func errActionFormat(uses string) string {
364 return "Expected format {org}/{repo}[/path]@ref. Actual '" + uses + "'"
365}
366
367func errUnresolvedRef(a actionRef) string {
368 return "Unable to resolve action `" + a.owner + "/" + a.repo + "@" + a.ref + "`, unable to find version `" + a.ref + "`"
369}
370
371func errUnresolvedRepo(a actionRef) string {
372 return "Unable to resolve action `" + a.owner + "/" + a.repo + "@" + a.ref + "`, repository not found"
373}
374
375// resolveAction resolves one reference. msg is the message of a reference that
376// does not resolve; err is a failure of the call.
377func (c *CI) resolveAction(ctx context.Context, j domain.CIJob, a actionRef, remote *remoteRefs, by pusherAccess) (res domain.CIActionResolution, msg string, err error) {
378 res = domain.CIActionResolution{RunID: j.RunID, RunAttempt: j.RunAttempt, Key: a.key()}
379 repo, onErmite, err := c.ermiteRepoOf(ctx, a)
380 if err != nil {
381 return res, "", err
382 }
383 if onErmite {
384 // The repository exists here, so github.com never answers for it. A
385 // repository the pusher may not read answers as one that does not exist,
386 // before any ref is looked at, so that the texts leak nothing about it.
387 if ok, err := c.pusherCanRead(ctx, by, j.Repo, repo); err != nil {
388 return res, "", err
389 } else if !ok {
390 return res, errUnresolvedRepo(a), nil
391 }
392 sha, found, err := c.resolveOnErmite(ctx, repo.Name, a.ref)
393 if err != nil {
394 return res, "", err
395 }
396 if !found {
397 return res, errUnresolvedRef(a), nil
398 }
399 res.NameWithOwner, res.ResolvedSHA, res.Source = c.cfg.Owner+"/"+repo.Name.String(), sha, domain.CIActionErmite
400 return res, "", nil
401 }
402 res.NameWithOwner, res.Source = a.owner+"/"+a.repo, domain.CIActionGitHub
403 if low := strings.ToLower(a.ref); isHex40(low) {
404 res.ResolvedSHA = low // a commit id is not checked: a missing one fails the download
405 return res, "", nil
406 }
407 refs, err := remote.list(ctx, a.owner, a.repo)
408 switch {
409 case errors.Is(err, domain.ErrNotFound):
410 return res, errUnresolvedRepo(a), nil
411 case errors.Is(err, domain.ErrUnavailable) && ctx.Err() == nil:
412 // The job learns which action and why, not only "unavailable".
413 return res, "", fmt.Errorf("%w: resolving `%s`: github.com did not answer (%v)", domain.ErrUnavailable, a.uses, err)
414 case err != nil:
415 return res, "", err
416 }
417 sha := pickRemoteRef(refs, a.ref)
418 if sha == "" {
419 return res, errUnresolvedRef(a), nil
420 }
421 res.ResolvedSHA = sha
422 return res, "", nil
423}
424
425// pickRemoteRef finds the commit of a tag, else of a branch, in the refs of a
426// remote. An annotated tag gives its peeled commit.
427func pickRemoteRef(refs []ports.RemoteRef, name string) string {
428 for _, prefix := range []string{"refs/tags/", "refs/heads/"} {
429 for _, r := range refs {
430 if r.Name != prefix+name {
431 continue
432 }
433 if r.Peeled != "" {
434 return r.Peeled
435 }
436 return r.ID
437 }
438 }
439 return ""
440}
441
442// resolveOnErmite resolves a ref on a repository of this server: a tag, else
443// a branch, else a full commit id that exists.
444func (c *CI) resolveOnErmite(ctx context.Context, repo domain.RepoName, ref string) (sha string, found bool, err error) {
445 refs, err := c.cfg.Reader.ListRefs(ctx, repo)
446 if err != nil && !errors.Is(err, domain.ErrNotFound) {
447 return "", false, err
448 }
449 for _, kind := range []domain.RefKind{domain.RefKindTag, domain.RefKindBranch} {
450 i := slices.IndexFunc(refs, func(r domain.Ref) bool {
451 return r.Kind == kind && r.ShortName == ref && !r.CommitID.IsZero()
452 })
453 if i >= 0 {
454 return refs[i].CommitID.String(), true, nil
455 }
456 }
457 if id, perr := domain.ParseObjectID(ref); perr == nil {
458 ok, err := c.cfg.Git.HasCommit(ctx, repo, id)
459 if err != nil && !errors.Is(err, domain.ErrNotFound) {
460 return "", false, err
461 }
462 if ok {
463 return id.String(), true, nil
464 }
465 }
466 return "", false, nil
467}
468
469// remoteRefs lists the refs of a remote once per repository within a call,
470// through the cache of the server (see refsCache).
471type remoteRefs struct {
472 remote ports.GitRemote
473 slots chan struct{}
474 cache *refsCache
475 now func() time.Time
476 seen map[string]remoteList
477 // refresh asks github.com even when the cache is fresh: a re-run resolves
478 // again (a moved tag is often why it is re-run). The cache still stands
479 // in when github.com does not answer.
480 refresh bool
481}
482
483type remoteList struct {
484 refs []ports.RemoteRef
485 err error
486}
487
488// newRemoteRefs lists for the jobs of an attempt: a re-run (attempt > 1)
489// refreshes.
490func (c *CI) newRemoteRefs(attempt int32) *remoteRefs {
491 return &remoteRefs{remote: c.cfg.Remote, slots: c.remoteSlots, cache: c.remoteCache, now: c.now, seen: map[string]remoteList{}, refresh: attempt > 1}
492}
493
494func (r *remoteRefs) list(ctx context.Context, owner, repo string) ([]ports.RemoteRef, error) {
495 if r.remote == nil {
496 return nil, fmt.Errorf("%w: this server cannot reach github.com", domain.ErrUnavailable)
497 }
498 key := strings.ToLower(owner + "/" + repo)
499 if l, ok := r.seen[key]; ok {
500 return l.refs, l.err
501 }
502 if refs, fresh := r.cache.get(key, r.now()); fresh && !r.refresh {
503 r.seen[key] = remoteList{refs, nil}
504 return refs, nil
505 }
506 // At most a few ls-remote at once on the whole server, whatever the number
507 // of jobs resolving: a process each, and github.com is slow to answer.
508 select {
509 case r.slots <- struct{}{}:
510 defer func() { <-r.slots }()
511 case <-ctx.Done():
512 return nil, ctx.Err()
513 }
514 // Another job may have listed it while this one waited for a slot.
515 if refs, fresh := r.cache.get(key, r.now()); fresh && !r.refresh {
516 r.seen[key] = remoteList{refs, nil}
517 return refs, nil
518 }
519 refs, err := r.remote.ListRefs(ctx, owner, repo)
520 if ctx.Err() != nil {
521 return nil, ctx.Err()
522 }
523 switch {
524 case err == nil:
525 r.cache.put(key, refs, r.now())
526 case errors.Is(err, domain.ErrUnavailable):
527 // github.com did not answer: the last list it gave is better than a
528 // failed job, for a while (refsStale).
529 if old, ok := r.cache.stale(key, r.now()); ok {
530 log := zerowrap.FromCtx(ctx)
531 log.Warn().Err(err).Str("repo", key).Msg("github.com did not answer: using the refs listed earlier")
532 refs, err = old, nil
533 }
534 }
535 r.seen[key] = remoteList{refs, err}
536 return refs, err
537}
538
539const (
540 // refsFresh is how long a list of the refs of a repository of github.com
541 // is used without asking again: the jobs of the next minutes resolve
542 // actions/checkout@v4 without a ls-remote. A tag or branch moved meanwhile
543 // is seen after at most that long.
544 refsFresh = 5 * time.Minute
545 // refsStale is how long a list is kept for when github.com does not answer.
546 refsStale = 24 * time.Hour
547 // refsMax bounds the repositories in the cache.
548 refsMax = 256
549)
550
551// refsCache keeps the refs of the repositories of github.com the actions come
552// from, for the whole server. Safe for concurrent use.
553type refsCache struct {
554 mu sync.Mutex
555 m map[string]cachedRefs
556}
557
558type cachedRefs struct {
559 refs []ports.RemoteRef
560 at time.Time
561}
562
563func newRefsCache() *refsCache { return &refsCache{m: map[string]cachedRefs{}} }
564
565// get returns the list of key and whether it is fresh.
566func (c *refsCache) get(key string, now time.Time) ([]ports.RemoteRef, bool) {
567 c.mu.Lock()
568 defer c.mu.Unlock()
569 e, ok := c.m[key]
570 return e.refs, ok && now.Sub(e.at) < refsFresh
571}
572
573// stale returns the list of key when it is not older than refsStale.
574func (c *refsCache) stale(key string, now time.Time) ([]ports.RemoteRef, bool) {
575 c.mu.Lock()
576 defer c.mu.Unlock()
577 e, ok := c.m[key]
578 return e.refs, ok && now.Sub(e.at) < refsStale
579}
580
581func (c *refsCache) put(key string, refs []ports.RemoteRef, now time.Time) {
582 c.mu.Lock()
583 defer c.mu.Unlock()
584 if _, ok := c.m[key]; !ok && len(c.m) >= refsMax {
585 var oldest string
586 for k, e := range c.m {
587 if oldest == "" || e.at.Before(c.m[oldest].at) {
588 oldest = k
589 }
590 }
591 delete(c.m, oldest)
592 }
593 c.m[key] = cachedRefs{refs: refs, at: now}
594}