1// SPDX-License-Identifier: AGPL-3.0-or-later
2
3package usecase
4
5import (
6 "context"
7 "errors"
8 "fmt"
9 "slices"
10 "strings"
11 "sync"
12 "time"
13
14 "github.com/bnema/zerowrap"
15
16 "bnema.dev/ermite/internal/domain"
17 "bnema.dev/ermite/internal/ports"
18)
19
20// ResolveActions turns the "uses" of a job into where to download the actions
21// (ADR 0025, CI-2), as GitHub's action download info does: ermite resolves, the
22// runner downloads from the sandbox.
23//
24// - owner = ERMITE_OWNER and the repository exists on ermite: ermite wins (as
25// the actions of a GitHub Enterprise Server do). The ref is a tag, else a
26// branch, else a full commit id that exists; the archive is served by the
27// REST subset and downloaded with a read token for that repository only.
28// - anything else: github.com. The ref is resolved with `git ls-remote` (tag,
29// then branch, the peeled commit of an annotated tag); a full commit id is
30// returned as it is, without checking. The archive is codeload's, with no
31// token.
32//
33// The commit a ref resolves to is kept for the run attempt: every job and every
34// call of the attempt gets the same one, the first that was stored. A
35// reference that cannot be resolved is not an error of the call: its answer
36// carries GitHub's message.
37func (c *CI) ResolveActions(ctx context.Context, secret, jobID string, uses []string) ([]ports.ResolvedAction, error) {
38 out, err := c.resolveActions(ctx, secret, jobID, uses)
39 if err != nil && ctx.Err() == nil && errors.Is(err, context.DeadlineExceeded) {
40 // Our own deadline, not the caller's: github.com (or the disk) was too
41 // slow. The runner may retry, or fail the job.
42 return nil, fmt.Errorf("%w: resolving the actions took more than %s", domain.ErrUnavailable, c.cfg.ResolveTimeout)
43 }
44 return out, err
45}
46
47func (c *CI) resolveActions(ctx context.Context, secret, jobID string, uses []string) ([]ports.ResolvedAction, error) {
48 ctx, cancel := context.WithTimeout(ctx, c.cfg.ResolveTimeout)
49 defer cancel()
50 j, err := c.jobFor(ctx, secret, jobID)
51 if err != nil {
52 return nil, err
53 }
54 if j.Ended() {
55 return nil, endedError(j)
56 }
57 if len(uses) > maxActionRefs {
58 return nil, fmt.Errorf("%w: at most %d actions per call", domain.ErrInvalid, maxActionRefs)
59 }
60 parsed := make([]actionRef, len(uses))
61 for i, u := range uses {
62 if strings.HasPrefix(u, "./") || strings.HasPrefix(u, ".\\") {
63 return nil, fmt.Errorf("%w: %q is a local action: the runner reads it from the workspace", domain.ErrInvalid, u)
64 }
65 if strings.HasPrefix(u, "docker://") {
66 return nil, fmt.Errorf("%w: %q is a Docker image: the runner pulls it", domain.ErrInvalid, u)
67 }
68 parsed[i] = parseActionRef(u)
69 }
70
71 src, err := c.cfg.Store.GetRunSource(ctx, j.RunID)
72 if err != nil {
73 return nil, err
74 }
75 by := c.onBehalfOf(src)
76
77 // Resolve what the attempt has not resolved yet, outside the lock of the
78 // run: a ls-remote takes seconds and must not hold up the job's calls.
79 found := map[string]domain.CIActionResolution{}
80 fails := map[string]string{}
81 remote := c.newRemoteRefs(j.RunAttempt)
82 for _, p := range parsed {
83 if p.err != "" {
84 continue
85 }
86 if _, done := found[p.key()]; done {
87 continue
88 }
89 if _, bad := fails[p.key()]; bad {
90 continue
91 }
92 if res, err := c.cfg.Store.GetActionResolution(ctx, j.RunID, j.RunAttempt, p.key()); err == nil {
93 found[p.key()] = res
94 continue
95 } else if !errors.Is(err, domain.ErrNotFound) {
96 return nil, err
97 }
98 res, msg, err := c.resolveAction(ctx, j, p, remote, by)
99 if err != nil {
100 return nil, err
101 }
102 if msg != "" {
103 fails[p.key()] = msg
104 continue
105 }
106 found[p.key()] = res
107 }
108
109 out := make([]ports.ResolvedAction, len(parsed))
110 err = c.lockedJob(ctx, j, func(cur *domain.CIJob) error {
111 if cur.Ended() {
112 return endedError(*cur)
113 }
114 rs, err := c.loadRun(ctx, cur.RunID, cur.RunAttempt)
115 if err != nil {
116 return err
117 }
118 own := rs.find(cur.ID)
119 if own == nil {
120 return domain.ErrNotFound
121 }
122 for i, p := range parsed {
123 out[i] = ports.ResolvedAction{Ref: p.uses}
124 if p.err != "" {
125 out[i].Error = p.err
126 continue
127 }
128 if msg, bad := fails[p.key()]; bad {
129 out[i].Error = msg
130 continue
131 }
132 // The first resolution stored wins, ours or a concurrent job's.
133 res, ok := found[p.key()]
134 if cached, err := c.cfg.Store.GetActionResolution(ctx, cur.RunID, cur.RunAttempt, p.key()); err == nil {
135 res, ok = cached, true
136 } else if !errors.Is(err, domain.ErrNotFound) {
137 return err
138 } else if ok {
139 rs.resolutions = append(rs.resolutions, res)
140 found[p.key()] = res // two refs of the call with the same key
141 }
142 if !ok {
143 continue
144 }
145 ra, err := c.answer(ctx, rs, own, p, res, by)
146 if err != nil {
147 return err
148 }
149 out[i] = ra
150 }
151 if err := rs.flush(ctx); err != nil {
152 // The tokens were not stored: forget their secrets.
153 c.mu.Lock()
154 delete(c.archiveTokens, jobRef(*cur))
155 c.mu.Unlock()
156 return err
157 }
158 return nil
159 })
160 if err != nil {
161 return nil, err
162 }
163 return out, nil
164}
165
166// maxActionRefs is the most references of one ResolveActions call.
167const maxActionRefs = 100
168
169// answer builds the answer for a resolved reference. The archive token of an
170// ermite action is bound to the commit of the answer: the one the job already
171// holds for it, else a new one, written with the next flush. Whether the pusher
172// may read the repository is asked again at each answer, not only when the
173// commit was resolved: a token revoked since, or a repository made private,
174// stops the answers.
175func (c *CI) answer(ctx context.Context, rs *runState, j *domain.CIJob, p actionRef, res domain.CIActionResolution, by pusherAccess) (ports.ResolvedAction, error) {
176 ra := ports.ResolvedAction{
177 Ref: p.uses, NameWithOwner: res.NameWithOwner, Path: p.path, ResolvedSHA: res.ResolvedSHA, Source: res.Source,
178 }
179 notFound := ports.ResolvedAction{Ref: p.uses, Error: errUnresolvedRepo(p)}
180
181 // A repository of this server that the pusher cannot read is never served
182 // from anywhere else: not from github.com under the same name, whatever
183 // source an earlier resolution of the attempt stored. It answers as a
184 // repository that does not exist. (When it does not exist on this server, a
185 // stored ermite resolution has nothing to serve either.)
186 repo, onErmite, err := c.ermiteRepoOf(ctx, p)
187 if err != nil {
188 return ports.ResolvedAction{}, err
189 }
190 if onErmite {
191 if ok, err := c.pusherCanRead(ctx, by, rs.run.Repo, repo); err != nil {
192 return ports.ResolvedAction{}, err
193 } else if !ok {
194 return notFound, nil
195 }
196 }
197 switch res.Source {
198 case domain.CIActionGitHub:
199 // The case the workflow wrote: the stored resolution is shared by
200 // spellings that differ only by case, and github.com ignores it.
201 ra.NameWithOwner = p.owner + "/" + p.repo
202 ra.TarballURL = "https://codeload.github.com/" + ra.NameWithOwner + "/tar.gz/" + res.ResolvedSHA
203 case domain.CIActionErmite:
204 if !onErmite {
205 return notFound, nil
206 }
207 ra.TarballURL = c.cfg.PublicURL + apiPath + "/repos/" + res.NameWithOwner + "/tarball/" + res.ResolvedSHA
208 ra.Token = c.archiveToken(rs, j, repo.Name, res.ResolvedSHA)
209 }
210 return ra, nil
211}
212
213// archiveToken returns the secret of the token of the job for the archive of
214// the commit sha of repo, issuing it when the job has none. The token downloads
215// that archive and nothing else (domain.RepoTokenPrincipal).
216func (c *CI) archiveToken(rs *runState, j *domain.CIJob, repo domain.RepoName, sha string) string {
217 ref, key := jobRef(*j), repo.String()+"@"+sha
218 c.mu.Lock()
219 tok, ok := c.archiveTokens[ref][key]
220 c.mu.Unlock()
221 if ok {
222 return tok
223 }
224 tok = rs.issueRepoTokenFor(j, repo, sha)
225 c.mu.Lock()
226 if c.archiveTokens[ref] == nil {
227 c.archiveTokens[ref] = map[string]string{}
228 }
229 c.archiveTokens[ref][key] = tok
230 c.mu.Unlock()
231 return tok
232}
233
234// ermiteRepoOf finds the repository of this server that a reference names: the
235// owner is ERMITE_OWNER (without regard to case) and the repository exists. ok
236// is false for any other reference, which github.com serves. It is the one
237// place that decides the source of an action, for a first resolution and for the
238// answer from a stored one.
239func (c *CI) ermiteRepoOf(ctx context.Context, a actionRef) (repo domain.Repo, ok bool, err error) {
240 if !domain.OwnerMatches(c.cfg.Owner, a.owner) {
241 return domain.Repo{}, false, nil
242 }
243 name, perr := domain.ParseRepoName(strings.ToLower(a.repo))
244 if perr != nil {
245 return domain.Repo{}, false, nil // not a name ermite can hold
246 }
247 repo, err = c.cfg.Meta.Get(ctx, name)
248 switch {
249 case errors.Is(err, domain.ErrNotFound):
250 return domain.Repo{}, false, nil
251 case err != nil:
252 return domain.Repo{}, false, err
253 }
254 return repo, true, nil
255}
256
257// pusherAccess is who the actions of a run are resolved on behalf of: the
258// pusher recorded with the run.
259type pusherAccess struct{ pusher domain.CIPusher }
260
261func (c *CI) onBehalfOf(src domain.CIRunSource) pusherAccess { return pusherAccess{src.Pusher} }
262
263// pusherCanRead tells whether the pusher of a run may read repo, with the rule
264// of the forge (canRead) applied to the principal the pusher is now: the root
265// token and the owner's session read everything; a stored token is looked up
266// again, so a token revoked or expired since reads nothing, and its scopes and
267// repository list apply; a run that records no pusher reads what an anonymous
268// caller reads. The repository of the run itself needs no check: the job reads
269// it with its own token.
270func (c *CI) pusherCanRead(ctx context.Context, by pusherAccess, runRepo domain.RepoName, repo domain.Repo) (bool, error) {
271 if repo.Name == runRepo {
272 return true, nil
273 }
274 p, _, err := principalOfPusher(ctx, c.cfg.Tokens, by.pusher, c.now())
275 if err != nil {
276 return false, err
277 }
278 return canRead(p, repo), nil
279}
280
281// principalOfPusher re-resolves a recorded actor without substituting a more
282// privileged identity when its token no longer exists or has expired.
283func principalOfPusher(ctx context.Context, tokens ports.TokenStore, by domain.CIPusher, now time.Time) (domain.Principal, bool, error) {
284 switch by.Kind {
285 case domain.CIPusherRoot:
286 return domain.Root(), true, nil
287 case domain.CIPusherOwner:
288 return domain.Owner(), true, nil
289 case domain.CIPusherToken:
290 if tokens == nil {
291 return domain.Anonymous(), false, nil
292 }
293 tok, err := tokens.Get(ctx, by.TokenID)
294 if errors.Is(err, domain.ErrNotFound) {
295 return domain.Anonymous(), false, nil
296 }
297 if err != nil {
298 return domain.Anonymous(), false, err
299 }
300 if tok.Expired(now) {
301 return domain.Anonymous(), false, nil
302 }
303 return domain.TokenPrincipal(tok), true, nil
304 default:
305 return domain.Anonymous(), false, nil
306 }
307}
308
309// actionRef is a parsed "uses": owner/repo[/path]@ref.
310type actionRef struct {
311 uses, owner, repo, path, ref string
312 // err is GitHub's message for a reference of the wrong shape.
313 err string
314}
315
316// key identifies the resolution: GitHub names are case-insensitive, refs are
317// not.
318func (a actionRef) key() string { return strings.ToLower(a.owner+"/"+a.repo) + "@" + a.ref }
319
320// parseActionRef splits a "uses" of a repository action. Only the shape
321// owner/repo[/path]@ref is valid; GitHub splits the path on '/' and '\' and
322// ignores empty segments (WorkflowTemplateConverter.cs), so do we.
323func parseActionRef(uses string) actionRef {
324 a := actionRef{uses: uses}
325 left, ref, ok := strings.Cut(uses, "@")
326 segs := strings.FieldsFunc(left, func(r rune) bool { return r == '/' || r == '\\' })
327 if !ok || strings.Contains(ref, "@") || len(segs) < 2 || ref == "" {
328 a.err = errActionFormat(uses)
329 return a
330 }
331 for _, s := range segs[2:] {
332 if s == "." || s == ".." {
333 a.err = errActionFormat(uses)
334 return a
335 }
336 }
337 if !validActionName(segs[0]) || !validActionName(segs[1]) {
338 a.err = errActionFormat(uses)
339 return a
340 }
341 a.owner, a.repo, a.ref = segs[0], segs[1], ref
342 a.path = strings.Join(segs[2:], "/")
343 return a
344}
345
346// validActionName reports whether s can be an owner or a repository name on
347// GitHub: letters, digits, '.', '_' and '-', 1 to 100 characters, not made of
348// dots only ("." and ".." are path elements). Every source takes the same names, so that nothing but a name reaches a
349// URL, a path or a command.
350func validActionName(s string) bool {
351 if s == "" || len(s) > 100 || strings.Trim(s, ".") == "" {
352 return false
353 }
354 for _, r := range s {
355 switch {
356 case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '.', r == '_', r == '-':
357 default:
358 return false
359 }
360 }
361 return true
362}
363
364// --- GitHub's texts ---
365//
366// All the messages a reference can fail with are here. The first is the
367// runner's own parser text; the two others are the ones GitHub gives for an
368// unknown version and an unknown repository, to be confirmed by recording
369// against github.com (ci2-execution plan, phase A).
370
371func errActionFormat(uses string) string {
372 return "Expected format {org}/{repo}[/path]@ref. Actual '" + uses + "'"
373}
374
375func errUnresolvedRef(a actionRef) string {
376 return "Unable to resolve action `" + a.owner + "/" + a.repo + "@" + a.ref + "`, unable to find version `" + a.ref + "`"
377}
378
379func errUnresolvedRepo(a actionRef) string {
380 return "Unable to resolve action `" + a.owner + "/" + a.repo + "@" + a.ref + "`, repository not found"
381}
382
383// resolveAction resolves one reference. msg is the message of a reference that
384// does not resolve; err is a failure of the call.
385func (c *CI) resolveAction(ctx context.Context, j domain.CIJob, a actionRef, remote *remoteRefs, by pusherAccess) (res domain.CIActionResolution, msg string, err error) {
386 res = domain.CIActionResolution{RunID: j.RunID, RunAttempt: j.RunAttempt, Key: a.key()}
387 repo, onErmite, err := c.ermiteRepoOf(ctx, a)
388 if err != nil {
389 return res, "", err
390 }
391 if onErmite {
392 // The repository exists here, so github.com never answers for it. A
393 // repository the pusher may not read answers as one that does not exist,
394 // before any ref is looked at, so that the texts leak nothing about it.
395 if ok, err := c.pusherCanRead(ctx, by, j.Repo, repo); err != nil {
396 return res, "", err
397 } else if !ok {
398 return res, errUnresolvedRepo(a), nil
399 }
400 sha, found, err := c.resolveOnErmite(ctx, repo.Name, a.ref)
401 if err != nil {
402 return res, "", err
403 }
404 if !found {
405 return res, errUnresolvedRef(a), nil
406 }
407 res.NameWithOwner, res.ResolvedSHA, res.Source = c.cfg.Owner+"/"+repo.Name.String(), sha, domain.CIActionErmite
408 return res, "", nil
409 }
410 res.NameWithOwner, res.Source = a.owner+"/"+a.repo, domain.CIActionGitHub
411 if low := strings.ToLower(a.ref); isHex40(low) {
412 res.ResolvedSHA = low // a commit id is not checked: a missing one fails the download
413 return res, "", nil
414 }
415 refs, err := remote.list(ctx, a.owner, a.repo)
416 switch {
417 case errors.Is(err, domain.ErrNotFound):
418 return res, errUnresolvedRepo(a), nil
419 case errors.Is(err, domain.ErrUnavailable) && ctx.Err() == nil:
420 // The job learns which action and why, not only "unavailable".
421 return res, "", fmt.Errorf("%w: resolving `%s`: github.com did not answer (%v)", domain.ErrUnavailable, a.uses, err)
422 case err != nil:
423 return res, "", err
424 }
425 sha := pickRemoteRef(refs, a.ref)
426 if sha == "" {
427 return res, errUnresolvedRef(a), nil
428 }
429 res.ResolvedSHA = sha
430 return res, "", nil
431}
432
433// pickRemoteRef finds the commit of a tag, else of a branch, in the refs of a
434// remote. An annotated tag gives its peeled commit.
435func pickRemoteRef(refs []ports.RemoteRef, name string) string {
436 for _, prefix := range []string{"refs/tags/", "refs/heads/"} {
437 for _, r := range refs {
438 if r.Name != prefix+name {
439 continue
440 }
441 if r.Peeled != "" {
442 return r.Peeled
443 }
444 return r.ID
445 }
446 }
447 return ""
448}
449
450// resolveOnErmite resolves a ref on a repository of this server: a tag, else
451// a branch, else a full commit id that exists.
452func (c *CI) resolveOnErmite(ctx context.Context, repo domain.RepoName, ref string) (sha string, found bool, err error) {
453 refs, err := c.cfg.Reader.ListRefs(ctx, repo)
454 if err != nil && !errors.Is(err, domain.ErrNotFound) {
455 return "", false, err
456 }
457 for _, kind := range []domain.RefKind{domain.RefKindTag, domain.RefKindBranch} {
458 i := slices.IndexFunc(refs, func(r domain.Ref) bool {
459 return r.Kind == kind && r.ShortName == ref && !r.CommitID.IsZero()
460 })
461 if i >= 0 {
462 return refs[i].CommitID.String(), true, nil
463 }
464 }
465 if id, perr := domain.ParseObjectID(ref); perr == nil {
466 ok, err := c.cfg.Git.HasCommit(ctx, repo, id)
467 if err != nil && !errors.Is(err, domain.ErrNotFound) {
468 return "", false, err
469 }
470 if ok {
471 return id.String(), true, nil
472 }
473 }
474 return "", false, nil
475}
476
477// remoteRefs lists the refs of a remote once per repository within a call,
478// through the cache of the server (see refsCache).
479type remoteRefs struct {
480 remote ports.GitRemote
481 slots chan struct{}
482 cache *refsCache
483 now func() time.Time
484 seen map[string]remoteList
485 // refresh asks github.com even when the cache is fresh: a re-run resolves
486 // again (a moved tag is often why it is re-run). The cache still stands
487 // in when github.com does not answer.
488 refresh bool
489}
490
491type remoteList struct {
492 refs []ports.RemoteRef
493 err error
494}
495
496// newRemoteRefs lists for the jobs of an attempt: a re-run (attempt > 1)
497// refreshes.
498func (c *CI) newRemoteRefs(attempt int32) *remoteRefs {
499 return &remoteRefs{remote: c.cfg.Remote, slots: c.remoteSlots, cache: c.remoteCache, now: c.now, seen: map[string]remoteList{}, refresh: attempt > 1}
500}
501
502func (r *remoteRefs) list(ctx context.Context, owner, repo string) ([]ports.RemoteRef, error) {
503 if r.remote == nil {
504 return nil, fmt.Errorf("%w: this server cannot reach github.com", domain.ErrUnavailable)
505 }
506 key := strings.ToLower(owner + "/" + repo)
507 if l, ok := r.seen[key]; ok {
508 return l.refs, l.err
509 }
510 if refs, fresh := r.cache.get(key, r.now()); fresh && !r.refresh {
511 r.seen[key] = remoteList{refs, nil}
512 return refs, nil
513 }
514 // At most a few ls-remote at once on the whole server, whatever the number
515 // of jobs resolving: a process each, and github.com is slow to answer.
516 select {
517 case r.slots <- struct{}{}:
518 defer func() { <-r.slots }()
519 case <-ctx.Done():
520 return nil, ctx.Err()
521 }
522 // Another job may have listed it while this one waited for a slot.
523 if refs, fresh := r.cache.get(key, r.now()); fresh && !r.refresh {
524 r.seen[key] = remoteList{refs, nil}
525 return refs, nil
526 }
527 refs, err := r.remote.ListRefs(ctx, owner, repo)
528 if ctx.Err() != nil {
529 return nil, ctx.Err()
530 }
531 switch {
532 case err == nil:
533 r.cache.put(key, refs, r.now())
534 case errors.Is(err, domain.ErrUnavailable):
535 // github.com did not answer: the last list it gave is better than a
536 // failed job, for a while (refsStale).
537 if old, ok := r.cache.stale(key, r.now()); ok {
538 log := zerowrap.FromCtx(ctx)
539 log.Warn().Err(err).Str("repo", key).Msg("github.com did not answer: using the refs listed earlier")
540 refs, err = old, nil
541 }
542 }
543 r.seen[key] = remoteList{refs, err}
544 return refs, err
545}
546
547const (
548 // refsFresh is how long a list of the refs of a repository of github.com
549 // is used without asking again: the jobs of the next minutes resolve
550 // actions/checkout@v4 without a ls-remote. A tag or branch moved meanwhile
551 // is seen after at most that long.
552 refsFresh = 5 * time.Minute
553 // refsStale is how long a list is kept for when github.com does not answer.
554 refsStale = 24 * time.Hour
555 // refsMax bounds the repositories in the cache.
556 refsMax = 256
557)
558
559// refsCache keeps the refs of the repositories of github.com the actions come
560// from, for the whole server. Safe for concurrent use.
561type refsCache struct {
562 mu sync.Mutex
563 m map[string]cachedRefs
564}
565
566type cachedRefs struct {
567 refs []ports.RemoteRef
568 at time.Time
569}
570
571func newRefsCache() *refsCache { return &refsCache{m: map[string]cachedRefs{}} }
572
573// get returns the list of key and whether it is fresh.
574func (c *refsCache) get(key string, now time.Time) ([]ports.RemoteRef, bool) {
575 c.mu.Lock()
576 defer c.mu.Unlock()
577 e, ok := c.m[key]
578 return e.refs, ok && now.Sub(e.at) < refsFresh
579}
580
581// stale returns the list of key when it is not older than refsStale.
582func (c *refsCache) stale(key string, now time.Time) ([]ports.RemoteRef, bool) {
583 c.mu.Lock()
584 defer c.mu.Unlock()
585 e, ok := c.m[key]
586 return e.refs, ok && now.Sub(e.at) < refsStale
587}
588
589func (c *refsCache) put(key string, refs []ports.RemoteRef, now time.Time) {
590 c.mu.Lock()
591 defer c.mu.Unlock()
592 if _, ok := c.m[key]; !ok && len(c.m) >= refsMax {
593 var oldest string
594 for k, e := range c.m {
595 if oldest == "" || e.at.Before(c.m[oldest].at) {
596 oldest = k
597 }
598 }
599 delete(c.m, oldest)
600 }
601 c.m[key] = cachedRefs{refs: refs, at: now}
602}