1// SPDX-License-Identifier: MIT
2
3package executor
4
5import (
6 "context"
7 "crypto/sha256"
8 "encoding/hex"
9 "errors"
10 "fmt"
11 "io"
12 "io/fs"
13 "os"
14 "path"
15 "regexp"
16 "sort"
17 "strings"
18 "syscall"
19 "time"
20
21 "bnema.dev/ermite-actions/expr"
22)
23
24// hashFiles is the function of the same name (Runner.Worker/Expressions/
25// HashFilesFunction.cs). The official runner starts node on its hashFiles
26// script (Misc/expressionFunc/hashFiles, which uses @actions/glob); this is a
27// port of that script, evaluated by the executor, which runs in the sandbox
28// next to the workspace.
29//
30// For each file that matches the patterns, in the order of @actions/glob's
31// traversal (a depth first walk with the names of a directory sorted), the
32// SHA-256 of the file is written to a second SHA-256; the result is its
33// hexadecimal digest, or "" when no file matched.
34//
35// Differences from the official function, on purpose: the files are read
36// through an os.Root on the workspace, so that a symbolic link that leaves the
37// workspace is not followed (the executor can be root, the workspace belongs
38// to the user of the steps) and its file is skipped; and only regular files
39// are hashed (a FIFO would block until the timeout). A matched symbolic link
40// whose target is missing fails the call, as it does in the script. Braces ({a,b}), extglobs (+(a|b)) and a leading
41// "!" in the middle of a pattern are not special, as in @actions/glob.
42const hashFilesTimeout = 120 * time.Second
43
44func (j *job) hashFilesFunc() expr.Func {
45 return func(ctx context.Context, args []expr.Value) (expr.Value, error) {
46 var patterns []string
47 follow := false
48 for i, a := range args {
49 p := expr.CoerceString(a)
50 if i == 0 && strings.HasPrefix(p, "--") {
51 if strings.EqualFold(p, "--follow-symbolic-links") {
52 follow = true
53 continue
54 }
55 // new ArgumentOutOfRangeException(message) takes the text as the parameter name.
56 //nolint:staticcheck,misspell // GitHub's text, typo included
57 return nil, fmt.Errorf("Specified argument was out of the range of valid values. (Parameter 'Invalid glob option %s, avaliable option: '--follow-symbolic-links'.')", p)
58 }
59 patterns = append(patterns, p)
60 }
61 ws := j.state.Layout().Workspace
62 ctx, cancel := context.WithTimeout(ctx, hashFilesTimeout)
63 defer cancel()
64 sum, err := hashFilesIn(ctx, ws, j.state.Layout().Home, patterns, follow)
65 desc := strings.ReplaceAll(strings.Join(patterns, ", "), "'", "''")
66 switch {
67 case errors.Is(err, context.DeadlineExceeded):
68 //nolint:staticcheck // GitHub's text, with its final period
69 return nil, fmt.Errorf("hashFiles('%s') couldn't finish within %d seconds.", desc, int(hashFilesTimeout/time.Second))
70 case err != nil:
71 j.diagDebug(err, "hashFiles failed")
72 return nil, fmt.Errorf("hashFiles('%s') failed. Fail to hash files under directory '%s'", desc, ws)
73 }
74 return expr.String(sum), nil
75 }
76}
77
78// ---------------------------------------------------------------- patterns
79
80// seg is a segment of a pattern: its literal value (getLiteral, "" when it has
81// a wildcard) and its minimatch form.
82type seg struct {
83 raw string // the segment as written, for the "**" test
84 lit string
85 mm mmSeg
86 root bool
87}
88
89// mmSeg is a parsed segment: the globstar, a literal, or a regular expression.
90type mmSeg struct {
91 globstar bool
92 lit string
93 re *regexp.Regexp
94}
95
96type globPattern struct {
97 negate bool
98 segs []seg
99 trailingSep bool
100 implicit bool
101 searchPath string
102}
103
104const (
105 kindNone = 0
106 kindDir = 1
107 kindFile = 2
108 kindAll = 3
109)
110
111func splitSlash(s string) []string {
112 return slashRunRe.Split(s, -1)
113}
114
115var errPattern = errors.New("invalid pattern")
116
117// newGlobPattern is Pattern's constructor for a pattern written by the user,
118// relative to cwd (the workspace, as segments).
119func newGlobPattern(text, cwd, home string) (*globPattern, error) {
120 p := &globPattern{}
121 text = strings.TrimSpace(text)
122 for strings.HasPrefix(text, "!") {
123 p.negate = !p.negate
124 text = strings.TrimSpace(text[1:])
125 }
126 if text == "" {
127 return nil, fmt.Errorf("%w: pattern cannot be empty", errPattern)
128 }
129 norm := doubleSlashRe.ReplaceAllString(text, "/")
130 p.trailingSep = strings.HasSuffix(norm, "/")
131 trimmed := norm
132 if trimmed != "/" {
133 trimmed = strings.TrimSuffix(trimmed, "/")
134 }
135 rooted := strings.HasPrefix(trimmed, "/")
136 var rawSegs []string
137 if rooted {
138 rawSegs = append([]string{"/"}, splitNonEmpty(trimmed)...)
139 } else {
140 rawSegs = strings.Split(trimmed, "/")
141 }
142 for i, s := range rawSegs {
143 l := getLiteral(s)
144 if (l == "." && i != 0) || l == ".." {
145 return nil, fmt.Errorf("%w: relative pathing '.' and '..' is not allowed: %s", errPattern, text)
146 }
147 }
148 var prefix []seg
149 rest := rawSegs
150 switch {
151 case trimmed == "." || strings.HasPrefix(trimmed, "./"):
152 prefix, rest = literalSegs(cwd), rawSegs[1:]
153 case trimmed == "~" || strings.HasPrefix(trimmed, "~/"):
154 prefix, rest = literalSegs(home), rawSegs[1:]
155 case rooted:
156 rest = rawSegs[1:]
157 prefix = []seg{{raw: "/", lit: "/", mm: mmSeg{lit: ""}, root: true}}
158 default:
159 prefix = literalSegs(cwd)
160 }
161 p.segs = prefix
162 for _, r := range rest {
163 p.segs = append(p.segs, seg{raw: r, lit: getLiteral(r), mm: parseMM(r)})
164 }
165 p.computeSearchPath()
166 return p, nil
167}
168
169func splitNonEmpty(s string) []string {
170 var out []string
171 for _, x := range strings.Split(s, "/") {
172 if x != "" {
173 out = append(out, x)
174 }
175 }
176 return out
177}
178
179// literalSegs is the absolute path abs as literal segments.
180func literalSegs(abs string) []seg {
181 out := []seg{{raw: "/", lit: "/", mm: mmSeg{lit: ""}, root: true}}
182 for _, s := range splitNonEmpty(abs) {
183 out = append(out, seg{raw: "", lit: s, mm: mmSeg{lit: s}})
184 }
185 return out
186}
187
188func (p *globPattern) computeSearchPath() {
189 var parts []string
190 for _, s := range p.segs {
191 if s.lit == "" {
192 break
193 }
194 parts = append(parts, s.lit)
195 }
196 p.searchPath = joinSegs(parts)
197}
198
199func joinSegs(parts []string) string {
200 if len(parts) == 0 {
201 return ""
202 }
203 out := parts[0]
204 for _, s := range parts[1:] {
205 if !strings.HasSuffix(out, "/") {
206 out += "/"
207 }
208 out += s
209 }
210 return out
211}
212
213func (p *globPattern) withDescendants() *globPattern {
214 q := *p
215 q.segs = append(append([]seg(nil), p.segs...), seg{raw: "**", lit: "", mm: mmSeg{globstar: true}})
216 q.trailingSep = false
217 q.implicit = true
218 q.computeSearchPath()
219 return &q
220}
221
222// getLiteral is Pattern.getLiteral: the segment with its escapes removed, or ""
223// when it has a wildcard.
224func getLiteral(s string) string {
225 r := []rune(s)
226 var lit []rune
227 for i := 0; i < len(r); i++ {
228 c := r[i]
229 switch {
230 case c == '\\' && i+1 < len(r):
231 i++
232 lit = append(lit, r[i])
233 case c == '*' || c == '?':
234 return ""
235 case c == '[' && i+1 < len(r):
236 var set []rune
237 closed := -1
238 for i2 := i + 1; i2 < len(r); i2++ {
239 c2 := r[i2]
240 if c2 == '\\' && i2+1 < len(r) {
241 i2++
242 set = append(set, r[i2])
243 continue
244 }
245 if c2 == ']' {
246 closed = i2
247 break
248 }
249 set = append(set, c2)
250 }
251 if closed >= 0 {
252 if len(set) > 1 {
253 return ""
254 }
255 if len(set) == 1 {
256 lit = append(lit, set...)
257 i = closed
258 continue
259 }
260 }
261 lit = append(lit, c)
262 default:
263 lit = append(lit, c)
264 }
265 }
266 return string(lit)
267}
268
269var (
270 slashRunRe = regexp.MustCompile(`/+`)
271 doubleSlashRe = regexp.MustCompile(`//+`)
272 neverMatchesRe = regexp.MustCompile(`$.`)
273)
274
275const reSpecials = "().*{}+?[]^$\\!"
276
277// parseMM is minimatch's parse for one segment with the options of
278// @actions/glob (dot, nobrace, noext, nonegate, nocomment).
279func parseMM(s string) mmSeg {
280 if s == "**" {
281 return mmSeg{globstar: true}
282 }
283 if s == "" {
284 return mmSeg{lit: ""}
285 }
286 re, magic := translateMM([]rune(s))
287 if !magic {
288 return mmSeg{lit: globUnescape(s)}
289 }
290 rx, err := regexp.Compile("^" + re + "$")
291 if err != nil {
292 return mmSeg{re: neverMatchesRe}
293 }
294 return mmSeg{re: rx}
295}
296
297func globUnescape(s string) string {
298 r := []rune(s)
299 var out []rune
300 for i := 0; i < len(r); i++ {
301 if r[i] == '\\' && i+1 < len(r) {
302 i++
303 }
304 out = append(out, r[i])
305 }
306 return string(out)
307}
308
309func translateMM(p []rune) (re string, magic bool) {
310 var sb strings.Builder
311 escaping, inClass := false, false
312 classStart, reClassStart := -1, -1
313 for i := 0; i < len(p); i++ {
314 c := p[i]
315 if escaping && strings.ContainsRune(reSpecials, c) {
316 sb.WriteString(`\` + string(c))
317 escaping = false
318 continue
319 }
320 switch c {
321 case '\\':
322 escaping = true
323 case '?', '*', '+', '@', '!':
324 if inClass {
325 if c == '!' && i == classStart+1 {
326 c = '^'
327 }
328 sb.WriteRune(c)
329 continue
330 }
331 switch c {
332 case '*':
333 sb.WriteString(`[^/]*?`)
334 magic = true
335 case '?':
336 sb.WriteString(`[^/]`)
337 magic = true
338 default:
339 sb.WriteString(`\` + string(c))
340 }
341 case '(':
342 if inClass {
343 sb.WriteRune('(')
344 } else {
345 sb.WriteString(`\(`)
346 }
347 case ')':
348 sb.WriteString(`\)`)
349 case '|':
350 sb.WriteString(`\|`)
351 escaping = false
352 case '[':
353 if inClass {
354 sb.WriteString(`\[`)
355 continue
356 }
357 inClass, classStart, reClassStart = true, i, sb.Len()
358 sb.WriteRune('[')
359 case ']':
360 if i == classStart+1 || !inClass {
361 sb.WriteString(`\]`)
362 escaping = false
363 continue
364 }
365 cs := string(p[classStart+1 : i])
366 if _, err := regexp.Compile("[" + cs + "]"); err != nil {
367 sub, m := translateMM([]rune(cs))
368 cur := sb.String()
369 sb.Reset()
370 sb.WriteString(cur[:reClassStart] + `\[` + sub + `\]`)
371 magic = magic || m
372 inClass = false
373 continue
374 }
375 magic = true
376 inClass = false
377 sb.WriteRune(']')
378 default:
379 if escaping {
380 escaping = false
381 } else if strings.ContainsRune(reSpecials, c) && (c != '^' || !inClass) {
382 sb.WriteRune('\\')
383 }
384 sb.WriteRune(c)
385 }
386 }
387 if inClass {
388 cs := string(p[classStart+1:])
389 sub, m := translateMM([]rune(cs))
390 cur := sb.String()
391 sb.Reset()
392 sb.WriteString(cur[:reClassStart] + `\[` + sub)
393 magic = magic || m
394 }
395 if escaping {
396 sb.WriteString(`\\`)
397 }
398 return sb.String(), magic
399}
400
401// match is Pattern.match.
402func (p *globPattern) match(item string) int {
403 if p.segs[len(p.segs)-1].raw == "**" {
404 item = doubleSlashRe.ReplaceAllString(item, "/")
405 if !strings.HasSuffix(item, "/") && !p.implicit {
406 item += "/"
407 }
408 } else {
409 item = trimTrailingSep(item)
410 }
411 if p.matchOne(splitSlash(item), p.mmSegs(), 0, 0, false) {
412 if p.trailingSep {
413 return kindDir
414 }
415 return kindAll
416 }
417 return kindNone
418}
419
420func (p *globPattern) mmSegs() []mmSeg {
421 out := make([]mmSeg, len(p.segs))
422 for i, s := range p.segs {
423 out[i] = s.mm
424 }
425 return out
426}
427
428// partialMatch is Pattern.partialMatch.
429func (p *globPattern) partialMatch(item string) bool {
430 item = trimTrailingSep(item)
431 if path.Dir(item) == item {
432 return true // the root matches the root of every pattern
433 }
434 return p.matchOne(splitSlash(item), p.mmSegs(), 0, 0, true)
435}
436
437func trimTrailingSep(s string) string {
438 s = doubleSlashRe.ReplaceAllString(s, "/")
439 if s != "/" && strings.HasSuffix(s, "/") {
440 return s[:len(s)-1]
441 }
442 return s
443}
444
445func isDotName(s string) bool { return s == "." || s == ".." }
446
447// matchOne is Minimatch.matchOne with the dot option set.
448func (p *globPattern) matchOne(file []string, pat []mmSeg, fi, pi int, partial bool) bool {
449 fl, pl := len(file), len(pat)
450 for ; fi < fl && pi < pl; fi, pi = fi+1, pi+1 {
451 ps, f := pat[pi], file[fi]
452 if ps.globstar {
453 fr, pr := fi, pi+1
454 if pr == pl {
455 for ; fi < fl; fi++ {
456 if isDotName(file[fi]) {
457 return false
458 }
459 }
460 return true
461 }
462 for fr < fl {
463 if p.matchOne(file, pat, fr, pr, partial) {
464 return true
465 }
466 if isDotName(file[fr]) {
467 break
468 }
469 fr++
470 }
471 if partial && fr == fl {
472 return true
473 }
474 return false
475 }
476 var hit bool
477 if ps.re != nil {
478 hit = ps.re.MatchString(f)
479 } else {
480 hit = f == ps.lit
481 }
482 if !hit {
483 return false
484 }
485 }
486 switch {
487 case fi == fl && pi == pl:
488 return true
489 case fi == fl:
490 return partial
491 default:
492 return fi == fl-1 && file[fi] == ""
493 }
494}
495
496// ---------------------------------------------------------------- walk
497
498func matchAll(patterns []*globPattern, item string) int {
499 r := kindNone
500 for _, p := range patterns {
501 if p.negate {
502 r &^= p.match(item)
503 } else {
504 r |= p.match(item)
505 }
506 }
507 return r
508}
509
510// getSearchPaths is internal-pattern-helper's getSearchPaths.
511func getSearchPaths(patterns []*globPattern) []string {
512 var pos []*globPattern
513 for _, p := range patterns {
514 if !p.negate {
515 pos = append(pos, p)
516 }
517 }
518 state := map[string]string{}
519 for _, p := range pos {
520 state[p.searchPath] = "candidate"
521 }
522 var out []string
523 for _, p := range pos {
524 if state[p.searchPath] == "included" {
525 continue
526 }
527 found := false
528 key := p.searchPath
529 for parent := path.Dir(key); parent != key; parent = path.Dir(key) {
530 if state[parent] != "" {
531 found = true
532 break
533 }
534 key = parent
535 }
536 if !found {
537 out = append(out, p.searchPath)
538 state[p.searchPath] = "included"
539 }
540 }
541 return out
542}
543
544type searchItem struct {
545 path string
546 level int
547}
548
549// hashFilesIn is the hashFiles script: the hex SHA-256 of the SHA-256 digests
550// of the files under ws that match the patterns, "" when none does.
551func hashFilesIn(ctx context.Context, ws, home string, lines []string, follow bool) (string, error) {
552 cwd := path.Clean(ws)
553 var patterns []*globPattern
554 for _, line := range strings.Split(strings.Join(lines, "\n"), "\n") {
555 line = strings.TrimSpace(line)
556 if line == "" || strings.HasPrefix(line, "#") {
557 continue
558 }
559 p, err := newGlobPattern(line, cwd, home)
560 if err != nil {
561 return "", err
562 }
563 patterns = append(patterns, p)
564 }
565 all := make([]*globPattern, 0, 2*len(patterns))
566 for _, p := range patterns {
567 all = append(all, p)
568 if p.trailingSep || p.segs[len(p.segs)-1].raw != "**" {
569 all = append(all, p.withDescendants())
570 }
571 }
572 root, err := os.OpenRoot(cwd)
573 if err != nil {
574 if errors.Is(err, fs.ErrNotExist) {
575 return "", nil
576 }
577 return "", err
578 }
579 defer func() { _ = root.Close() }()
580 rel := func(abs string) (string, bool) {
581 if abs == cwd {
582 return ".", true
583 }
584 if strings.HasPrefix(abs, cwd+"/") {
585 return abs[len(cwd)+1:], true
586 }
587 return "", false
588 }
589
590 var stack []searchItem
591 for _, sp := range getSearchPaths(all) {
592 if _, ok := rel(sp); !ok {
593 if !strings.HasPrefix(cwd+"/", strings.TrimSuffix(sp, "/")+"/") {
594 continue // neither under the workspace nor above it
595 }
596 sp = cwd
597 }
598 r, _ := rel(sp)
599 if _, err := root.Lstat(r); err != nil {
600 if errors.Is(err, fs.ErrNotExist) {
601 continue
602 }
603 return "", err
604 }
605 stack = append([]searchItem{{sp, 1}}, stack...)
606 }
607
608 total := sha256.New()
609 hashed := 0
610 var chain []fs.FileInfo
611 for len(stack) > 0 {
612 if err := ctx.Err(); err != nil {
613 return "", err
614 }
615 item := stack[len(stack)-1]
616 stack = stack[:len(stack)-1]
617 match := matchAll(all, item.path)
618 partial := match != kindNone
619 if !partial {
620 for _, p := range all {
621 if !p.negate && p.partialMatch(item.path) {
622 partial = true
623 break
624 }
625 }
626 }
627 if !partial {
628 continue
629 }
630 r, under := rel(item.path)
631 if !under {
632 // Above the workspace (an absolute pattern): only the way down matters.
633 r = "."
634 if item.path != "/" && !strings.HasPrefix(cwd+"/", item.path+"/") {
635 continue
636 }
637 }
638 var st fs.FileInfo
639 if follow {
640 st, err = root.Stat(r)
641 } else {
642 st, err = root.Lstat(r)
643 }
644 if err != nil {
645 if errors.Is(err, fs.ErrNotExist) || follow {
646 continue // vanished, or a broken or escaping link
647 }
648 return "", err
649 }
650 if st.IsDir() && follow {
651 for len(chain) >= item.level {
652 chain = chain[:len(chain)-1]
653 }
654 cycle := false
655 for _, c := range chain {
656 if os.SameFile(c, st) {
657 cycle = true
658 }
659 }
660 if cycle {
661 continue
662 }
663 chain = append(chain, st)
664 }
665 if st.IsDir() {
666 if match&kindDir == 0 && !partial {
667 continue
668 }
669 names, err := readDirNames(root, r)
670 if err != nil {
671 return "", err
672 }
673 base := item.path
674 for i := len(names) - 1; i >= 0; i-- {
675 stack = append(stack, searchItem{path.Join(base, names[i]), item.level + 1})
676 }
677 continue // directories are skipped by hashFiles
678 }
679 if match&kindFile == 0 || !under {
680 continue
681 }
682 if !st.Mode().IsRegular() && !follow && st.Mode()&fs.ModeSymlink == 0 {
683 continue
684 }
685 sum, ok, err := hashOne(ctx, root, r)
686 if err != nil {
687 return "", err
688 }
689 if !ok {
690 if st.Mode()&fs.ModeSymlink != 0 {
691 if _, serr := root.Stat(r); errors.Is(serr, fs.ErrNotExist) {
692 // The script stats the link's target and fails when it is
693 // missing (fs.statSync throws ENOENT): so does the function.
694 return "", fmt.Errorf("broken symbolic link %s: %w", item.path, serr)
695 }
696 }
697 continue
698 }
699 total.Write(sum)
700 hashed++
701 }
702 if hashed == 0 {
703 return "", nil
704 }
705 return hex.EncodeToString(total.Sum(nil)), nil
706}
707
708func readDirNames(root *os.Root, rel string) ([]string, error) {
709 d, err := root.Open(rel)
710 if err != nil {
711 return nil, err
712 }
713 defer func() { _ = d.Close() }()
714 ents, err := d.ReadDir(-1)
715 if err != nil {
716 return nil, err
717 }
718 names := make([]string, 0, len(ents))
719 for _, e := range ents {
720 names = append(names, e.Name())
721 }
722 sort.Strings(names)
723 return names, nil
724}
725
726// hashOne is the SHA-256 digest of a regular file. ok is false for what is not
727// hashed: a directory (reached through a link), a special file (a FIFO, a
728// socket, a device: skipped, where the official script would hang on a FIFO),
729// a link that leaves the workspace or is broken.
730func hashOne(ctx context.Context, root *os.Root, rel string) (sum []byte, ok bool, err error) {
731 // O_NONBLOCK: opening a FIFO for reading blocks until a writer comes, and
732 // nothing can interrupt the open. Reached through a symbolic link (or with
733 // --follow-symbolic-links) a FIFO would hang the job until its timeout.
734 f, err := root.OpenFile(rel, os.O_RDONLY|syscall.O_NONBLOCK, 0)
735 if err != nil {
736 return nil, false, nil
737 }
738 defer func() { _ = f.Close() }()
739 st, err := f.Stat()
740 if err != nil || !st.Mode().IsRegular() {
741 return nil, false, nil
742 }
743 h := sha256.New()
744 if _, err := io.Copy(h, &ctxReader{ctx, f}); err != nil {
745 return nil, false, err
746 }
747 return h.Sum(nil), true, nil
748}
749
750type ctxReader struct {
751 ctx context.Context
752 r io.Reader
753}
754
755func (c *ctxReader) Read(p []byte) (int, error) {
756 if err := c.ctx.Err(); err != nil {
757 return 0, err
758 }
759 return c.r.Read(p)
760}