detached fc8849e

Merge pull request 'fix(executor): GitHub parity gaps found by conformance compare' (#7) from feat/ci-parity-2 into main

commit
fc8849e4f226dc0ad2a4218d757d5a7c383910b0
author
bnema · · 2026-10-13 04:22 UTC
merge of
3418cfa + cec204f · diff against the first parent
signature
SSH UnverifiedSHA256:VOCA5lYARHcbKDZ7GhTV45VmcjWnJ6ZQIBazj17LivA The key is unknown
14 files · +1610 -17
Unified
UnifiedSplit
M go.mod +1 -1
@@ -3,7 +3,7 @@ module bnema.dev/ermite-runner
33 go 1.27.1
44
55 require (
6- bnema.dev/ermite-actions v0.0.0-20261012192445-fcc31accb5a4
6+ bnema.dev/ermite-actions v0.0.0-20261013040121-5c0e25d50362
77 bnema.dev/ermite-api v0.0.0-20261012155218-f8f64413e63c
88 connectrpc.com/connect v1.21.0
99 github.com/bnema/zerowrap v1.4.1
@@ -3,7 +3,7 @@ module bnema.dev/ermite-runner
3 go 1.27.13 go 1.27.1
4 4
5 require (5 require (
6- bnema.dev/ermite-actions v0.0.0-20261012192445-fcc31accb5a46+ bnema.dev/ermite-actions v0.0.0-20261013040121-5c0e25d50362
7 bnema.dev/ermite-api v0.0.0-20261012155218-f8f64413e63c7 bnema.dev/ermite-api v0.0.0-20261012155218-f8f64413e63c
8 connectrpc.com/connect v1.21.08 connectrpc.com/connect v1.21.0
9 github.com/bnema/zerowrap v1.4.19 github.com/bnema/zerowrap v1.4.1
M go.sum +2 -2 generated file: collapsed
load diff
M internal/executor/contexts.go +47 -1
@@ -3,6 +3,7 @@
33 package executor
44
55 import (
6+ "encoding/base64"
67 "fmt"
78 "path/filepath"
89 "runtime"
@@ -241,6 +242,8 @@ func NewState(contextsJSON, matrixJSON, jobKey string, r RunnerInfo, repoFallbac
241242 s := &State{other: map[string]expr.Value{}, env: newOrdered(), steps: map[string]*stepCtx{}, status: "success"}
242243
243244 // github: token first, then what the server gave (ExecutionContext.cs:1023-1037).
245+ // The token is the one the server sent in github.token (GitHub's
246+ // system.github.token variable), empty when it sent none.
244247 gh := expr.NewDictionary()
245248 gh.Add("token", expr.String(""))
246249 if v, ok := root.Get("github"); ok {
@@ -250,6 +253,9 @@ func NewState(contextsJSON, matrixJSON, jobKey string, r RunnerInfo, repoFallbac
250253 }
251254 for _, p := range d.Pairs() {
252255 if strings.EqualFold(p.Key, "token") {
256+ if t, isStr := p.Value.(expr.String); isStr {
257+ gh = setKey(gh, "token", t)
258+ }
253259 continue
254260 }
255261 gh.Add(p.Key, p.Value)
@@ -258,6 +264,12 @@ func NewState(contextsJSON, matrixJSON, jobKey string, r RunnerInfo, repoFallbac
258264 s.serverGitHub = expr.NewDictionary()
259265 if v, ok := root.Get("github"); ok {
260266 s.serverGitHub = v.(*expr.Dictionary)
267+ // GitHub evaluates names with github.job empty (recorded:
268+ // ctx-github-names): the job key is only added to the contexts the
269+ // job runs with.
270+ if _, has := s.serverGitHub.Get("job"); has {
271+ s.serverGitHub = setKey(s.serverGitHub, "job", expr.String(""))
272+ }
261273 }
262274 if str(gh, "job") == "" && jobKey != "" {
263275 gh = setKeySorted(gh, "job", expr.String(jobKey))
@@ -457,7 +469,7 @@ func (s *State) Values(env *ordered) map[string]expr.Value {
457469 "job": s.job,
458470 "env": env.Dict(),
459471 "steps": s.stepsValue(),
460- "secrets": expr.NewDictionary(),
472+ "secrets": s.secrets(),
461473 }
462474 for k, v := range s.other {
463475 m[k] = v
@@ -465,6 +477,40 @@ func (s *State) Values(env *ordered) map[string]expr.Value {
465477 return m
466478 }
467479
480+// secrets is the secrets context. Today it holds one entry, GITHUB_TOKEN, as
481+// on GitHub (Variables.ToSecretsContext lists the secret variables but the
482+// access token; GitHub's message carries the token as the secret GITHUB_TOKEN,
483+// recorded: ctx-inputs-vars, secrets.GITHUB_TOKEN == github.token).
484+//
485+// Where the value comes from: the "token" key of the github context that the
486+// server sends in Job.contexts, the same value as github.token. It must be a
487+// token of its own, scoped to the job (the permissions "Set up job" prints) and
488+// short lived. It must never be the "ermitejob_" job token: that one
489+// authenticates the runner's calls to ermite, stays in the runner host process
490+// and must not reach the sandbox (runner.proto, "Secrets and trust"). ermite
491+// does not send a token yet (planned for CI-3, with the secrets of the
492+// repository); until it does, github.token is empty, so is secrets, and
493+// a test of secrets.GITHUB_TOKEN against the empty string is false. Without a
494+// token nothing is masked.
495+func (s *State) secrets() *expr.Dictionary {
496+ d := expr.NewDictionary()
497+ if t := s.GitHub("token"); t != "" {
498+ d.Add("GITHUB_TOKEN", expr.String(t))
499+ }
500+ return d
501+}
502+
503+// Secrets returns the values that must never leave the job: the token and the
504+// base64 of "x-access-token:<token>" (ExecutionContext.cs:1026-1028); newMasker
505+// adds their other encodings.
506+func (s *State) Secrets() []string {
507+ t := s.GitHub("token")
508+ if t == "" {
509+ return nil
510+ }
511+ return []string{t, base64.StdEncoding.EncodeToString([]byte("x-access-token:" + t))}
512+}
513+
468514 // ServerContexts returns the contexts that the server has when it evaluates
469515 // the name of a job: github as received (without the keys the runner adds),
470516 // needs, strategy, matrix, vars and inputs.
@@ -3,6 +3,7 @@
3 package executor3 package executor
4 4
5 import (5 import (
6+ "encoding/base64"
6 "fmt"7 "fmt"
7 "path/filepath"8 "path/filepath"
8 "runtime"9 "runtime"
@@ -241,6 +242,8 @@ func NewState(contextsJSON, matrixJSON, jobKey string, r RunnerInfo, repoFallbac
241 s := &State{other: map[string]expr.Value{}, env: newOrdered(), steps: map[string]*stepCtx{}, status: "success"}242 s := &State{other: map[string]expr.Value{}, env: newOrdered(), steps: map[string]*stepCtx{}, status: "success"}
242 243
243 // github: token first, then what the server gave (ExecutionContext.cs:1023-1037).244 // github: token first, then what the server gave (ExecutionContext.cs:1023-1037).
245+ // The token is the one the server sent in github.token (GitHub's
246+ // system.github.token variable), empty when it sent none.
244 gh := expr.NewDictionary()247 gh := expr.NewDictionary()
245 gh.Add("token", expr.String(""))248 gh.Add("token", expr.String(""))
246 if v, ok := root.Get("github"); ok {249 if v, ok := root.Get("github"); ok {
@@ -250,6 +253,9 @@ func NewState(contextsJSON, matrixJSON, jobKey string, r RunnerInfo, repoFallbac
250 }253 }
251 for _, p := range d.Pairs() {254 for _, p := range d.Pairs() {
252 if strings.EqualFold(p.Key, "token") {255 if strings.EqualFold(p.Key, "token") {
256+ if t, isStr := p.Value.(expr.String); isStr {
257+ gh = setKey(gh, "token", t)
258+ }
253 continue259 continue
254 }260 }
255 gh.Add(p.Key, p.Value)261 gh.Add(p.Key, p.Value)
@@ -258,6 +264,12 @@ func NewState(contextsJSON, matrixJSON, jobKey string, r RunnerInfo, repoFallbac
258 s.serverGitHub = expr.NewDictionary()264 s.serverGitHub = expr.NewDictionary()
259 if v, ok := root.Get("github"); ok {265 if v, ok := root.Get("github"); ok {
260 s.serverGitHub = v.(*expr.Dictionary)266 s.serverGitHub = v.(*expr.Dictionary)
267+ // GitHub evaluates names with github.job empty (recorded:
268+ // ctx-github-names): the job key is only added to the contexts the
269+ // job runs with.
270+ if _, has := s.serverGitHub.Get("job"); has {
271+ s.serverGitHub = setKey(s.serverGitHub, "job", expr.String(""))
272+ }
261 }273 }
262 if str(gh, "job") == "" && jobKey != "" {274 if str(gh, "job") == "" && jobKey != "" {
263 gh = setKeySorted(gh, "job", expr.String(jobKey))275 gh = setKeySorted(gh, "job", expr.String(jobKey))
@@ -457,7 +469,7 @@ func (s *State) Values(env *ordered) map[string]expr.Value {
457 "job": s.job,469 "job": s.job,
458 "env": env.Dict(),470 "env": env.Dict(),
459 "steps": s.stepsValue(),471 "steps": s.stepsValue(),
460- "secrets": expr.NewDictionary(),472+ "secrets": s.secrets(),
461 }473 }
462 for k, v := range s.other {474 for k, v := range s.other {
463 m[k] = v475 m[k] = v
@@ -465,6 +477,40 @@ func (s *State) Values(env *ordered) map[string]expr.Value {
465 return m477 return m
466 }478 }
467 479
480+// secrets is the secrets context. Today it holds one entry, GITHUB_TOKEN, as
481+// on GitHub (Variables.ToSecretsContext lists the secret variables but the
482+// access token; GitHub's message carries the token as the secret GITHUB_TOKEN,
483+// recorded: ctx-inputs-vars, secrets.GITHUB_TOKEN == github.token).
484+//
485+// Where the value comes from: the "token" key of the github context that the
486+// server sends in Job.contexts, the same value as github.token. It must be a
487+// token of its own, scoped to the job (the permissions "Set up job" prints) and
488+// short lived. It must never be the "ermitejob_" job token: that one
489+// authenticates the runner's calls to ermite, stays in the runner host process
490+// and must not reach the sandbox (runner.proto, "Secrets and trust"). ermite
491+// does not send a token yet (planned for CI-3, with the secrets of the
492+// repository); until it does, github.token is empty, so is secrets, and
493+// a test of secrets.GITHUB_TOKEN against the empty string is false. Without a
494+// token nothing is masked.
495+func (s *State) secrets() *expr.Dictionary {
496+ d := expr.NewDictionary()
497+ if t := s.GitHub("token"); t != "" {
498+ d.Add("GITHUB_TOKEN", expr.String(t))
499+ }
500+ return d
501+}
502+
503+// Secrets returns the values that must never leave the job: the token and the
504+// base64 of "x-access-token:<token>" (ExecutionContext.cs:1026-1028); newMasker
505+// adds their other encodings.
506+func (s *State) Secrets() []string {
507+ t := s.GitHub("token")
508+ if t == "" {
509+ return nil
510+ }
511+ return []string{t, base64.StdEncoding.EncodeToString([]byte("x-access-token:" + t))}
512+}
513+
468 // ServerContexts returns the contexts that the server has when it evaluates514 // ServerContexts returns the contexts that the server has when it evaluates
469 // the name of a job: github as received (without the keys the runner adds),515 // the name of a job: github as received (without the keys the runner adds),
470 // needs, strategy, matrix, vars and inputs.516 // needs, strategy, matrix, vars and inputs.
M internal/executor/exec_step.go +5 -1
@@ -22,6 +22,10 @@ func (j *job) execute(s *stepRun, stepCancel <-chan struct{}) stepResult {
2222 return resFailed
2323 }
2424 fail := func(err error) stepResult { j.errorf(num, "%s", err.Error()); return resFailed }
25+ failTemplate := func(err error) stepResult {
26+ j.errorf(num, "%s", templateNotValid(err, true))
27+ return resFailed
28+ }
2529
2630 if err := j.prepareFileCommands(st); err != nil {
2731 return fail(err)
@@ -31,7 +35,7 @@ func (j *job) execute(s *stepRun, stepCancel <-chan struct{}) stepResult {
3135
3236 in, err := j.scriptInputs(s)
3337 if err != nil {
34- return fail(err)
38+ return failTemplate(err)
3539 }
3640 plan, err := j.printHeader(s, in)
3741 if err != nil {
@@ -22,6 +22,10 @@ func (j *job) execute(s *stepRun, stepCancel <-chan struct{}) stepResult {
22 return resFailed22 return resFailed
23 }23 }
24 fail := func(err error) stepResult { j.errorf(num, "%s", err.Error()); return resFailed }24 fail := func(err error) stepResult { j.errorf(num, "%s", err.Error()); return resFailed }
25+ failTemplate := func(err error) stepResult {
26+ j.errorf(num, "%s", templateNotValid(err, true))
27+ return resFailed
28+ }
25 29
26 if err := j.prepareFileCommands(st); err != nil {30 if err := j.prepareFileCommands(st); err != nil {
27 return fail(err)31 return fail(err)
@@ -31,7 +35,7 @@ func (j *job) execute(s *stepRun, stepCancel <-chan struct{}) stepResult {
31 35
32 in, err := j.scriptInputs(s)36 in, err := j.scriptInputs(s)
33 if err != nil {37 if err != nil {
34- return fail(err)38+ return failTemplate(err)
35 }39 }
36 plan, err := j.printHeader(s, in)40 plan, err := j.printHeader(s, in)
37 if err != nil {41 if err != nil {
A internal/executor/fromjson_test.go +64 -0
@@ -0,0 +1,64 @@
1+// SPDX-License-Identifier: MIT
2+
3+package executor
4+
5+import (
6+ "strings"
7+ "testing"
8+)
9+
10+// Recorded on github.com (expr-runtime-errors): a fromJSON that fails in a
11+// step's env or run is "The template is not valid. <file> (Line: l, Col: c):
12+// <Json.NET message>"; in a step's if there is no location and the exception
13+// class comes first. The line is the line of the expression in the workflow
14+// file the executor was given (GitHub's Line 80 and the 90 of a current
15+// recording differ only because the harness injected a shorter dump step then).
16+func TestFromJSONErrorsAreTemplateErrors(t *testing.T) {
17+ needTools(t, "bash")
18+ const msg = "Unexpected character encountered while parsing value: n. Path '', line 0, position 0."
19+ yaml := `on: push
20+env:
21+ NOT_JSON: not json
22+jobs:
23+ bad-env:
24+ runs-on: ubuntu-24.04
25+ steps:
26+ - name: bad env expression
27+ env:
28+ BAD: ${{ fromJSON(env.NOT_JSON) }}
29+ run: echo SHOULD-NOT-RUN
30+ - name: after
31+ if: always()
32+ run: echo after-ran
33+ bad-if:
34+ runs-on: ubuntu-24.04
35+ steps:
36+ - name: bad if expression
37+ if: fromJSON(env.NOT_JSON)
38+ run: echo SHOULD-NOT-RUN
39+ bad-run:
40+ runs-on: ubuntu-24.04
41+ steps:
42+ - name: bad run expression
43+ run: echo ${{ fromJSON(env.NOT_JSON) }}
44+`
45+ for _, tc := range []struct{ job, step, want string }{
46+ {"bad-env", "bad env expression", "##[error]The template is not valid. .github/workflows/ci.yml (Line: 10, Col: 16): " + msg},
47+ {"bad-if", "bad if expression", "##[error]The template is not valid. Newtonsoft.Json.JsonReaderException: " + msg},
48+ {"bad-run", "bad run expression", "##[error]The template is not valid. .github/workflows/ci.yml (Line: 25, Col: 14): " + msg},
49+ } {
50+ t.Run(tc.job, func(t *testing.T) {
51+ rec, res := runLocal(t, yaml, tc.job, nil)
52+ if res.Conclusion != ConclusionFailure {
53+ t.Fatalf("%+v", res)
54+ }
55+ s := rec.step(t, tc.step)
56+ if s.Conclusion != ConclusionFailure || len(s.Log) != 1 || s.Log[0] != tc.want {
57+ t.Errorf("%s: %s\nwant %s", s.Conclusion, strings.Join(s.Log, "\n"), tc.want)
58+ }
59+ if tc.job == "bad-env" && rec.step(t, "after").Conclusion != ConclusionSuccess {
60+ t.Error("the always() step must run")
61+ }
62+ })
63+ }
64+}
@@ -0,0 +1,64 @@
1+// SPDX-License-Identifier: MIT
2+
3+package executor
4+
5+import (
6+ "strings"
7+ "testing"
8+)
9+
10+// Recorded on github.com (expr-runtime-errors): a fromJSON that fails in a
11+// step's env or run is "The template is not valid. <file> (Line: l, Col: c):
12+// <Json.NET message>"; in a step's if there is no location and the exception
13+// class comes first. The line is the line of the expression in the workflow
14+// file the executor was given (GitHub's Line 80 and the 90 of a current
15+// recording differ only because the harness injected a shorter dump step then).
16+func TestFromJSONErrorsAreTemplateErrors(t *testing.T) {
17+ needTools(t, "bash")
18+ const msg = "Unexpected character encountered while parsing value: n. Path '', line 0, position 0."
19+ yaml := `on: push
20+env:
21+ NOT_JSON: not json
22+jobs:
23+ bad-env:
24+ runs-on: ubuntu-24.04
25+ steps:
26+ - name: bad env expression
27+ env:
28+ BAD: ${{ fromJSON(env.NOT_JSON) }}
29+ run: echo SHOULD-NOT-RUN
30+ - name: after
31+ if: always()
32+ run: echo after-ran
33+ bad-if:
34+ runs-on: ubuntu-24.04
35+ steps:
36+ - name: bad if expression
37+ if: fromJSON(env.NOT_JSON)
38+ run: echo SHOULD-NOT-RUN
39+ bad-run:
40+ runs-on: ubuntu-24.04
41+ steps:
42+ - name: bad run expression
43+ run: echo ${{ fromJSON(env.NOT_JSON) }}
44+`
45+ for _, tc := range []struct{ job, step, want string }{
46+ {"bad-env", "bad env expression", "##[error]The template is not valid. .github/workflows/ci.yml (Line: 10, Col: 16): " + msg},
47+ {"bad-if", "bad if expression", "##[error]The template is not valid. Newtonsoft.Json.JsonReaderException: " + msg},
48+ {"bad-run", "bad run expression", "##[error]The template is not valid. .github/workflows/ci.yml (Line: 25, Col: 14): " + msg},
49+ } {
50+ t.Run(tc.job, func(t *testing.T) {
51+ rec, res := runLocal(t, yaml, tc.job, nil)
52+ if res.Conclusion != ConclusionFailure {
53+ t.Fatalf("%+v", res)
54+ }
55+ s := rec.step(t, tc.step)
56+ if s.Conclusion != ConclusionFailure || len(s.Log) != 1 || s.Log[0] != tc.want {
57+ t.Errorf("%s: %s\nwant %s", s.Conclusion, strings.Join(s.Log, "\n"), tc.want)
58+ }
59+ if tc.job == "bad-env" && rec.step(t, "after").Conclusion != ConclusionSuccess {
60+ t.Error("the always() step must run")
61+ }
62+ })
63+ }
64+}
A internal/executor/hashfiles.go +760 -0
@@ -0,0 +1,760 @@
1+// SPDX-License-Identifier: MIT
2+
3+package executor
4+
5+import (
6+ "context"
7+ "crypto/sha256"
8+ "encoding/hex"
9+ "errors"
10+ "fmt"
11+ "io"
12+ "io/fs"
13+ "os"
14+ "path"
15+ "regexp"
16+ "sort"
17+ "strings"
18+ "syscall"
19+ "time"
20+
21+ "bnema.dev/ermite-actions/expr"
22+)
23+
24+// hashFiles is the function of the same name (Runner.Worker/Expressions/
25+// HashFilesFunction.cs). The official runner starts node on its hashFiles
26+// script (Misc/expressionFunc/hashFiles, which uses @actions/glob); this is a
27+// port of that script, evaluated by the executor, which runs in the sandbox
28+// next to the workspace.
29+//
30+// For each file that matches the patterns, in the order of @actions/glob's
31+// traversal (a depth first walk with the names of a directory sorted), the
32+// SHA-256 of the file is written to a second SHA-256; the result is its
33+// hexadecimal digest, or "" when no file matched.
34+//
35+// Differences from the official function, on purpose: the files are read
36+// through an os.Root on the workspace, so that a symbolic link that leaves the
37+// workspace is not followed (the executor can be root, the workspace belongs
38+// to the user of the steps) and its file is skipped; and only regular files
39+// are hashed (a FIFO would block until the timeout). A matched symbolic link
40+// whose target is missing fails the call, as it does in the script. Braces ({a,b}), extglobs (+(a|b)) and a leading
41+// "!" in the middle of a pattern are not special, as in @actions/glob.
42+const hashFilesTimeout = 120 * time.Second
43+
44+func (j *job) hashFilesFunc() expr.Func {
45+ return func(ctx context.Context, args []expr.Value) (expr.Value, error) {
46+ var patterns []string
47+ follow := false
48+ for i, a := range args {
49+ p := expr.CoerceString(a)
50+ if i == 0 && strings.HasPrefix(p, "--") {
51+ if strings.EqualFold(p, "--follow-symbolic-links") {
52+ follow = true
53+ continue
54+ }
55+ // new ArgumentOutOfRangeException(message) takes the text as the parameter name.
56+ //nolint:staticcheck,misspell // GitHub's text, typo included
57+ return nil, fmt.Errorf("Specified argument was out of the range of valid values. (Parameter 'Invalid glob option %s, avaliable option: '--follow-symbolic-links'.')", p)
58+ }
59+ patterns = append(patterns, p)
60+ }
61+ ws := j.state.Layout().Workspace
62+ ctx, cancel := context.WithTimeout(ctx, hashFilesTimeout)
63+ defer cancel()
64+ sum, err := hashFilesIn(ctx, ws, j.state.Layout().Home, patterns, follow)
65+ desc := strings.ReplaceAll(strings.Join(patterns, ", "), "'", "''")
66+ switch {
67+ case errors.Is(err, context.DeadlineExceeded):
68+ //nolint:staticcheck // GitHub's text, with its final period
69+ return nil, fmt.Errorf("hashFiles('%s') couldn't finish within %d seconds.", desc, int(hashFilesTimeout/time.Second))
70+ case err != nil:
71+ j.cfg.Log.Debug().Err(err).Msg("hashFiles failed")
72+ return nil, fmt.Errorf("hashFiles('%s') failed. Fail to hash files under directory '%s'", desc, ws)
73+ }
74+ return expr.String(sum), nil
75+ }
76+}
77+
78+// ---------------------------------------------------------------- patterns
79+
80+// seg is a segment of a pattern: its literal value (getLiteral, "" when it has
81+// a wildcard) and its minimatch form.
82+type seg struct {
83+ raw string // the segment as written, for the "**" test
84+ lit string
85+ mm mmSeg
86+ root bool
87+}
88+
89+// mmSeg is a parsed segment: the globstar, a literal, or a regular expression.
90+type mmSeg struct {
91+ globstar bool
92+ lit string
93+ re *regexp.Regexp
94+}
95+
96+type globPattern struct {
97+ negate bool
98+ segs []seg
99+ trailingSep bool
100+ implicit bool
101+ searchPath string
102+}
103+
104+const (
105+ kindNone = 0
106+ kindDir = 1
107+ kindFile = 2
108+ kindAll = 3
109+)
110+
111+func splitSlash(s string) []string {
112+ return slashRunRe.Split(s, -1)
113+}
114+
115+var errPattern = errors.New("invalid pattern")
116+
117+// newGlobPattern is Pattern's constructor for a pattern written by the user,
118+// relative to cwd (the workspace, as segments).
119+func newGlobPattern(text, cwd, home string) (*globPattern, error) {
120+ p := &globPattern{}
121+ text = strings.TrimSpace(text)
122+ for strings.HasPrefix(text, "!") {
123+ p.negate = !p.negate
124+ text = strings.TrimSpace(text[1:])
125+ }
126+ if text == "" {
127+ return nil, fmt.Errorf("%w: pattern cannot be empty", errPattern)
128+ }
129+ norm := doubleSlashRe.ReplaceAllString(text, "/")
130+ p.trailingSep = strings.HasSuffix(norm, "/")
131+ trimmed := norm
132+ if trimmed != "/" {
133+ trimmed = strings.TrimSuffix(trimmed, "/")
134+ }
135+ rooted := strings.HasPrefix(trimmed, "/")
136+ var rawSegs []string
137+ if rooted {
138+ rawSegs = append([]string{"/"}, splitNonEmpty(trimmed)...)
139+ } else {
140+ rawSegs = strings.Split(trimmed, "/")
141+ }
142+ for i, s := range rawSegs {
143+ l := getLiteral(s)
144+ if (l == "." && i != 0) || l == ".." {
145+ return nil, fmt.Errorf("%w: relative pathing '.' and '..' is not allowed: %s", errPattern, text)
146+ }
147+ }
148+ var prefix []seg
149+ rest := rawSegs
150+ switch {
151+ case trimmed == "." || strings.HasPrefix(trimmed, "./"):
152+ prefix, rest = literalSegs(cwd), rawSegs[1:]
153+ case trimmed == "~" || strings.HasPrefix(trimmed, "~/"):
154+ prefix, rest = literalSegs(home), rawSegs[1:]
155+ case rooted:
156+ rest = rawSegs[1:]
157+ prefix = []seg{{raw: "/", lit: "/", mm: mmSeg{lit: ""}, root: true}}
158+ default:
159+ prefix = literalSegs(cwd)
160+ }
161+ p.segs = prefix
162+ for _, r := range rest {
163+ p.segs = append(p.segs, seg{raw: r, lit: getLiteral(r), mm: parseMM(r)})
164+ }
165+ p.computeSearchPath()
166+ return p, nil
167+}
168+
169+func splitNonEmpty(s string) []string {
170+ var out []string
171+ for _, x := range strings.Split(s, "/") {
172+ if x != "" {
173+ out = append(out, x)
174+ }
175+ }
176+ return out
177+}
178+
179+// literalSegs is the absolute path abs as literal segments.
180+func literalSegs(abs string) []seg {
181+ out := []seg{{raw: "/", lit: "/", mm: mmSeg{lit: ""}, root: true}}
182+ for _, s := range splitNonEmpty(abs) {
183+ out = append(out, seg{raw: "", lit: s, mm: mmSeg{lit: s}})
184+ }
185+ return out
186+}
187+
188+func (p *globPattern) computeSearchPath() {
189+ var parts []string
190+ for _, s := range p.segs {
191+ if s.lit == "" {
192+ break
193+ }
194+ parts = append(parts, s.lit)
195+ }
196+ p.searchPath = joinSegs(parts)
197+}
198+
199+func joinSegs(parts []string) string {
200+ if len(parts) == 0 {
201+ return ""
202+ }
203+ out := parts[0]
204+ for _, s := range parts[1:] {
205+ if !strings.HasSuffix(out, "/") {
206+ out += "/"
207+ }
208+ out += s
209+ }
210+ return out
211+}
212+
213+func (p *globPattern) withDescendants() *globPattern {
214+ q := *p
215+ q.segs = append(append([]seg(nil), p.segs...), seg{raw: "**", lit: "", mm: mmSeg{globstar: true}})
216+ q.trailingSep = false
217+ q.implicit = true
218+ q.computeSearchPath()
219+ return &q
220+}
221+
222+// getLiteral is Pattern.getLiteral: the segment with its escapes removed, or ""
223+// when it has a wildcard.
224+func getLiteral(s string) string {
225+ r := []rune(s)
226+ var lit []rune
227+ for i := 0; i < len(r); i++ {
228+ c := r[i]
229+ switch {
230+ case c == '\\' && i+1 < len(r):
231+ i++
232+ lit = append(lit, r[i])
233+ case c == '*' || c == '?':
234+ return ""
235+ case c == '[' && i+1 < len(r):
236+ var set []rune
237+ closed := -1
238+ for i2 := i + 1; i2 < len(r); i2++ {
239+ c2 := r[i2]
240+ if c2 == '\\' && i2+1 < len(r) {
241+ i2++
242+ set = append(set, r[i2])
243+ continue
244+ }
245+ if c2 == ']' {
246+ closed = i2
247+ break
248+ }
249+ set = append(set, c2)
250+ }
251+ if closed >= 0 {
252+ if len(set) > 1 {
253+ return ""
254+ }
255+ if len(set) == 1 {
256+ lit = append(lit, set...)
257+ i = closed
258+ continue
259+ }
260+ }
261+ lit = append(lit, c)
262+ default:
263+ lit = append(lit, c)
264+ }
265+ }
266+ return string(lit)
267+}
268+
269+var (
270+ slashRunRe = regexp.MustCompile(`/+`)
271+ doubleSlashRe = regexp.MustCompile(`//+`)
272+ neverMatchesRe = regexp.MustCompile(`$.`)
273+)
274+
275+const reSpecials = "().*{}+?[]^$\\!"
276+
277+// parseMM is minimatch's parse for one segment with the options of
278+// @actions/glob (dot, nobrace, noext, nonegate, nocomment).
279+func parseMM(s string) mmSeg {
280+ if s == "**" {
281+ return mmSeg{globstar: true}
282+ }
283+ if s == "" {
284+ return mmSeg{lit: ""}
285+ }
286+ re, magic := translateMM([]rune(s))
287+ if !magic {
288+ return mmSeg{lit: globUnescape(s)}
289+ }
290+ rx, err := regexp.Compile("^" + re + "$")
291+ if err != nil {
292+ return mmSeg{re: neverMatchesRe}
293+ }
294+ return mmSeg{re: rx}
295+}
296+
297+func globUnescape(s string) string {
298+ r := []rune(s)
299+ var out []rune
300+ for i := 0; i < len(r); i++ {
301+ if r[i] == '\\' && i+1 < len(r) {
302+ i++
303+ }
304+ out = append(out, r[i])
305+ }
306+ return string(out)
307+}
308+
309+func translateMM(p []rune) (re string, magic bool) {
310+ var sb strings.Builder
311+ escaping, inClass := false, false
312+ classStart, reClassStart := -1, -1
313+ for i := 0; i < len(p); i++ {
314+ c := p[i]
315+ if escaping && strings.ContainsRune(reSpecials, c) {
316+ sb.WriteString(`\` + string(c))
317+ escaping = false
318+ continue
319+ }
320+ switch c {
321+ case '\\':
322+ escaping = true
323+ case '?', '*', '+', '@', '!':
324+ if inClass {
325+ if c == '!' && i == classStart+1 {
326+ c = '^'
327+ }
328+ sb.WriteRune(c)
329+ continue
330+ }
331+ switch c {
332+ case '*':
333+ sb.WriteString(`[^/]*?`)
334+ magic = true
335+ case '?':
336+ sb.WriteString(`[^/]`)
337+ magic = true
338+ default:
339+ sb.WriteString(`\` + string(c))
340+ }
341+ case '(':
342+ if inClass {
343+ sb.WriteRune('(')
344+ } else {
345+ sb.WriteString(`\(`)
346+ }
347+ case ')':
348+ sb.WriteString(`\)`)
349+ case '|':
350+ sb.WriteString(`\|`)
351+ escaping = false
352+ case '[':
353+ if inClass {
354+ sb.WriteString(`\[`)
355+ continue
356+ }
357+ inClass, classStart, reClassStart = true, i, sb.Len()
358+ sb.WriteRune('[')
359+ case ']':
360+ if i == classStart+1 || !inClass {
361+ sb.WriteString(`\]`)
362+ escaping = false
363+ continue
364+ }
365+ cs := string(p[classStart+1 : i])
366+ if _, err := regexp.Compile("[" + cs + "]"); err != nil {
367+ sub, m := translateMM([]rune(cs))
368+ cur := sb.String()
369+ sb.Reset()
370+ sb.WriteString(cur[:reClassStart] + `\[` + sub + `\]`)
371+ magic = magic || m
372+ inClass = false
373+ continue
374+ }
375+ magic = true
376+ inClass = false
377+ sb.WriteRune(']')
378+ default:
379+ if escaping {
380+ escaping = false
381+ } else if strings.ContainsRune(reSpecials, c) && (c != '^' || !inClass) {
382+ sb.WriteRune('\\')
383+ }
384+ sb.WriteRune(c)
385+ }
386+ }
387+ if inClass {
388+ cs := string(p[classStart+1:])
389+ sub, m := translateMM([]rune(cs))
390+ cur := sb.String()
391+ sb.Reset()
392+ sb.WriteString(cur[:reClassStart] + `\[` + sub)
393+ magic = magic || m
394+ }
395+ if escaping {
396+ sb.WriteString(`\\`)
397+ }
398+ return sb.String(), magic
399+}
400+
401+// match is Pattern.match.
402+func (p *globPattern) match(item string) int {
403+ if p.segs[len(p.segs)-1].raw == "**" {
404+ item = doubleSlashRe.ReplaceAllString(item, "/")
405+ if !strings.HasSuffix(item, "/") && !p.implicit {
406+ item += "/"
407+ }
408+ } else {
409+ item = trimTrailingSep(item)
410+ }
411+ if p.matchOne(splitSlash(item), p.mmSegs(), 0, 0, false) {
412+ if p.trailingSep {
413+ return kindDir
414+ }
415+ return kindAll
416+ }
417+ return kindNone
418+}
419+
420+func (p *globPattern) mmSegs() []mmSeg {
421+ out := make([]mmSeg, len(p.segs))
422+ for i, s := range p.segs {
423+ out[i] = s.mm
424+ }
425+ return out
426+}
427+
428+// partialMatch is Pattern.partialMatch.
429+func (p *globPattern) partialMatch(item string) bool {
430+ item = trimTrailingSep(item)
431+ if path.Dir(item) == item {
432+ return true // the root matches the root of every pattern
433+ }
434+ return p.matchOne(splitSlash(item), p.mmSegs(), 0, 0, true)
435+}
436+
437+func trimTrailingSep(s string) string {
438+ s = doubleSlashRe.ReplaceAllString(s, "/")
439+ if s != "/" && strings.HasSuffix(s, "/") {
440+ return s[:len(s)-1]
441+ }
442+ return s
443+}
444+
445+func isDotName(s string) bool { return s == "." || s == ".." }
446+
447+// matchOne is Minimatch.matchOne with the dot option set.
448+func (p *globPattern) matchOne(file []string, pat []mmSeg, fi, pi int, partial bool) bool {
449+ fl, pl := len(file), len(pat)
450+ for ; fi < fl && pi < pl; fi, pi = fi+1, pi+1 {
451+ ps, f := pat[pi], file[fi]
452+ if ps.globstar {
453+ fr, pr := fi, pi+1
454+ if pr == pl {
455+ for ; fi < fl; fi++ {
456+ if isDotName(file[fi]) {
457+ return false
458+ }
459+ }
460+ return true
461+ }
462+ for fr < fl {
463+ if p.matchOne(file, pat, fr, pr, partial) {
464+ return true
465+ }
466+ if isDotName(file[fr]) {
467+ break
468+ }
469+ fr++
470+ }
471+ if partial && fr == fl {
472+ return true
473+ }
474+ return false
475+ }
476+ var hit bool
477+ if ps.re != nil {
478+ hit = ps.re.MatchString(f)
479+ } else {
480+ hit = f == ps.lit
481+ }
482+ if !hit {
483+ return false
484+ }
485+ }
486+ switch {
487+ case fi == fl && pi == pl:
488+ return true
489+ case fi == fl:
490+ return partial
491+ default:
492+ return fi == fl-1 && file[fi] == ""
493+ }
494+}
495+
496+// ---------------------------------------------------------------- walk
497+
498+func matchAll(patterns []*globPattern, item string) int {
499+ r := kindNone
500+ for _, p := range patterns {
501+ if p.negate {
502+ r &^= p.match(item)
503+ } else {
504+ r |= p.match(item)
505+ }
506+ }
507+ return r
508+}
509+
510+// getSearchPaths is internal-pattern-helper's getSearchPaths.
511+func getSearchPaths(patterns []*globPattern) []string {
512+ var pos []*globPattern
513+ for _, p := range patterns {
514+ if !p.negate {
515+ pos = append(pos, p)
516+ }
517+ }
518+ state := map[string]string{}
519+ for _, p := range pos {
520+ state[p.searchPath] = "candidate"
521+ }
522+ var out []string
523+ for _, p := range pos {
524+ if state[p.searchPath] == "included" {
525+ continue
526+ }
527+ found := false
528+ key := p.searchPath
529+ for parent := path.Dir(key); parent != key; parent = path.Dir(key) {
530+ if state[parent] != "" {
531+ found = true
532+ break
533+ }
534+ key = parent
535+ }
536+ if !found {
537+ out = append(out, p.searchPath)
538+ state[p.searchPath] = "included"
539+ }
540+ }
541+ return out
542+}
543+
544+type searchItem struct {
545+ path string
546+ level int
547+}
548+
549+// hashFilesIn is the hashFiles script: the hex SHA-256 of the SHA-256 digests
550+// of the files under ws that match the patterns, "" when none does.
551+func hashFilesIn(ctx context.Context, ws, home string, lines []string, follow bool) (string, error) {
552+ cwd := path.Clean(ws)
553+ var patterns []*globPattern
554+ for _, line := range strings.Split(strings.Join(lines, "\n"), "\n") {
555+ line = strings.TrimSpace(line)
556+ if line == "" || strings.HasPrefix(line, "#") {
557+ continue
558+ }
559+ p, err := newGlobPattern(line, cwd, home)
560+ if err != nil {
561+ return "", err
562+ }
563+ patterns = append(patterns, p)
564+ }
565+ all := make([]*globPattern, 0, 2*len(patterns))
566+ for _, p := range patterns {
567+ all = append(all, p)
568+ if p.trailingSep || p.segs[len(p.segs)-1].raw != "**" {
569+ all = append(all, p.withDescendants())
570+ }
571+ }
572+ root, err := os.OpenRoot(cwd)
573+ if err != nil {
574+ if errors.Is(err, fs.ErrNotExist) {
575+ return "", nil
576+ }
577+ return "", err
578+ }
579+ defer func() { _ = root.Close() }()
580+ rel := func(abs string) (string, bool) {
581+ if abs == cwd {
582+ return ".", true
583+ }
584+ if strings.HasPrefix(abs, cwd+"/") {
585+ return abs[len(cwd)+1:], true
586+ }
587+ return "", false
588+ }
589+
590+ var stack []searchItem
591+ for _, sp := range getSearchPaths(all) {
592+ if _, ok := rel(sp); !ok {
593+ if !strings.HasPrefix(cwd+"/", strings.TrimSuffix(sp, "/")+"/") {
594+ continue // neither under the workspace nor above it
595+ }
596+ sp = cwd
597+ }
598+ r, _ := rel(sp)
599+ if _, err := root.Lstat(r); err != nil {
600+ if errors.Is(err, fs.ErrNotExist) {
601+ continue
602+ }
603+ return "", err
604+ }
605+ stack = append([]searchItem{{sp, 1}}, stack...)
606+ }
607+
608+ total := sha256.New()
609+ hashed := 0
610+ var chain []fs.FileInfo
611+ for len(stack) > 0 {
612+ if err := ctx.Err(); err != nil {
613+ return "", err
614+ }
615+ item := stack[len(stack)-1]
616+ stack = stack[:len(stack)-1]
617+ match := matchAll(all, item.path)
618+ partial := match != kindNone
619+ if !partial {
620+ for _, p := range all {
621+ if !p.negate && p.partialMatch(item.path) {
622+ partial = true
623+ break
624+ }
625+ }
626+ }
627+ if !partial {
628+ continue
629+ }
630+ r, under := rel(item.path)
631+ if !under {
632+ // Above the workspace (an absolute pattern): only the way down matters.
633+ r = "."
634+ if item.path != "/" && !strings.HasPrefix(cwd+"/", item.path+"/") {
635+ continue
636+ }
637+ }
638+ var st fs.FileInfo
639+ if follow {
640+ st, err = root.Stat(r)
641+ } else {
642+ st, err = root.Lstat(r)
643+ }
644+ if err != nil {
645+ if errors.Is(err, fs.ErrNotExist) || follow {
646+ continue // vanished, or a broken or escaping link
647+ }
648+ return "", err
649+ }
650+ if st.IsDir() && follow {
651+ for len(chain) >= item.level {
652+ chain = chain[:len(chain)-1]
653+ }
654+ cycle := false
655+ for _, c := range chain {
656+ if os.SameFile(c, st) {
657+ cycle = true
658+ }
659+ }
660+ if cycle {
661+ continue
662+ }
663+ chain = append(chain, st)
664+ }
665+ if st.IsDir() {
666+ if match&kindDir == 0 && !partial {
667+ continue
668+ }
669+ names, err := readDirNames(root, r)
670+ if err != nil {
671+ return "", err
672+ }
673+ base := item.path
674+ for i := len(names) - 1; i >= 0; i-- {
675+ stack = append(stack, searchItem{path.Join(base, names[i]), item.level + 1})
676+ }
677+ continue // directories are skipped by hashFiles
678+ }
679+ if match&kindFile == 0 || !under {
680+ continue
681+ }
682+ if !st.Mode().IsRegular() && !follow && st.Mode()&fs.ModeSymlink == 0 {
683+ continue
684+ }
685+ sum, ok, err := hashOne(ctx, root, r)
686+ if err != nil {
687+ return "", err
688+ }
689+ if !ok {
690+ if st.Mode()&fs.ModeSymlink != 0 {
691+ if _, serr := root.Stat(r); errors.Is(serr, fs.ErrNotExist) {
692+ // The script stats the link's target and fails when it is
693+ // missing (fs.statSync throws ENOENT): so does the function.
694+ return "", fmt.Errorf("broken symbolic link %s: %w", item.path, serr)
695+ }
696+ }
697+ continue
698+ }
699+ total.Write(sum)
700+ hashed++
701+ }
702+ if hashed == 0 {
703+ return "", nil
704+ }
705+ return hex.EncodeToString(total.Sum(nil)), nil
706+}
707+
708+func readDirNames(root *os.Root, rel string) ([]string, error) {
709+ d, err := root.Open(rel)
710+ if err != nil {
711+ return nil, err
712+ }
713+ defer func() { _ = d.Close() }()
714+ ents, err := d.ReadDir(-1)
715+ if err != nil {
716+ return nil, err
717+ }
718+ names := make([]string, 0, len(ents))
719+ for _, e := range ents {
720+ names = append(names, e.Name())
721+ }
722+ sort.Strings(names)
723+ return names, nil
724+}
725+
726+// hashOne is the SHA-256 digest of a regular file. ok is false for what is not
727+// hashed: a directory (reached through a link), a special file (a FIFO, a
728+// socket, a device: skipped, where the official script would hang on a FIFO),
729+// a link that leaves the workspace or is broken.
730+func hashOne(ctx context.Context, root *os.Root, rel string) (sum []byte, ok bool, err error) {
731+ // O_NONBLOCK: opening a FIFO for reading blocks until a writer comes, and
732+ // nothing can interrupt the open. Reached through a symbolic link (or with
733+ // --follow-symbolic-links) a FIFO would hang the job until its timeout.
734+ f, err := root.OpenFile(rel, os.O_RDONLY|syscall.O_NONBLOCK, 0)
735+ if err != nil {
736+ return nil, false, nil
737+ }
738+ defer func() { _ = f.Close() }()
739+ st, err := f.Stat()
740+ if err != nil || !st.Mode().IsRegular() {
741+ return nil, false, nil
742+ }
743+ h := sha256.New()
744+ if _, err := io.Copy(h, &ctxReader{ctx, f}); err != nil {
745+ return nil, false, err
746+ }
747+ return h.Sum(nil), true, nil
748+}
749+
750+type ctxReader struct {
751+ ctx context.Context
752+ r io.Reader
753+}
754+
755+func (c *ctxReader) Read(p []byte) (int, error) {
756+ if err := c.ctx.Err(); err != nil {
757+ return 0, err
758+ }
759+ return c.r.Read(p)
760+}
@@ -0,0 +1,760 @@
1+// SPDX-License-Identifier: MIT
2+
3+package executor
4+
5+import (
6+ "context"
7+ "crypto/sha256"
8+ "encoding/hex"
9+ "errors"
10+ "fmt"
11+ "io"
12+ "io/fs"
13+ "os"
14+ "path"
15+ "regexp"
16+ "sort"
17+ "strings"
18+ "syscall"
19+ "time"
20+
21+ "bnema.dev/ermite-actions/expr"
22+)
23+
24+// hashFiles is the function of the same name (Runner.Worker/Expressions/
25+// HashFilesFunction.cs). The official runner starts node on its hashFiles
26+// script (Misc/expressionFunc/hashFiles, which uses @actions/glob); this is a
27+// port of that script, evaluated by the executor, which runs in the sandbox
28+// next to the workspace.
29+//
30+// For each file that matches the patterns, in the order of @actions/glob's
31+// traversal (a depth first walk with the names of a directory sorted), the
32+// SHA-256 of the file is written to a second SHA-256; the result is its
33+// hexadecimal digest, or "" when no file matched.
34+//
35+// Differences from the official function, on purpose: the files are read
36+// through an os.Root on the workspace, so that a symbolic link that leaves the
37+// workspace is not followed (the executor can be root, the workspace belongs
38+// to the user of the steps) and its file is skipped; and only regular files
39+// are hashed (a FIFO would block until the timeout). A matched symbolic link
40+// whose target is missing fails the call, as it does in the script. Braces ({a,b}), extglobs (+(a|b)) and a leading
41+// "!" in the middle of a pattern are not special, as in @actions/glob.
42+const hashFilesTimeout = 120 * time.Second
43+
44+func (j *job) hashFilesFunc() expr.Func {
45+ return func(ctx context.Context, args []expr.Value) (expr.Value, error) {
46+ var patterns []string
47+ follow := false
48+ for i, a := range args {
49+ p := expr.CoerceString(a)
50+ if i == 0 && strings.HasPrefix(p, "--") {
51+ if strings.EqualFold(p, "--follow-symbolic-links") {
52+ follow = true
53+ continue
54+ }
55+ // new ArgumentOutOfRangeException(message) takes the text as the parameter name.
56+ //nolint:staticcheck,misspell // GitHub's text, typo included
57+ return nil, fmt.Errorf("Specified argument was out of the range of valid values. (Parameter 'Invalid glob option %s, avaliable option: '--follow-symbolic-links'.')", p)
58+ }
59+ patterns = append(patterns, p)
60+ }
61+ ws := j.state.Layout().Workspace
62+ ctx, cancel := context.WithTimeout(ctx, hashFilesTimeout)
63+ defer cancel()
64+ sum, err := hashFilesIn(ctx, ws, j.state.Layout().Home, patterns, follow)
65+ desc := strings.ReplaceAll(strings.Join(patterns, ", "), "'", "''")
66+ switch {
67+ case errors.Is(err, context.DeadlineExceeded):
68+ //nolint:staticcheck // GitHub's text, with its final period
69+ return nil, fmt.Errorf("hashFiles('%s') couldn't finish within %d seconds.", desc, int(hashFilesTimeout/time.Second))
70+ case err != nil:
71+ j.cfg.Log.Debug().Err(err).Msg("hashFiles failed")
72+ return nil, fmt.Errorf("hashFiles('%s') failed. Fail to hash files under directory '%s'", desc, ws)
73+ }
74+ return expr.String(sum), nil
75+ }
76+}
77+
78+// ---------------------------------------------------------------- patterns
79+
80+// seg is a segment of a pattern: its literal value (getLiteral, "" when it has
81+// a wildcard) and its minimatch form.
82+type seg struct {
83+ raw string // the segment as written, for the "**" test
84+ lit string
85+ mm mmSeg
86+ root bool
87+}
88+
89+// mmSeg is a parsed segment: the globstar, a literal, or a regular expression.
90+type mmSeg struct {
91+ globstar bool
92+ lit string
93+ re *regexp.Regexp
94+}
95+
96+type globPattern struct {
97+ negate bool
98+ segs []seg
99+ trailingSep bool
100+ implicit bool
101+ searchPath string
102+}
103+
104+const (
105+ kindNone = 0
106+ kindDir = 1
107+ kindFile = 2
108+ kindAll = 3
109+)
110+
111+func splitSlash(s string) []string {
112+ return slashRunRe.Split(s, -1)
113+}
114+
115+var errPattern = errors.New("invalid pattern")
116+
117+// newGlobPattern is Pattern's constructor for a pattern written by the user,
118+// relative to cwd (the workspace, as segments).
119+func newGlobPattern(text, cwd, home string) (*globPattern, error) {
120+ p := &globPattern{}
121+ text = strings.TrimSpace(text)
122+ for strings.HasPrefix(text, "!") {
123+ p.negate = !p.negate
124+ text = strings.TrimSpace(text[1:])
125+ }
126+ if text == "" {
127+ return nil, fmt.Errorf("%w: pattern cannot be empty", errPattern)
128+ }
129+ norm := doubleSlashRe.ReplaceAllString(text, "/")
130+ p.trailingSep = strings.HasSuffix(norm, "/")
131+ trimmed := norm
132+ if trimmed != "/" {
133+ trimmed = strings.TrimSuffix(trimmed, "/")
134+ }
135+ rooted := strings.HasPrefix(trimmed, "/")
136+ var rawSegs []string
137+ if rooted {
138+ rawSegs = append([]string{"/"}, splitNonEmpty(trimmed)...)
139+ } else {
140+ rawSegs = strings.Split(trimmed, "/")
141+ }
142+ for i, s := range rawSegs {
143+ l := getLiteral(s)
144+ if (l == "." && i != 0) || l == ".." {
145+ return nil, fmt.Errorf("%w: relative pathing '.' and '..' is not allowed: %s", errPattern, text)
146+ }
147+ }
148+ var prefix []seg
149+ rest := rawSegs
150+ switch {
151+ case trimmed == "." || strings.HasPrefix(trimmed, "./"):
152+ prefix, rest = literalSegs(cwd), rawSegs[1:]
153+ case trimmed == "~" || strings.HasPrefix(trimmed, "~/"):
154+ prefix, rest = literalSegs(home), rawSegs[1:]
155+ case rooted:
156+ rest = rawSegs[1:]
157+ prefix = []seg{{raw: "/", lit: "/", mm: mmSeg{lit: ""}, root: true}}
158+ default:
159+ prefix = literalSegs(cwd)
160+ }
161+ p.segs = prefix
162+ for _, r := range rest {
163+ p.segs = append(p.segs, seg{raw: r, lit: getLiteral(r), mm: parseMM(r)})
164+ }
165+ p.computeSearchPath()
166+ return p, nil
167+}
168+
169+func splitNonEmpty(s string) []string {
170+ var out []string
171+ for _, x := range strings.Split(s, "/") {
172+ if x != "" {
173+ out = append(out, x)
174+ }
175+ }
176+ return out
177+}
178+
179+// literalSegs is the absolute path abs as literal segments.
180+func literalSegs(abs string) []seg {
181+ out := []seg{{raw: "/", lit: "/", mm: mmSeg{lit: ""}, root: true}}
182+ for _, s := range splitNonEmpty(abs) {
183+ out = append(out, seg{raw: "", lit: s, mm: mmSeg{lit: s}})
184+ }
185+ return out
186+}
187+
188+func (p *globPattern) computeSearchPath() {
189+ var parts []string
190+ for _, s := range p.segs {
191+ if s.lit == "" {
192+ break
193+ }
194+ parts = append(parts, s.lit)
195+ }
196+ p.searchPath = joinSegs(parts)
197+}
198+
199+func joinSegs(parts []string) string {
200+ if len(parts) == 0 {
201+ return ""
202+ }
203+ out := parts[0]
204+ for _, s := range parts[1:] {
205+ if !strings.HasSuffix(out, "/") {
206+ out += "/"
207+ }
208+ out += s
209+ }
210+ return out
211+}
212+
213+func (p *globPattern) withDescendants() *globPattern {
214+ q := *p
215+ q.segs = append(append([]seg(nil), p.segs...), seg{raw: "**", lit: "", mm: mmSeg{globstar: true}})
216+ q.trailingSep = false
217+ q.implicit = true
218+ q.computeSearchPath()
219+ return &q
220+}
221+
222+// getLiteral is Pattern.getLiteral: the segment with its escapes removed, or ""
223+// when it has a wildcard.
224+func getLiteral(s string) string {
225+ r := []rune(s)
226+ var lit []rune
227+ for i := 0; i < len(r); i++ {
228+ c := r[i]
229+ switch {
230+ case c == '\\' && i+1 < len(r):
231+ i++
232+ lit = append(lit, r[i])
233+ case c == '*' || c == '?':
234+ return ""
235+ case c == '[' && i+1 < len(r):
236+ var set []rune
237+ closed := -1
238+ for i2 := i + 1; i2 < len(r); i2++ {
239+ c2 := r[i2]
240+ if c2 == '\\' && i2+1 < len(r) {
241+ i2++
242+ set = append(set, r[i2])
243+ continue
244+ }
245+ if c2 == ']' {
246+ closed = i2
247+ break
248+ }
249+ set = append(set, c2)
250+ }
251+ if closed >= 0 {
252+ if len(set) > 1 {
253+ return ""
254+ }
255+ if len(set) == 1 {
256+ lit = append(lit, set...)
257+ i = closed
258+ continue
259+ }
260+ }
261+ lit = append(lit, c)
262+ default:
263+ lit = append(lit, c)
264+ }
265+ }
266+ return string(lit)
267+}
268+
269+var (
270+ slashRunRe = regexp.MustCompile(`/+`)
271+ doubleSlashRe = regexp.MustCompile(`//+`)
272+ neverMatchesRe = regexp.MustCompile(`$.`)
273+)
274+
275+const reSpecials = "().*{}+?[]^$\\!"
276+
277+// parseMM is minimatch's parse for one segment with the options of
278+// @actions/glob (dot, nobrace, noext, nonegate, nocomment).
279+func parseMM(s string) mmSeg {
280+ if s == "**" {
281+ return mmSeg{globstar: true}
282+ }
283+ if s == "" {
284+ return mmSeg{lit: ""}
285+ }
286+ re, magic := translateMM([]rune(s))
287+ if !magic {
288+ return mmSeg{lit: globUnescape(s)}
289+ }
290+ rx, err := regexp.Compile("^" + re + "$")
291+ if err != nil {
292+ return mmSeg{re: neverMatchesRe}
293+ }
294+ return mmSeg{re: rx}
295+}
296+
297+func globUnescape(s string) string {
298+ r := []rune(s)
299+ var out []rune
300+ for i := 0; i < len(r); i++ {
301+ if r[i] == '\\' && i+1 < len(r) {
302+ i++
303+ }
304+ out = append(out, r[i])
305+ }
306+ return string(out)
307+}
308+
309+func translateMM(p []rune) (re string, magic bool) {
310+ var sb strings.Builder
311+ escaping, inClass := false, false
312+ classStart, reClassStart := -1, -1
313+ for i := 0; i < len(p); i++ {
314+ c := p[i]
315+ if escaping && strings.ContainsRune(reSpecials, c) {
316+ sb.WriteString(`\` + string(c))
317+ escaping = false
318+ continue
319+ }
320+ switch c {
321+ case '\\':
322+ escaping = true
323+ case '?', '*', '+', '@', '!':
324+ if inClass {
325+ if c == '!' && i == classStart+1 {
326+ c = '^'
327+ }
328+ sb.WriteRune(c)
329+ continue
330+ }
331+ switch c {
332+ case '*':
333+ sb.WriteString(`[^/]*?`)
334+ magic = true
335+ case '?':
336+ sb.WriteString(`[^/]`)
337+ magic = true
338+ default:
339+ sb.WriteString(`\` + string(c))
340+ }
341+ case '(':
342+ if inClass {
343+ sb.WriteRune('(')
344+ } else {
345+ sb.WriteString(`\(`)
346+ }
347+ case ')':
348+ sb.WriteString(`\)`)
349+ case '|':
350+ sb.WriteString(`\|`)
351+ escaping = false
352+ case '[':
353+ if inClass {
354+ sb.WriteString(`\[`)
355+ continue
356+ }
357+ inClass, classStart, reClassStart = true, i, sb.Len()
358+ sb.WriteRune('[')
359+ case ']':
360+ if i == classStart+1 || !inClass {
361+ sb.WriteString(`\]`)
362+ escaping = false
363+ continue
364+ }
365+ cs := string(p[classStart+1 : i])
366+ if _, err := regexp.Compile("[" + cs + "]"); err != nil {
367+ sub, m := translateMM([]rune(cs))
368+ cur := sb.String()
369+ sb.Reset()
370+ sb.WriteString(cur[:reClassStart] + `\[` + sub + `\]`)
371+ magic = magic || m
372+ inClass = false
373+ continue
374+ }
375+ magic = true
376+ inClass = false
377+ sb.WriteRune(']')
378+ default:
379+ if escaping {
380+ escaping = false
381+ } else if strings.ContainsRune(reSpecials, c) && (c != '^' || !inClass) {
382+ sb.WriteRune('\\')
383+ }
384+ sb.WriteRune(c)
385+ }
386+ }
387+ if inClass {
388+ cs := string(p[classStart+1:])
389+ sub, m := translateMM([]rune(cs))
390+ cur := sb.String()
391+ sb.Reset()
392+ sb.WriteString(cur[:reClassStart] + `\[` + sub)
393+ magic = magic || m
394+ }
395+ if escaping {
396+ sb.WriteString(`\\`)
397+ }
398+ return sb.String(), magic
399+}
400+
401+// match is Pattern.match.
402+func (p *globPattern) match(item string) int {
403+ if p.segs[len(p.segs)-1].raw == "**" {
404+ item = doubleSlashRe.ReplaceAllString(item, "/")
405+ if !strings.HasSuffix(item, "/") && !p.implicit {
406+ item += "/"
407+ }
408+ } else {
409+ item = trimTrailingSep(item)
410+ }
411+ if p.matchOne(splitSlash(item), p.mmSegs(), 0, 0, false) {
412+ if p.trailingSep {
413+ return kindDir
414+ }
415+ return kindAll
416+ }
417+ return kindNone
418+}
419+
420+func (p *globPattern) mmSegs() []mmSeg {
421+ out := make([]mmSeg, len(p.segs))
422+ for i, s := range p.segs {
423+ out[i] = s.mm
424+ }
425+ return out
426+}
427+
428+// partialMatch is Pattern.partialMatch.
429+func (p *globPattern) partialMatch(item string) bool {
430+ item = trimTrailingSep(item)
431+ if path.Dir(item) == item {
432+ return true // the root matches the root of every pattern
433+ }
434+ return p.matchOne(splitSlash(item), p.mmSegs(), 0, 0, true)
435+}
436+
437+func trimTrailingSep(s string) string {
438+ s = doubleSlashRe.ReplaceAllString(s, "/")
439+ if s != "/" && strings.HasSuffix(s, "/") {
440+ return s[:len(s)-1]
441+ }
442+ return s
443+}
444+
445+func isDotName(s string) bool { return s == "." || s == ".." }
446+
447+// matchOne is Minimatch.matchOne with the dot option set.
448+func (p *globPattern) matchOne(file []string, pat []mmSeg, fi, pi int, partial bool) bool {
449+ fl, pl := len(file), len(pat)
450+ for ; fi < fl && pi < pl; fi, pi = fi+1, pi+1 {
451+ ps, f := pat[pi], file[fi]
452+ if ps.globstar {
453+ fr, pr := fi, pi+1
454+ if pr == pl {
455+ for ; fi < fl; fi++ {
456+ if isDotName(file[fi]) {
457+ return false
458+ }
459+ }
460+ return true
461+ }
462+ for fr < fl {
463+ if p.matchOne(file, pat, fr, pr, partial) {
464+ return true
465+ }
466+ if isDotName(file[fr]) {
467+ break
468+ }
469+ fr++
470+ }
471+ if partial && fr == fl {
472+ return true
473+ }
474+ return false
475+ }
476+ var hit bool
477+ if ps.re != nil {
478+ hit = ps.re.MatchString(f)
479+ } else {
480+ hit = f == ps.lit
481+ }
482+ if !hit {
483+ return false
484+ }
485+ }
486+ switch {
487+ case fi == fl && pi == pl:
488+ return true
489+ case fi == fl:
490+ return partial
491+ default:
492+ return fi == fl-1 && file[fi] == ""
493+ }
494+}
495+
496+// ---------------------------------------------------------------- walk
497+
498+func matchAll(patterns []*globPattern, item string) int {
499+ r := kindNone
500+ for _, p := range patterns {
501+ if p.negate {
502+ r &^= p.match(item)
503+ } else {
504+ r |= p.match(item)
505+ }
506+ }
507+ return r
508+}
509+
510+// getSearchPaths is internal-pattern-helper's getSearchPaths.
511+func getSearchPaths(patterns []*globPattern) []string {
512+ var pos []*globPattern
513+ for _, p := range patterns {
514+ if !p.negate {
515+ pos = append(pos, p)
516+ }
517+ }
518+ state := map[string]string{}
519+ for _, p := range pos {
520+ state[p.searchPath] = "candidate"
521+ }
522+ var out []string
523+ for _, p := range pos {
524+ if state[p.searchPath] == "included" {
525+ continue
526+ }
527+ found := false
528+ key := p.searchPath
529+ for parent := path.Dir(key); parent != key; parent = path.Dir(key) {
530+ if state[parent] != "" {
531+ found = true
532+ break
533+ }
534+ key = parent
535+ }
536+ if !found {
537+ out = append(out, p.searchPath)
538+ state[p.searchPath] = "included"
539+ }
540+ }
541+ return out
542+}
543+
544+type searchItem struct {
545+ path string
546+ level int
547+}
548+
549+// hashFilesIn is the hashFiles script: the hex SHA-256 of the SHA-256 digests
550+// of the files under ws that match the patterns, "" when none does.
551+func hashFilesIn(ctx context.Context, ws, home string, lines []string, follow bool) (string, error) {
552+ cwd := path.Clean(ws)
553+ var patterns []*globPattern
554+ for _, line := range strings.Split(strings.Join(lines, "\n"), "\n") {
555+ line = strings.TrimSpace(line)
556+ if line == "" || strings.HasPrefix(line, "#") {
557+ continue
558+ }
559+ p, err := newGlobPattern(line, cwd, home)
560+ if err != nil {
561+ return "", err
562+ }
563+ patterns = append(patterns, p)
564+ }
565+ all := make([]*globPattern, 0, 2*len(patterns))
566+ for _, p := range patterns {
567+ all = append(all, p)
568+ if p.trailingSep || p.segs[len(p.segs)-1].raw != "**" {
569+ all = append(all, p.withDescendants())
570+ }
571+ }
572+ root, err := os.OpenRoot(cwd)
573+ if err != nil {
574+ if errors.Is(err, fs.ErrNotExist) {
575+ return "", nil
576+ }
577+ return "", err
578+ }
579+ defer func() { _ = root.Close() }()
580+ rel := func(abs string) (string, bool) {
581+ if abs == cwd {
582+ return ".", true
583+ }
584+ if strings.HasPrefix(abs, cwd+"/") {
585+ return abs[len(cwd)+1:], true
586+ }
587+ return "", false
588+ }
589+
590+ var stack []searchItem
591+ for _, sp := range getSearchPaths(all) {
592+ if _, ok := rel(sp); !ok {
593+ if !strings.HasPrefix(cwd+"/", strings.TrimSuffix(sp, "/")+"/") {
594+ continue // neither under the workspace nor above it
595+ }
596+ sp = cwd
597+ }
598+ r, _ := rel(sp)
599+ if _, err := root.Lstat(r); err != nil {
600+ if errors.Is(err, fs.ErrNotExist) {
601+ continue
602+ }
603+ return "", err
604+ }
605+ stack = append([]searchItem{{sp, 1}}, stack...)
606+ }
607+
608+ total := sha256.New()
609+ hashed := 0
610+ var chain []fs.FileInfo
611+ for len(stack) > 0 {
612+ if err := ctx.Err(); err != nil {
613+ return "", err
614+ }
615+ item := stack[len(stack)-1]
616+ stack = stack[:len(stack)-1]
617+ match := matchAll(all, item.path)
618+ partial := match != kindNone
619+ if !partial {
620+ for _, p := range all {
621+ if !p.negate && p.partialMatch(item.path) {
622+ partial = true
623+ break
624+ }
625+ }
626+ }
627+ if !partial {
628+ continue
629+ }
630+ r, under := rel(item.path)
631+ if !under {
632+ // Above the workspace (an absolute pattern): only the way down matters.
633+ r = "."
634+ if item.path != "/" && !strings.HasPrefix(cwd+"/", item.path+"/") {
635+ continue
636+ }
637+ }
638+ var st fs.FileInfo
639+ if follow {
640+ st, err = root.Stat(r)
641+ } else {
642+ st, err = root.Lstat(r)
643+ }
644+ if err != nil {
645+ if errors.Is(err, fs.ErrNotExist) || follow {
646+ continue // vanished, or a broken or escaping link
647+ }
648+ return "", err
649+ }
650+ if st.IsDir() && follow {
651+ for len(chain) >= item.level {
652+ chain = chain[:len(chain)-1]
653+ }
654+ cycle := false
655+ for _, c := range chain {
656+ if os.SameFile(c, st) {
657+ cycle = true
658+ }
659+ }
660+ if cycle {
661+ continue
662+ }
663+ chain = append(chain, st)
664+ }
665+ if st.IsDir() {
666+ if match&kindDir == 0 && !partial {
667+ continue
668+ }
669+ names, err := readDirNames(root, r)
670+ if err != nil {
671+ return "", err
672+ }
673+ base := item.path
674+ for i := len(names) - 1; i >= 0; i-- {
675+ stack = append(stack, searchItem{path.Join(base, names[i]), item.level + 1})
676+ }
677+ continue // directories are skipped by hashFiles
678+ }
679+ if match&kindFile == 0 || !under {
680+ continue
681+ }
682+ if !st.Mode().IsRegular() && !follow && st.Mode()&fs.ModeSymlink == 0 {
683+ continue
684+ }
685+ sum, ok, err := hashOne(ctx, root, r)
686+ if err != nil {
687+ return "", err
688+ }
689+ if !ok {
690+ if st.Mode()&fs.ModeSymlink != 0 {
691+ if _, serr := root.Stat(r); errors.Is(serr, fs.ErrNotExist) {
692+ // The script stats the link's target and fails when it is
693+ // missing (fs.statSync throws ENOENT): so does the function.
694+ return "", fmt.Errorf("broken symbolic link %s: %w", item.path, serr)
695+ }
696+ }
697+ continue
698+ }
699+ total.Write(sum)
700+ hashed++
701+ }
702+ if hashed == 0 {
703+ return "", nil
704+ }
705+ return hex.EncodeToString(total.Sum(nil)), nil
706+}
707+
708+func readDirNames(root *os.Root, rel string) ([]string, error) {
709+ d, err := root.Open(rel)
710+ if err != nil {
711+ return nil, err
712+ }
713+ defer func() { _ = d.Close() }()
714+ ents, err := d.ReadDir(-1)
715+ if err != nil {
716+ return nil, err
717+ }
718+ names := make([]string, 0, len(ents))
719+ for _, e := range ents {
720+ names = append(names, e.Name())
721+ }
722+ sort.Strings(names)
723+ return names, nil
724+}
725+
726+// hashOne is the SHA-256 digest of a regular file. ok is false for what is not
727+// hashed: a directory (reached through a link), a special file (a FIFO, a
728+// socket, a device: skipped, where the official script would hang on a FIFO),
729+// a link that leaves the workspace or is broken.
730+func hashOne(ctx context.Context, root *os.Root, rel string) (sum []byte, ok bool, err error) {
731+ // O_NONBLOCK: opening a FIFO for reading blocks until a writer comes, and
732+ // nothing can interrupt the open. Reached through a symbolic link (or with
733+ // --follow-symbolic-links) a FIFO would hang the job until its timeout.
734+ f, err := root.OpenFile(rel, os.O_RDONLY|syscall.O_NONBLOCK, 0)
735+ if err != nil {
736+ return nil, false, nil
737+ }
738+ defer func() { _ = f.Close() }()
739+ st, err := f.Stat()
740+ if err != nil || !st.Mode().IsRegular() {
741+ return nil, false, nil
742+ }
743+ h := sha256.New()
744+ if _, err := io.Copy(h, &ctxReader{ctx, f}); err != nil {
745+ return nil, false, err
746+ }
747+ return h.Sum(nil), true, nil
748+}
749+
750+type ctxReader struct {
751+ ctx context.Context
752+ r io.Reader
753+}
754+
755+func (c *ctxReader) Read(p []byte) (int, error) {
756+ if err := c.ctx.Err(); err != nil {
757+ return 0, err
758+ }
759+ return c.r.Read(p)
760+}
A internal/executor/hashfiles_test.go +216 -0
@@ -0,0 +1,216 @@
1+// SPDX-License-Identifier: MIT
2+
3+package executor
4+
5+import (
6+ "os"
7+ "os/exec"
8+ "path/filepath"
9+ "strings"
10+ "syscall"
11+ "testing"
12+ "time"
13+)
14+
15+func writeTree(t *testing.T, root string, files map[string]string) {
16+ t.Helper()
17+ for name, body := range files {
18+ p := filepath.Join(root, filepath.FromSlash(name))
19+ if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
20+ t.Fatal(err)
21+ }
22+ if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
23+ t.Fatal(err)
24+ }
25+ }
26+}
27+
28+// Values recorded on github.com (expr-hashfiles), for the files the scenario
29+// creates: alpha, beta and gamma.
30+func TestHashFilesRecordedValues(t *testing.T) {
31+ ws := t.TempDir()
32+ writeTree(t, ws, map[string]string{"hf/a.txt": "alpha\n", "hf/b.txt": "beta\n", "hf/sub/c.md": "gamma\n"})
33+ for _, tc := range []struct {
34+ patterns []string
35+ want string
36+ }{
37+ {[]string{"hf/a.txt"}, "4bb706b95c7ea23f44bc5d035ad8841af479871295d2ae0c685d07174705c880"},
38+ {[]string{"hf/a.txt", "hf/b.txt"}, "24d116e0411b3a4a8d3d5c9c88c150bc4d4603a490294bd4b23d3ef549e1f1a0"},
39+ {[]string{"hf/b.txt", "hf/a.txt"}, "a3013a35661fd0340c6651bfd9b89fd7c86e916011c7e9cfecfb835d90ec7505"},
40+ {[]string{"hf/*.txt"}, "24d116e0411b3a4a8d3d5c9c88c150bc4d4603a490294bd4b23d3ef549e1f1a0"},
41+ {[]string{"hf/**"}, "58d680606dbb1f9fab754d2d071e528858b6ca169b64be479ae9557d72c18d77"},
42+ {[]string{"hf/**/*.md"}, "b9f1217dc10c76d1ab32754b4df630a29025a4a878aeddd113fda7045f9faf46"},
43+ {[]string{"hf/**", "!hf/sub/**"}, "24d116e0411b3a4a8d3d5c9c88c150bc4d4603a490294bd4b23d3ef549e1f1a0"},
44+ {[]string{"hf/nothing-*"}, ""},
45+ } {
46+ got, err := hashFilesIn(t.Context(), ws, "/home/x", tc.patterns, false)
47+ if err != nil || got != tc.want {
48+ t.Errorf("%q = %q (%v), want %q", tc.patterns, got, err, tc.want)
49+ }
50+ }
51+ // "before files exist": the directory is not there yet.
52+ got, err := hashFilesIn(t.Context(), t.TempDir(), "/home/x", []string{"hf/**"}, false)
53+ if err != nil || got != "" {
54+ t.Errorf("empty workspace: %q %v", got, err)
55+ }
56+}
57+
58+// The scenario end to end, in expressions, against the recording.
59+func TestCorpusHashFiles(t *testing.T) {
60+ compareWithRecording(t, "expr-hashfiles")
61+}
62+
63+// Differential test: the official script (Misc/expressionFunc/hashFiles, run by
64+// node) and this port must agree. Opt in with ERMITE_HASHFILES_JS=<path of the
65+// script, Misc/layoutbin/hashFiles/index.js of actions/runner>.
66+func TestHashFilesMatchesOfficialScript(t *testing.T) {
67+ script := os.Getenv("ERMITE_HASHFILES_JS")
68+ if script == "" {
69+ t.Skip("set ERMITE_HASHFILES_JS to Misc/layoutbin/hashFiles/index.js of actions/runner")
70+ }
71+ needTools(t, "node")
72+ ws := t.TempDir()
73+ writeTree(t, ws, map[string]string{
74+ "a.txt": "a", "b.go": "b", ".hidden/x": "h", ".dotfile": "d", "src/main.go": "m", "src/util/u.go": "u",
75+ "src/util/u_test.go": "t", "src/Zeta.go": "z", "src/alpha.GO": "A", "docs/readme.md": "r", "docs/img/p.png": "p",
76+ "node_modules/pkg/index.js": "n", "weird [name]/f.txt": "w", "a b/c d.txt": "s", "go.mod": "g", "go.sum": "s2",
77+ "deep/1/2/3/4/f": "deep", "ünï/çode.txt": "u",
78+ })
79+ if err := os.Symlink("src", filepath.Join(ws, "link")); err != nil {
80+ t.Fatal(err)
81+ }
82+ if err := os.Symlink("missing", filepath.Join(ws, "broken")); err != nil {
83+ t.Fatal(err)
84+ }
85+ for _, patterns := range [][]string{
86+ {"**"}, {"*"}, {"**/*.go"}, {"**/*.go", "!**/*_test.go"}, {"src/**", "!src/util/**"}, {"**/go.sum", "**/go.mod"},
87+ {"go.mod", "go.sum"}, {"src/*.go"}, {"src/?lpha.GO"}, {"src/[Zz]eta.go"}, {"**/*.{go,md}"}, {"docs"}, {"docs/"}, {"docs/*"},
88+ {"./src/main.go"}, {"**/.hidden/**"}, {".*"}, {"**/node_modules/**", "src/**"}, {"!src/**", "**/*.go"}, {"nothing"}, {"**/*.md", "# a comment", ""},
89+ {"weird [[]name]/f.txt"}, {"weird*/*"}, {"a b/*"}, {"src/**/", "docs/**"}, {"ünï/*"}, {"deep/**/f"}, {"**/f"}, {"link/**"}, {"link/main.go"},
90+ {"broken"}, {"**/*.GO"}, {"src/util/../main.go"}, {"/etc/hostname"}, {"src\\main.go"}, {"**/u*"}, {"*/*/*"}, {"src/**/*", "!src/util"},
91+ } {
92+ for _, follow := range []bool{false, true} {
93+ want := officialHash(t, script, ws, patterns, follow)
94+ got, err := hashFilesIn(t.Context(), ws, "/home/x", patterns, follow)
95+ if want.failed {
96+ if err == nil {
97+ t.Errorf("%q follow=%v: the script fails (%s), got %q", patterns, follow, want.msg, got)
98+ }
99+ continue
100+ }
101+ if err != nil || got != want.sum {
102+ t.Errorf("%q follow=%v: got %q (%v), script %q", patterns, follow, got, err, want.sum)
103+ }
104+ }
105+ }
106+}
107+
108+// A symbolic link that leaves the workspace is not followed, whatever the
109+// script does: the executor may run as root and the workspace is the user's.
110+func TestHashFilesDoesNotLeaveTheWorkspace(t *testing.T) {
111+ outside := t.TempDir()
112+ writeTree(t, outside, map[string]string{"secret": "s"})
113+ ws := t.TempDir()
114+ writeTree(t, ws, map[string]string{"a.txt": "alpha\n"})
115+ if err := os.Symlink(filepath.Join(outside, "secret"), filepath.Join(ws, "leak")); err != nil {
116+ t.Fatal(err)
117+ }
118+ if err := os.Symlink(outside, filepath.Join(ws, "leakdir")); err != nil {
119+ t.Fatal(err)
120+ }
121+ only, _ := hashFilesIn(t.Context(), ws, "/h", []string{"a.txt"}, false)
122+ for _, follow := range []bool{false, true} {
123+ got, err := hashFilesIn(t.Context(), ws, "/h", []string{"**"}, follow)
124+ if err != nil || got != only {
125+ t.Errorf("follow=%v: %q (%v), want the hash of a.txt alone %q", follow, got, err, only)
126+ }
127+ }
128+}
129+
130+// A matched symbolic link without a target fails the call, like the script.
131+func TestHashFilesBrokenLinkFails(t *testing.T) {
132+ ws := t.TempDir()
133+ if err := os.Symlink("missing", filepath.Join(ws, "broken")); err != nil {
134+ t.Fatal(err)
135+ }
136+ if _, err := hashFilesIn(t.Context(), ws, "/h", []string{"**"}, false); err == nil {
137+ t.Error("want an error")
138+ }
139+ if got, err := hashFilesIn(t.Context(), ws, "/h", []string{"**"}, true); err != nil || got != "" {
140+ t.Errorf("followed, the link is omitted: %q %v", got, err)
141+ }
142+}
143+
144+// A FIFO in the workspace must not hang the function: opening it for reading
145+// blocks until a writer comes. It is skipped, directly or through a symbolic
146+// link, with or without --follow-symbolic-links.
147+func TestHashFilesSkipsFIFOs(t *testing.T) {
148+ ws := t.TempDir()
149+ writeTree(t, ws, map[string]string{"a.txt": "alpha\n"})
150+ if err := syscall.Mkfifo(filepath.Join(ws, "pipe"), 0o644); err != nil {
151+ t.Skipf("no FIFO here: %v", err)
152+ }
153+ if err := os.Symlink("pipe", filepath.Join(ws, "pipelink")); err != nil {
154+ t.Fatal(err)
155+ }
156+ only, err := hashFilesIn(t.Context(), ws, "/h", []string{"a.txt"}, false)
157+ if err != nil || only == "" {
158+ t.Fatalf("%q %v", only, err)
159+ }
160+ for _, tc := range []struct {
161+ patterns []string
162+ follow bool
163+ }{
164+ {[]string{"**"}, false}, {[]string{"**"}, true}, {[]string{"pipe"}, false}, {[]string{"pipe"}, true},
165+ {[]string{"pipelink"}, false}, {[]string{"pipelink"}, true},
166+ } {
167+ done := make(chan struct{})
168+ var got string
169+ var err error
170+ go func() {
171+ defer close(done)
172+ got, err = hashFilesIn(t.Context(), ws, "/h", tc.patterns, tc.follow)
173+ }()
174+ select {
175+ case <-done:
176+ case <-time.After(5 * time.Second):
177+ t.Fatalf("%q follow=%v: hangs on a FIFO", tc.patterns, tc.follow)
178+ }
179+ if err != nil {
180+ t.Errorf("%q follow=%v: %v", tc.patterns, tc.follow, err)
181+ }
182+ if tc.patterns[0] == "**" && got != only {
183+ t.Errorf("%q follow=%v: %q, want a.txt alone %q", tc.patterns, tc.follow, got, only)
184+ }
185+ if tc.patterns[0] != "**" && got != "" {
186+ t.Errorf("%q follow=%v: %q, want no file", tc.patterns, tc.follow, got)
187+ }
188+ }
189+}
190+
191+type officialResult struct {
192+ sum string
193+ failed bool
194+ msg string
195+}
196+
197+func officialHash(t *testing.T, script, ws string, patterns []string, follow bool) officialResult {
198+ t.Helper()
199+ cmd := exec.Command("node", script)
200+ cmd.Dir = ws
201+ cmd.Env = append(os.Environ(), "patterns="+strings.Join(patterns, "\n"))
202+ if follow {
203+ cmd.Env = append(cmd.Env, "followSymbolicLinks=true")
204+ }
205+ out, err := cmd.CombinedOutput()
206+ if err != nil {
207+ return officialResult{failed: true, msg: string(out)}
208+ }
209+ s := string(out)
210+ i := strings.Index(s, "__OUTPUT__")
211+ j := strings.LastIndex(s, "__OUTPUT__")
212+ if i < 0 || j <= i {
213+ t.Fatalf("no output: %s", s)
214+ }
215+ return officialResult{sum: s[i+10 : j]}
216+}
@@ -0,0 +1,216 @@
1+// SPDX-License-Identifier: MIT
2+
3+package executor
4+
5+import (
6+ "os"
7+ "os/exec"
8+ "path/filepath"
9+ "strings"
10+ "syscall"
11+ "testing"
12+ "time"
13+)
14+
15+func writeTree(t *testing.T, root string, files map[string]string) {
16+ t.Helper()
17+ for name, body := range files {
18+ p := filepath.Join(root, filepath.FromSlash(name))
19+ if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
20+ t.Fatal(err)
21+ }
22+ if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
23+ t.Fatal(err)
24+ }
25+ }
26+}
27+
28+// Values recorded on github.com (expr-hashfiles), for the files the scenario
29+// creates: alpha, beta and gamma.
30+func TestHashFilesRecordedValues(t *testing.T) {
31+ ws := t.TempDir()
32+ writeTree(t, ws, map[string]string{"hf/a.txt": "alpha\n", "hf/b.txt": "beta\n", "hf/sub/c.md": "gamma\n"})
33+ for _, tc := range []struct {
34+ patterns []string
35+ want string
36+ }{
37+ {[]string{"hf/a.txt"}, "4bb706b95c7ea23f44bc5d035ad8841af479871295d2ae0c685d07174705c880"},
38+ {[]string{"hf/a.txt", "hf/b.txt"}, "24d116e0411b3a4a8d3d5c9c88c150bc4d4603a490294bd4b23d3ef549e1f1a0"},
39+ {[]string{"hf/b.txt", "hf/a.txt"}, "a3013a35661fd0340c6651bfd9b89fd7c86e916011c7e9cfecfb835d90ec7505"},
40+ {[]string{"hf/*.txt"}, "24d116e0411b3a4a8d3d5c9c88c150bc4d4603a490294bd4b23d3ef549e1f1a0"},
41+ {[]string{"hf/**"}, "58d680606dbb1f9fab754d2d071e528858b6ca169b64be479ae9557d72c18d77"},
42+ {[]string{"hf/**/*.md"}, "b9f1217dc10c76d1ab32754b4df630a29025a4a878aeddd113fda7045f9faf46"},
43+ {[]string{"hf/**", "!hf/sub/**"}, "24d116e0411b3a4a8d3d5c9c88c150bc4d4603a490294bd4b23d3ef549e1f1a0"},
44+ {[]string{"hf/nothing-*"}, ""},
45+ } {
46+ got, err := hashFilesIn(t.Context(), ws, "/home/x", tc.patterns, false)
47+ if err != nil || got != tc.want {
48+ t.Errorf("%q = %q (%v), want %q", tc.patterns, got, err, tc.want)
49+ }
50+ }
51+ // "before files exist": the directory is not there yet.
52+ got, err := hashFilesIn(t.Context(), t.TempDir(), "/home/x", []string{"hf/**"}, false)
53+ if err != nil || got != "" {
54+ t.Errorf("empty workspace: %q %v", got, err)
55+ }
56+}
57+
58+// The scenario end to end, in expressions, against the recording.
59+func TestCorpusHashFiles(t *testing.T) {
60+ compareWithRecording(t, "expr-hashfiles")
61+}
62+
63+// Differential test: the official script (Misc/expressionFunc/hashFiles, run by
64+// node) and this port must agree. Opt in with ERMITE_HASHFILES_JS=<path of the
65+// script, Misc/layoutbin/hashFiles/index.js of actions/runner>.
66+func TestHashFilesMatchesOfficialScript(t *testing.T) {
67+ script := os.Getenv("ERMITE_HASHFILES_JS")
68+ if script == "" {
69+ t.Skip("set ERMITE_HASHFILES_JS to Misc/layoutbin/hashFiles/index.js of actions/runner")
70+ }
71+ needTools(t, "node")
72+ ws := t.TempDir()
73+ writeTree(t, ws, map[string]string{
74+ "a.txt": "a", "b.go": "b", ".hidden/x": "h", ".dotfile": "d", "src/main.go": "m", "src/util/u.go": "u",
75+ "src/util/u_test.go": "t", "src/Zeta.go": "z", "src/alpha.GO": "A", "docs/readme.md": "r", "docs/img/p.png": "p",
76+ "node_modules/pkg/index.js": "n", "weird [name]/f.txt": "w", "a b/c d.txt": "s", "go.mod": "g", "go.sum": "s2",
77+ "deep/1/2/3/4/f": "deep", "ünï/çode.txt": "u",
78+ })
79+ if err := os.Symlink("src", filepath.Join(ws, "link")); err != nil {
80+ t.Fatal(err)
81+ }
82+ if err := os.Symlink("missing", filepath.Join(ws, "broken")); err != nil {
83+ t.Fatal(err)
84+ }
85+ for _, patterns := range [][]string{
86+ {"**"}, {"*"}, {"**/*.go"}, {"**/*.go", "!**/*_test.go"}, {"src/**", "!src/util/**"}, {"**/go.sum", "**/go.mod"},
87+ {"go.mod", "go.sum"}, {"src/*.go"}, {"src/?lpha.GO"}, {"src/[Zz]eta.go"}, {"**/*.{go,md}"}, {"docs"}, {"docs/"}, {"docs/*"},
88+ {"./src/main.go"}, {"**/.hidden/**"}, {".*"}, {"**/node_modules/**", "src/**"}, {"!src/**", "**/*.go"}, {"nothing"}, {"**/*.md", "# a comment", ""},
89+ {"weird [[]name]/f.txt"}, {"weird*/*"}, {"a b/*"}, {"src/**/", "docs/**"}, {"ünï/*"}, {"deep/**/f"}, {"**/f"}, {"link/**"}, {"link/main.go"},
90+ {"broken"}, {"**/*.GO"}, {"src/util/../main.go"}, {"/etc/hostname"}, {"src\\main.go"}, {"**/u*"}, {"*/*/*"}, {"src/**/*", "!src/util"},
91+ } {
92+ for _, follow := range []bool{false, true} {
93+ want := officialHash(t, script, ws, patterns, follow)
94+ got, err := hashFilesIn(t.Context(), ws, "/home/x", patterns, follow)
95+ if want.failed {
96+ if err == nil {
97+ t.Errorf("%q follow=%v: the script fails (%s), got %q", patterns, follow, want.msg, got)
98+ }
99+ continue
100+ }
101+ if err != nil || got != want.sum {
102+ t.Errorf("%q follow=%v: got %q (%v), script %q", patterns, follow, got, err, want.sum)
103+ }
104+ }
105+ }
106+}
107+
108+// A symbolic link that leaves the workspace is not followed, whatever the
109+// script does: the executor may run as root and the workspace is the user's.
110+func TestHashFilesDoesNotLeaveTheWorkspace(t *testing.T) {
111+ outside := t.TempDir()
112+ writeTree(t, outside, map[string]string{"secret": "s"})
113+ ws := t.TempDir()
114+ writeTree(t, ws, map[string]string{"a.txt": "alpha\n"})
115+ if err := os.Symlink(filepath.Join(outside, "secret"), filepath.Join(ws, "leak")); err != nil {
116+ t.Fatal(err)
117+ }
118+ if err := os.Symlink(outside, filepath.Join(ws, "leakdir")); err != nil {
119+ t.Fatal(err)
120+ }
121+ only, _ := hashFilesIn(t.Context(), ws, "/h", []string{"a.txt"}, false)
122+ for _, follow := range []bool{false, true} {
123+ got, err := hashFilesIn(t.Context(), ws, "/h", []string{"**"}, follow)
124+ if err != nil || got != only {
125+ t.Errorf("follow=%v: %q (%v), want the hash of a.txt alone %q", follow, got, err, only)
126+ }
127+ }
128+}
129+
130+// A matched symbolic link without a target fails the call, like the script.
131+func TestHashFilesBrokenLinkFails(t *testing.T) {
132+ ws := t.TempDir()
133+ if err := os.Symlink("missing", filepath.Join(ws, "broken")); err != nil {
134+ t.Fatal(err)
135+ }
136+ if _, err := hashFilesIn(t.Context(), ws, "/h", []string{"**"}, false); err == nil {
137+ t.Error("want an error")
138+ }
139+ if got, err := hashFilesIn(t.Context(), ws, "/h", []string{"**"}, true); err != nil || got != "" {
140+ t.Errorf("followed, the link is omitted: %q %v", got, err)
141+ }
142+}
143+
144+// A FIFO in the workspace must not hang the function: opening it for reading
145+// blocks until a writer comes. It is skipped, directly or through a symbolic
146+// link, with or without --follow-symbolic-links.
147+func TestHashFilesSkipsFIFOs(t *testing.T) {
148+ ws := t.TempDir()
149+ writeTree(t, ws, map[string]string{"a.txt": "alpha\n"})
150+ if err := syscall.Mkfifo(filepath.Join(ws, "pipe"), 0o644); err != nil {
151+ t.Skipf("no FIFO here: %v", err)
152+ }
153+ if err := os.Symlink("pipe", filepath.Join(ws, "pipelink")); err != nil {
154+ t.Fatal(err)
155+ }
156+ only, err := hashFilesIn(t.Context(), ws, "/h", []string{"a.txt"}, false)
157+ if err != nil || only == "" {
158+ t.Fatalf("%q %v", only, err)
159+ }
160+ for _, tc := range []struct {
161+ patterns []string
162+ follow bool
163+ }{
164+ {[]string{"**"}, false}, {[]string{"**"}, true}, {[]string{"pipe"}, false}, {[]string{"pipe"}, true},
165+ {[]string{"pipelink"}, false}, {[]string{"pipelink"}, true},
166+ } {
167+ done := make(chan struct{})
168+ var got string
169+ var err error
170+ go func() {
171+ defer close(done)
172+ got, err = hashFilesIn(t.Context(), ws, "/h", tc.patterns, tc.follow)
173+ }()
174+ select {
175+ case <-done:
176+ case <-time.After(5 * time.Second):
177+ t.Fatalf("%q follow=%v: hangs on a FIFO", tc.patterns, tc.follow)
178+ }
179+ if err != nil {
180+ t.Errorf("%q follow=%v: %v", tc.patterns, tc.follow, err)
181+ }
182+ if tc.patterns[0] == "**" && got != only {
183+ t.Errorf("%q follow=%v: %q, want a.txt alone %q", tc.patterns, tc.follow, got, only)
184+ }
185+ if tc.patterns[0] != "**" && got != "" {
186+ t.Errorf("%q follow=%v: %q, want no file", tc.patterns, tc.follow, got)
187+ }
188+ }
189+}
190+
191+type officialResult struct {
192+ sum string
193+ failed bool
194+ msg string
195+}
196+
197+func officialHash(t *testing.T, script, ws string, patterns []string, follow bool) officialResult {
198+ t.Helper()
199+ cmd := exec.Command("node", script)
200+ cmd.Dir = ws
201+ cmd.Env = append(os.Environ(), "patterns="+strings.Join(patterns, "\n"))
202+ if follow {
203+ cmd.Env = append(cmd.Env, "followSymbolicLinks=true")
204+ }
205+ out, err := cmd.CombinedOutput()
206+ if err != nil {
207+ return officialResult{failed: true, msg: string(out)}
208+ }
209+ s := string(out)
210+ i := strings.Index(s, "__OUTPUT__")
211+ j := strings.LastIndex(s, "__OUTPUT__")
212+ if i < 0 || j <= i {
213+ t.Fatalf("no output: %s", s)
214+ }
215+ return officialResult{sum: s[i+10 : j]}
216+}
M internal/executor/job.go +21 -9
@@ -10,6 +10,7 @@ import (
1010 "path/filepath"
1111 "strings"
1212 "sync"
13+ "sync/atomic"
1314 "syscall"
1415 "time"
1516
@@ -144,8 +145,13 @@ type job struct {
144145 baseEnv []string
145146 lookupPath string
146147 before map[string]bool
148+ // masker hides the secrets in everything reported; it is set once the
149+ // contexts are built, before any step runs.
150+ masker atomic.Pointer[masker]
147151 }
148152
153+func (j *job) mask(s string) string { return j.masker.Load().mask(s) }
154+
149155 func (j *job) now() time.Time {
150156 if j.cfg.Now != nil {
151157 return j.cfg.Now()
@@ -160,6 +166,7 @@ func Run(cancel *Canceller, cfg Config) Result {
160166 cancel = NewCanceller()
161167 }
162168 j := &job{cfg: cfg, cancel: cancel, trackID: "github_" + uuid.NewString()}
169+ j.cfg.Reporter = &maskReporter{j: j, next: cfg.Reporter}
163170 defer func() {
164171 if j.fc != nil {
165172 _ = j.fc.Close()
@@ -169,7 +176,8 @@ func Run(cancel *Canceller, cfg Config) Result {
169176 }
170177 }()
171178 res := j.run()
172- cfg.Reporter.JobCompleted(res)
179+ res.Error = j.mask(res.Error)
180+ j.cfg.Reporter.JobCompleted(res)
173181 return res
174182 }
175183
@@ -253,6 +261,7 @@ func (j *job) out(num int, text string) {
253261
254262 // tagged writes tag+message; only the first line carries the tag.
255263 func (j *job) tagged(num int, tag, msg string) {
264+ msg = j.mask(msg) // before the cut: a secret split by it would leak its start
256265 if len(msg) > maxIssueMessageLength && tag != "" {
257266 msg = msg[:maxIssueMessageLength]
258267 }
@@ -281,13 +290,7 @@ func (j *job) debug(num int, msg string) {
281290 func (j *job) evalEnv(env *ordered) template.Env {
282291 return template.Env{
283292 Contexts: j.state.Values(env),
284- Funcs: map[string]expr.Func{
285- // hashFiles is a CI-2 feature: the schema allows it in steps, so it
286- // parses, and fails when evaluated.
287- "hashFiles": func(context.Context, []expr.Value) (expr.Value, error) {
288- return nil, errors.New("not supported yet: hashFiles")
289- },
290- },
293+ Funcs: map[string]expr.Func{"hashFiles": j.hashFilesFunc()},
291294 }
292295 }
293296
@@ -372,6 +375,7 @@ func (j *job) setUp() ([]*workflow.Step, error) {
372375 state.WithLayout(layout)
373376 }
374377 j.state = state
378+ j.masker.Store(newMasker(state.Secrets()...))
375379
376380 // Features that arrive in later phases fail the job with a clear message.
377381 switch {
@@ -543,6 +547,11 @@ func (j *job) completeJob(num int) map[string]string {
543547 j.debug(num, fmt.Sprintf("Skip output '%s' since it's empty", p.Key))
544548 continue
545549 }
550+ // An output that holds a secret is dropped (JobExtension.cs:819-824).
551+ if j.masker.Load().has(p.Value) {
552+ j.tagged(num, "##[warning]", fmt.Sprintf("Skip output '%s' since it may contain secret.", p.Key))
553+ continue
554+ }
546555 // ermite refuses outputs above 1 MiB in total (runner.proto,
547556 // CompleteJobRequest.outputs), and they would not fit a frame of the
548557 // stream either: the job fails without outputs.
@@ -642,7 +651,10 @@ func (j *job) jobDisplayName() string {
642651 if err != nil {
643652 return fallback("the name does not evaluate", err)
644653 }
645- return name
654+ // GitHub shows "Display name of", not "Display name of " (recorded:
655+ // matrix-include-exclude, an empty last matrix value). The recordings only
656+ // show trailing spaces; both ends are trimmed, as ermite's server does.
657+ return strings.TrimSpace(name)
646658 }
647659
648660 // comboIndex finds the combination of the strategy that is this job. Without a
@@ -10,6 +10,7 @@ import (
10 "path/filepath"10 "path/filepath"
11 "strings"11 "strings"
12 "sync"12 "sync"
13+ "sync/atomic"
13 "syscall"14 "syscall"
14 "time"15 "time"
15 16
@@ -144,8 +145,13 @@ type job struct {
144 baseEnv []string145 baseEnv []string
145 lookupPath string146 lookupPath string
146 before map[string]bool147 before map[string]bool
148+ // masker hides the secrets in everything reported; it is set once the
149+ // contexts are built, before any step runs.
150+ masker atomic.Pointer[masker]
147 }151 }
148 152
153+func (j *job) mask(s string) string { return j.masker.Load().mask(s) }
154+
149 func (j *job) now() time.Time {155 func (j *job) now() time.Time {
150 if j.cfg.Now != nil {156 if j.cfg.Now != nil {
151 return j.cfg.Now()157 return j.cfg.Now()
@@ -160,6 +166,7 @@ func Run(cancel *Canceller, cfg Config) Result {
160 cancel = NewCanceller()166 cancel = NewCanceller()
161 }167 }
162 j := &job{cfg: cfg, cancel: cancel, trackID: "github_" + uuid.NewString()}168 j := &job{cfg: cfg, cancel: cancel, trackID: "github_" + uuid.NewString()}
169+ j.cfg.Reporter = &maskReporter{j: j, next: cfg.Reporter}
163 defer func() {170 defer func() {
164 if j.fc != nil {171 if j.fc != nil {
165 _ = j.fc.Close()172 _ = j.fc.Close()
@@ -169,7 +176,8 @@ func Run(cancel *Canceller, cfg Config) Result {
169 }176 }
170 }()177 }()
171 res := j.run()178 res := j.run()
172- cfg.Reporter.JobCompleted(res)179+ res.Error = j.mask(res.Error)
180+ j.cfg.Reporter.JobCompleted(res)
173 return res181 return res
174 }182 }
175 183
@@ -253,6 +261,7 @@ func (j *job) out(num int, text string) {
253 261
254 // tagged writes tag+message; only the first line carries the tag.262 // tagged writes tag+message; only the first line carries the tag.
255 func (j *job) tagged(num int, tag, msg string) {263 func (j *job) tagged(num int, tag, msg string) {
264+ msg = j.mask(msg) // before the cut: a secret split by it would leak its start
256 if len(msg) > maxIssueMessageLength && tag != "" {265 if len(msg) > maxIssueMessageLength && tag != "" {
257 msg = msg[:maxIssueMessageLength]266 msg = msg[:maxIssueMessageLength]
258 }267 }
@@ -281,13 +290,7 @@ func (j *job) debug(num int, msg string) {
281 func (j *job) evalEnv(env *ordered) template.Env {290 func (j *job) evalEnv(env *ordered) template.Env {
282 return template.Env{291 return template.Env{
283 Contexts: j.state.Values(env),292 Contexts: j.state.Values(env),
284- Funcs: map[string]expr.Func{293+ Funcs: map[string]expr.Func{"hashFiles": j.hashFilesFunc()},
285- // hashFiles is a CI-2 feature: the schema allows it in steps, so it
286- // parses, and fails when evaluated.
287- "hashFiles": func(context.Context, []expr.Value) (expr.Value, error) {
288- return nil, errors.New("not supported yet: hashFiles")
289- },
290- },
291 }294 }
292 }295 }
293 296
@@ -372,6 +375,7 @@ func (j *job) setUp() ([]*workflow.Step, error) {
372 state.WithLayout(layout)375 state.WithLayout(layout)
373 }376 }
374 j.state = state377 j.state = state
378+ j.masker.Store(newMasker(state.Secrets()...))
375 379
376 // Features that arrive in later phases fail the job with a clear message.380 // Features that arrive in later phases fail the job with a clear message.
377 switch {381 switch {
@@ -543,6 +547,11 @@ func (j *job) completeJob(num int) map[string]string {
543 j.debug(num, fmt.Sprintf("Skip output '%s' since it's empty", p.Key))547 j.debug(num, fmt.Sprintf("Skip output '%s' since it's empty", p.Key))
544 continue548 continue
545 }549 }
550+ // An output that holds a secret is dropped (JobExtension.cs:819-824).
551+ if j.masker.Load().has(p.Value) {
552+ j.tagged(num, "##[warning]", fmt.Sprintf("Skip output '%s' since it may contain secret.", p.Key))
553+ continue
554+ }
546 // ermite refuses outputs above 1 MiB in total (runner.proto,555 // ermite refuses outputs above 1 MiB in total (runner.proto,
547 // CompleteJobRequest.outputs), and they would not fit a frame of the556 // CompleteJobRequest.outputs), and they would not fit a frame of the
548 // stream either: the job fails without outputs.557 // stream either: the job fails without outputs.
@@ -642,7 +651,10 @@ func (j *job) jobDisplayName() string {
642 if err != nil {651 if err != nil {
643 return fallback("the name does not evaluate", err)652 return fallback("the name does not evaluate", err)
644 }653 }
645- return name654+ // GitHub shows "Display name of", not "Display name of " (recorded:
655+ // matrix-include-exclude, an empty last matrix value). The recordings only
656+ // show trailing spaces; both ends are trimmed, as ermite's server does.
657+ return strings.TrimSpace(name)
646 }658 }
647 659
648 // comboIndex finds the combination of the strategy that is this job. Without a660 // comboIndex finds the combination of the strategy that is this job. Without a
A internal/executor/jobname_test.go +32 -0
@@ -0,0 +1,32 @@
1+// SPDX-License-Identifier: MIT
2+
3+package executor
4+
5+import "testing"
6+
7+// Recorded on github.com (matrix-include-exclude): "m two 2", not "m two 2 ".
8+func TestJobNameIsTrimmed(t *testing.T) {
9+ needTools(t, "bash")
10+ yaml := `on: push
11+jobs:
12+ main:
13+ name: m ${{ matrix.os }} ${{ matrix.extra }}
14+ runs-on: ubuntu-24.04
15+ strategy:
16+ matrix:
17+ os: [one]
18+ include:
19+ - os: one
20+ extra: ''
21+ steps:
22+ - run: echo hi
23+`
24+ rec, res := runLocal(t, yaml, "main", func(c *Config) { c.MatrixJSON = `{"os":"one","extra":""}` })
25+ if res.Conclusion != ConclusionSuccess {
26+ t.Fatalf("%+v", res)
27+ }
28+ log := rec.steps[0].Log
29+ if last := log[len(log)-1]; last != "Complete job name: m one" {
30+ t.Errorf("%q", last)
31+ }
32+}
@@ -0,0 +1,32 @@
1+// SPDX-License-Identifier: MIT
2+
3+package executor
4+
5+import "testing"
6+
7+// Recorded on github.com (matrix-include-exclude): "m two 2", not "m two 2 ".
8+func TestJobNameIsTrimmed(t *testing.T) {
9+ needTools(t, "bash")
10+ yaml := `on: push
11+jobs:
12+ main:
13+ name: m ${{ matrix.os }} ${{ matrix.extra }}
14+ runs-on: ubuntu-24.04
15+ strategy:
16+ matrix:
17+ os: [one]
18+ include:
19+ - os: one
20+ extra: ''
21+ steps:
22+ - run: echo hi
23+`
24+ rec, res := runLocal(t, yaml, "main", func(c *Config) { c.MatrixJSON = `{"os":"one","extra":""}` })
25+ if res.Conclusion != ConclusionSuccess {
26+ t.Fatalf("%+v", res)
27+ }
28+ log := rec.steps[0].Log
29+ if last := log[len(log)-1]; last != "Complete job name: m one" {
30+ t.Errorf("%q", last)
31+ }
32+}
A internal/executor/masker.go +100 -0
@@ -0,0 +1,100 @@
1+// SPDX-License-Identifier: MIT
2+
3+package executor
4+
5+import (
6+ "encoding/base64"
7+ "encoding/json"
8+ "net/url"
9+ "slices"
10+ "strings"
11+ "time"
12+)
13+
14+// masker replaces secret values by *** in text that leaves the executor
15+// (HostContext.SecretMasker). A value is masked as is and in the forms the
16+// official runner registers for it (HostContext.cs:103-113, ValueEncoders): its
17+// base64 at the three byte offsets a prefix of one or two bytes can give
18+// ("user:" + secret is base64-encoded as a whole), and its JSON and URI
19+// escapes. Longer forms are replaced first.
20+type masker struct{ values []string }
21+
22+const maskedValue = "***"
23+
24+func newMasker(secrets ...string) *masker {
25+ seen := map[string]bool{}
26+ var vals []string
27+ add := func(v string) {
28+ if v != "" && !seen[v] {
29+ seen[v] = true
30+ vals = append(vals, v)
31+ }
32+ }
33+ for _, s := range secrets {
34+ add(s)
35+ for _, e := range secretEncodings(s) {
36+ add(e)
37+ }
38+ }
39+ slices.SortStableFunc(vals, func(a, b string) int { return len(b) - len(a) })
40+ return &masker{values: vals}
41+}
42+
43+// secretEncodings are the other forms of v that the runner masks.
44+func secretEncodings(v string) []string {
45+ var out []string
46+ b := []byte(v)
47+ for shift := range 3 {
48+ if len(b) > shift {
49+ out = append(out, base64.StdEncoding.EncodeToString(b[shift:]))
50+ } else if shift == 0 {
51+ out = append(out, base64.StdEncoding.EncodeToString(b))
52+ }
53+ }
54+ if j, err := json.Marshal(v); err == nil && len(j) > 2 {
55+ out = append(out, string(j[1:len(j)-1]))
56+ }
57+ out = append(out, url.QueryEscape(v), url.PathEscape(v))
58+ return out
59+}
60+
61+// mask returns s with every secret replaced.
62+func (m *masker) mask(s string) string {
63+ if m == nil {
64+ return s
65+ }
66+ for _, v := range m.values {
67+ if strings.Contains(s, v) {
68+ s = strings.ReplaceAll(s, v, maskedValue)
69+ }
70+ }
71+ return s
72+}
73+
74+// has reports whether s contains a secret.
75+func (m *masker) has(s string) bool { return m != nil && m.mask(s) != s }
76+
77+// maskReporter masks everything the executor reports: the log lines, the names
78+// of the steps and the error of the job. The outputs are checked where they are
79+// evaluated (a job output with a secret is dropped).
80+type maskReporter struct {
81+ j *job
82+ next Reporter
83+}
84+
85+func (r *maskReporter) StepStarted(n int, name string, at time.Time) {
86+ r.next.StepStarted(n, r.j.mask(name), at)
87+}
88+
89+func (r *maskReporter) Log(n int, at time.Time, line string) {
90+ r.next.Log(n, at, r.j.mask(line))
91+}
92+
93+func (r *maskReporter) StepCompleted(n int, name string, c Conclusion, at time.Time) {
94+ r.next.StepCompleted(n, r.j.mask(name), c, at)
95+}
96+
97+func (r *maskReporter) JobCompleted(res Result) {
98+ res.Error = r.j.mask(res.Error)
99+ r.next.JobCompleted(res)
100+}
@@ -0,0 +1,100 @@
1+// SPDX-License-Identifier: MIT
2+
3+package executor
4+
5+import (
6+ "encoding/base64"
7+ "encoding/json"
8+ "net/url"
9+ "slices"
10+ "strings"
11+ "time"
12+)
13+
14+// masker replaces secret values by *** in text that leaves the executor
15+// (HostContext.SecretMasker). A value is masked as is and in the forms the
16+// official runner registers for it (HostContext.cs:103-113, ValueEncoders): its
17+// base64 at the three byte offsets a prefix of one or two bytes can give
18+// ("user:" + secret is base64-encoded as a whole), and its JSON and URI
19+// escapes. Longer forms are replaced first.
20+type masker struct{ values []string }
21+
22+const maskedValue = "***"
23+
24+func newMasker(secrets ...string) *masker {
25+ seen := map[string]bool{}
26+ var vals []string
27+ add := func(v string) {
28+ if v != "" && !seen[v] {
29+ seen[v] = true
30+ vals = append(vals, v)
31+ }
32+ }
33+ for _, s := range secrets {
34+ add(s)
35+ for _, e := range secretEncodings(s) {
36+ add(e)
37+ }
38+ }
39+ slices.SortStableFunc(vals, func(a, b string) int { return len(b) - len(a) })
40+ return &masker{values: vals}
41+}
42+
43+// secretEncodings are the other forms of v that the runner masks.
44+func secretEncodings(v string) []string {
45+ var out []string
46+ b := []byte(v)
47+ for shift := range 3 {
48+ if len(b) > shift {
49+ out = append(out, base64.StdEncoding.EncodeToString(b[shift:]))
50+ } else if shift == 0 {
51+ out = append(out, base64.StdEncoding.EncodeToString(b))
52+ }
53+ }
54+ if j, err := json.Marshal(v); err == nil && len(j) > 2 {
55+ out = append(out, string(j[1:len(j)-1]))
56+ }
57+ out = append(out, url.QueryEscape(v), url.PathEscape(v))
58+ return out
59+}
60+
61+// mask returns s with every secret replaced.
62+func (m *masker) mask(s string) string {
63+ if m == nil {
64+ return s
65+ }
66+ for _, v := range m.values {
67+ if strings.Contains(s, v) {
68+ s = strings.ReplaceAll(s, v, maskedValue)
69+ }
70+ }
71+ return s
72+}
73+
74+// has reports whether s contains a secret.
75+func (m *masker) has(s string) bool { return m != nil && m.mask(s) != s }
76+
77+// maskReporter masks everything the executor reports: the log lines, the names
78+// of the steps and the error of the job. The outputs are checked where they are
79+// evaluated (a job output with a secret is dropped).
80+type maskReporter struct {
81+ j *job
82+ next Reporter
83+}
84+
85+func (r *maskReporter) StepStarted(n int, name string, at time.Time) {
86+ r.next.StepStarted(n, r.j.mask(name), at)
87+}
88+
89+func (r *maskReporter) Log(n int, at time.Time, line string) {
90+ r.next.Log(n, at, r.j.mask(line))
91+}
92+
93+func (r *maskReporter) StepCompleted(n int, name string, c Conclusion, at time.Time) {
94+ r.next.StepCompleted(n, r.j.mask(name), c, at)
95+}
96+
97+func (r *maskReporter) JobCompleted(res Result) {
98+ res.Error = r.j.mask(res.Error)
99+ r.next.JobCompleted(res)
100+}
A internal/executor/masker_test.go +165 -0
@@ -0,0 +1,165 @@
1+// SPDX-License-Identifier: MIT
2+
3+package executor
4+
5+import (
6+ "encoding/base64"
7+ "strings"
8+ "testing"
9+ "time"
10+)
11+
12+const testToken = "ghs_testtoken123"
13+
14+func tokenContexts() string {
15+ return strings.Replace(testGitHubContext("main"), `"github":{`, `"github":{"token":"`+testToken+`",`, 1)
16+}
17+
18+func TestMaskerForms(t *testing.T) {
19+ m := newMasker(testToken)
20+ for _, tc := range []struct{ name, text string }{
21+ {"plain", "x " + testToken + " y"},
22+ {"base64", "Authorization: Basic " + base64.StdEncoding.EncodeToString([]byte(testToken))},
23+ {"base64, one byte of prefix", base64.StdEncoding.EncodeToString([]byte("u" + testToken))},
24+ {"base64, two bytes of prefix", base64.StdEncoding.EncodeToString([]byte("us" + testToken))},
25+ {"base64, three bytes of prefix", base64.StdEncoding.EncodeToString([]byte("usr" + testToken))},
26+ } {
27+ if got := m.mask(tc.text); strings.Contains(got, testToken) || got == tc.text || !strings.Contains(got, "***") {
28+ t.Errorf("%s: %q -> %q", tc.name, tc.text, got)
29+ }
30+ if !m.has(tc.text) {
31+ t.Errorf("%s: has() is false", tc.name)
32+ }
33+ }
34+ // base64 at the three offsets: the secret's own text is found in every form
35+ for shift := 0; shift < 3; shift++ {
36+ prefix := strings.Repeat("p", shift)
37+ enc := base64.StdEncoding.EncodeToString([]byte(prefix + testToken))
38+ if got := m.mask("[" + enc + "]"); strings.Contains(got, enc[4:len(enc)-4]) {
39+ t.Errorf("shift %d: %q", shift, got)
40+ }
41+ }
42+ for _, tc := range []string{"", "nothing here", "ghs_other"} {
43+ if m.has(tc) {
44+ t.Errorf("%q must not be masked", tc)
45+ }
46+ }
47+ if !newMasker().has("") == false || newMasker().mask("a") != "a" {
48+ t.Error("an empty masker changes text")
49+ }
50+ // special characters: the JSON and URL forms
51+ s := newMasker(`a"b/c d`)
52+ for _, text := range []string{`a"b/c d`, `a\"b/c d`, `a%22b%2Fc+d`, `a%22b%2Fc%20d`} {
53+ if got := s.mask(text); strings.Contains(got, "b") {
54+ t.Errorf("%q -> %q", text, got)
55+ }
56+ }
57+}
58+
59+// The name of a step that holds the token is masked before it reaches the
60+// server (StepStarted and StepCompleted).
61+func TestStepNameIsMasked(t *testing.T) {
62+ needTools(t, "bash")
63+ yaml := `on: push
64+jobs:
65+ main:
66+ runs-on: ubuntu-24.04
67+ steps:
68+ - name: leak ${{ github.token }}
69+ run: echo hi
70+ - name: leak2 ${{ secrets.GITHUB_TOKEN }} ${{ env.X }}
71+ env:
72+ X: x
73+ run: echo hi
74+`
75+ rec, res := runLocal(t, yaml, "main", func(c *Config) { c.ContextsJSON = tokenContexts() })
76+ if res.Conclusion != ConclusionSuccess {
77+ t.Fatalf("%+v", res)
78+ }
79+ for _, s := range rec.steps {
80+ if strings.Contains(s.Name, testToken) {
81+ t.Errorf("step name %q holds the token", s.Name)
82+ }
83+ }
84+ rec.step(t, "leak ***")
85+ rec.step(t, "leak2 *** x")
86+}
87+
88+// A job output that holds the token is dropped with GitHub's warning; the
89+// other outputs stay.
90+func TestJobOutputWithTokenIsDropped(t *testing.T) {
91+ needTools(t, "bash")
92+ yaml := `on: push
93+jobs:
94+ main:
95+ runs-on: ubuntu-24.04
96+ outputs:
97+ leak: ${{ steps.s.outputs.v }}
98+ fine: ${{ steps.s.outputs.w }}
99+ steps:
100+ - id: s
101+ env:
102+ T: ${{ github.token }}
103+ run: |
104+ echo "v=prefix-$T-suffix" >> "$GITHUB_OUTPUT"
105+ echo "w=ok" >> "$GITHUB_OUTPUT"
106+`
107+ rec, res := runLocal(t, yaml, "main", func(c *Config) { c.ContextsJSON = tokenContexts() })
108+ if res.Conclusion != ConclusionSuccess {
109+ t.Fatalf("%+v", res)
110+ }
111+ if _, has := res.Outputs["leak"]; has || res.Outputs["fine"] != "ok" {
112+ t.Errorf("outputs %v", res.Outputs)
113+ }
114+ for _, v := range res.Outputs {
115+ if strings.Contains(v, testToken) {
116+ t.Errorf("the token is in an output: %q", v)
117+ }
118+ }
119+ if log := logOf(rec.step(t, CompleteJobName)); !strings.Contains(log, "##[warning]Skip output 'leak' since it may contain secret.") {
120+ t.Errorf("warning missing:\n%s", log)
121+ }
122+}
123+
124+// Everything the executor reports goes through the masker: the names of the
125+// steps, the log, and the error of the job (no real run puts a token there, so
126+// the reporter is exercised directly).
127+func TestMaskReporter(t *testing.T) {
128+ rec := newRecorder()
129+ j := &job{}
130+ j.masker.Store(newMasker(testToken))
131+ r := &maskReporter{j: j, next: rec}
132+ r.StepStarted(1, "name "+testToken, time.Now())
133+ r.Log(1, time.Now(), "line "+testToken)
134+ r.StepCompleted(1, "name "+testToken, ConclusionSuccess, time.Now())
135+ r.JobCompleted(Result{Conclusion: ConclusionFailure, Error: "boom " + testToken})
136+ if s := rec.steps[0]; s.Name != "name ***" || s.Log[0] != "line ***" {
137+ t.Errorf("%+v", s)
138+ }
139+ if rec.result.Error != "boom ***" {
140+ t.Errorf("error %q", rec.result.Error)
141+ }
142+}
143+
144+// The token is masked before a message is cut at 4096 bytes: a token split by
145+// the cut would otherwise leave its start in the line.
146+func TestMaskBeforeTruncation(t *testing.T) {
147+ needTools(t, "bash")
148+ pad := strings.Repeat("a", maxIssueMessageLength-5)
149+ yaml := `on: push
150+jobs:
151+ main:
152+ runs-on: ubuntu-24.04
153+ steps:
154+ - run: |
155+ echo "::error::` + pad + `${{ github.token }}"
156+`
157+ rec, _ := runLocal(t, yaml, "main", func(c *Config) { c.ContextsJSON = tokenContexts() })
158+ for _, s := range rec.steps {
159+ for _, l := range s.Log {
160+ if strings.Contains(l, "ghs_") {
161+ t.Errorf("a piece of the token leaked: %q", l[len(l)-20:])
162+ }
163+ }
164+ }
165+}
@@ -0,0 +1,165 @@
1+// SPDX-License-Identifier: MIT
2+
3+package executor
4+
5+import (
6+ "encoding/base64"
7+ "strings"
8+ "testing"
9+ "time"
10+)
11+
12+const testToken = "ghs_testtoken123"
13+
14+func tokenContexts() string {
15+ return strings.Replace(testGitHubContext("main"), `"github":{`, `"github":{"token":"`+testToken+`",`, 1)
16+}
17+
18+func TestMaskerForms(t *testing.T) {
19+ m := newMasker(testToken)
20+ for _, tc := range []struct{ name, text string }{
21+ {"plain", "x " + testToken + " y"},
22+ {"base64", "Authorization: Basic " + base64.StdEncoding.EncodeToString([]byte(testToken))},
23+ {"base64, one byte of prefix", base64.StdEncoding.EncodeToString([]byte("u" + testToken))},
24+ {"base64, two bytes of prefix", base64.StdEncoding.EncodeToString([]byte("us" + testToken))},
25+ {"base64, three bytes of prefix", base64.StdEncoding.EncodeToString([]byte("usr" + testToken))},
26+ } {
27+ if got := m.mask(tc.text); strings.Contains(got, testToken) || got == tc.text || !strings.Contains(got, "***") {
28+ t.Errorf("%s: %q -> %q", tc.name, tc.text, got)
29+ }
30+ if !m.has(tc.text) {
31+ t.Errorf("%s: has() is false", tc.name)
32+ }
33+ }
34+ // base64 at the three offsets: the secret's own text is found in every form
35+ for shift := 0; shift < 3; shift++ {
36+ prefix := strings.Repeat("p", shift)
37+ enc := base64.StdEncoding.EncodeToString([]byte(prefix + testToken))
38+ if got := m.mask("[" + enc + "]"); strings.Contains(got, enc[4:len(enc)-4]) {
39+ t.Errorf("shift %d: %q", shift, got)
40+ }
41+ }
42+ for _, tc := range []string{"", "nothing here", "ghs_other"} {
43+ if m.has(tc) {
44+ t.Errorf("%q must not be masked", tc)
45+ }
46+ }
47+ if !newMasker().has("") == false || newMasker().mask("a") != "a" {
48+ t.Error("an empty masker changes text")
49+ }
50+ // special characters: the JSON and URL forms
51+ s := newMasker(`a"b/c d`)
52+ for _, text := range []string{`a"b/c d`, `a\"b/c d`, `a%22b%2Fc+d`, `a%22b%2Fc%20d`} {
53+ if got := s.mask(text); strings.Contains(got, "b") {
54+ t.Errorf("%q -> %q", text, got)
55+ }
56+ }
57+}
58+
59+// The name of a step that holds the token is masked before it reaches the
60+// server (StepStarted and StepCompleted).
61+func TestStepNameIsMasked(t *testing.T) {
62+ needTools(t, "bash")
63+ yaml := `on: push
64+jobs:
65+ main:
66+ runs-on: ubuntu-24.04
67+ steps:
68+ - name: leak ${{ github.token }}
69+ run: echo hi
70+ - name: leak2 ${{ secrets.GITHUB_TOKEN }} ${{ env.X }}
71+ env:
72+ X: x
73+ run: echo hi
74+`
75+ rec, res := runLocal(t, yaml, "main", func(c *Config) { c.ContextsJSON = tokenContexts() })
76+ if res.Conclusion != ConclusionSuccess {
77+ t.Fatalf("%+v", res)
78+ }
79+ for _, s := range rec.steps {
80+ if strings.Contains(s.Name, testToken) {
81+ t.Errorf("step name %q holds the token", s.Name)
82+ }
83+ }
84+ rec.step(t, "leak ***")
85+ rec.step(t, "leak2 *** x")
86+}
87+
88+// A job output that holds the token is dropped with GitHub's warning; the
89+// other outputs stay.
90+func TestJobOutputWithTokenIsDropped(t *testing.T) {
91+ needTools(t, "bash")
92+ yaml := `on: push
93+jobs:
94+ main:
95+ runs-on: ubuntu-24.04
96+ outputs:
97+ leak: ${{ steps.s.outputs.v }}
98+ fine: ${{ steps.s.outputs.w }}
99+ steps:
100+ - id: s
101+ env:
102+ T: ${{ github.token }}
103+ run: |
104+ echo "v=prefix-$T-suffix" >> "$GITHUB_OUTPUT"
105+ echo "w=ok" >> "$GITHUB_OUTPUT"
106+`
107+ rec, res := runLocal(t, yaml, "main", func(c *Config) { c.ContextsJSON = tokenContexts() })
108+ if res.Conclusion != ConclusionSuccess {
109+ t.Fatalf("%+v", res)
110+ }
111+ if _, has := res.Outputs["leak"]; has || res.Outputs["fine"] != "ok" {
112+ t.Errorf("outputs %v", res.Outputs)
113+ }
114+ for _, v := range res.Outputs {
115+ if strings.Contains(v, testToken) {
116+ t.Errorf("the token is in an output: %q", v)
117+ }
118+ }
119+ if log := logOf(rec.step(t, CompleteJobName)); !strings.Contains(log, "##[warning]Skip output 'leak' since it may contain secret.") {
120+ t.Errorf("warning missing:\n%s", log)
121+ }
122+}
123+
124+// Everything the executor reports goes through the masker: the names of the
125+// steps, the log, and the error of the job (no real run puts a token there, so
126+// the reporter is exercised directly).
127+func TestMaskReporter(t *testing.T) {
128+ rec := newRecorder()
129+ j := &job{}
130+ j.masker.Store(newMasker(testToken))
131+ r := &maskReporter{j: j, next: rec}
132+ r.StepStarted(1, "name "+testToken, time.Now())
133+ r.Log(1, time.Now(), "line "+testToken)
134+ r.StepCompleted(1, "name "+testToken, ConclusionSuccess, time.Now())
135+ r.JobCompleted(Result{Conclusion: ConclusionFailure, Error: "boom " + testToken})
136+ if s := rec.steps[0]; s.Name != "name ***" || s.Log[0] != "line ***" {
137+ t.Errorf("%+v", s)
138+ }
139+ if rec.result.Error != "boom ***" {
140+ t.Errorf("error %q", rec.result.Error)
141+ }
142+}
143+
144+// The token is masked before a message is cut at 4096 bytes: a token split by
145+// the cut would otherwise leave its start in the line.
146+func TestMaskBeforeTruncation(t *testing.T) {
147+ needTools(t, "bash")
148+ pad := strings.Repeat("a", maxIssueMessageLength-5)
149+ yaml := `on: push
150+jobs:
151+ main:
152+ runs-on: ubuntu-24.04
153+ steps:
154+ - run: |
155+ echo "::error::` + pad + `${{ github.token }}"
156+`
157+ rec, _ := runLocal(t, yaml, "main", func(c *Config) { c.ContextsJSON = tokenContexts() })
158+ for _, s := range rec.steps {
159+ for _, l := range s.Log {
160+ if strings.Contains(l, "ghs_") {
161+ t.Errorf("a piece of the token leaked: %q", l[len(l)-20:])
162+ }
163+ }
164+ }
165+}
A internal/executor/names_test.go +41 -0
@@ -0,0 +1,41 @@
1+// SPDX-License-Identifier: MIT
2+
3+package executor
4+
5+import (
6+ "strings"
7+ "testing"
8+)
9+
10+// Recorded on github.com (ctx-github-names): names that only read the
11+// contexts of the job message are evaluated when the job starts, with
12+// github.job empty, while the script of the step runs with the real job key.
13+func TestNamesSeeAnEmptyGithubJob(t *testing.T) {
14+ needTools(t, "bash")
15+ yaml := `on: push
16+jobs:
17+ first-job:
18+ name: Display name of ${{ github.job }}
19+ runs-on: ubuntu-24.04
20+ steps:
21+ - run: echo "job=${{ github.job }}"
22+ - name: late ${{ github.job }} ${{ env.X }}
23+ env:
24+ X: x
25+ run: echo hi
26+`
27+ rec, res := runLocal(t, yaml, "first-job", nil)
28+ if res.Conclusion != ConclusionSuccess {
29+ t.Fatalf("%+v %v", res, rec.names())
30+ }
31+ if got := rec.steps[0].Log; got[len(got)-1] != "Complete job name: Display name of" {
32+ t.Errorf("job name: %v", got)
33+ }
34+ s := rec.step(t, `Run echo "job="`)
35+ if !strings.Contains(logOf(s), `##[group]Run echo "job=first-job"`) || !strings.Contains(logOf(s), "\njob=first-job") {
36+ t.Errorf("the script must see the real job key:\n%s", logOf(s))
37+ }
38+ // A name that reads env is not available at job start: evaluated with the
39+ // step's contexts, where github.job is the key.
40+ rec.step(t, "late first-job x")
41+}
@@ -0,0 +1,41 @@
1+// SPDX-License-Identifier: MIT
2+
3+package executor
4+
5+import (
6+ "strings"
7+ "testing"
8+)
9+
10+// Recorded on github.com (ctx-github-names): names that only read the
11+// contexts of the job message are evaluated when the job starts, with
12+// github.job empty, while the script of the step runs with the real job key.
13+func TestNamesSeeAnEmptyGithubJob(t *testing.T) {
14+ needTools(t, "bash")
15+ yaml := `on: push
16+jobs:
17+ first-job:
18+ name: Display name of ${{ github.job }}
19+ runs-on: ubuntu-24.04
20+ steps:
21+ - run: echo "job=${{ github.job }}"
22+ - name: late ${{ github.job }} ${{ env.X }}
23+ env:
24+ X: x
25+ run: echo hi
26+`
27+ rec, res := runLocal(t, yaml, "first-job", nil)
28+ if res.Conclusion != ConclusionSuccess {
29+ t.Fatalf("%+v %v", res, rec.names())
30+ }
31+ if got := rec.steps[0].Log; got[len(got)-1] != "Complete job name: Display name of" {
32+ t.Errorf("job name: %v", got)
33+ }
34+ s := rec.step(t, `Run echo "job="`)
35+ if !strings.Contains(logOf(s), `##[group]Run echo "job=first-job"`) || !strings.Contains(logOf(s), "\njob=first-job") {
36+ t.Errorf("the script must see the real job key:\n%s", logOf(s))
37+ }
38+ // A name that reads env is not available at job start: evaluated with the
39+ // step's contexts, where github.job is the key.
40+ rec.step(t, "late first-job x")
41+}
M internal/executor/step.go +99 -3
@@ -4,10 +4,12 @@ package executor
44
55 import (
66 "context"
7+ "errors"
78 "fmt"
89 "strings"
910 "sync"
1011
12+ "bnema.dev/ermite-actions/expr"
1113 "bnema.dev/ermite-actions/template"
1214 "bnema.dev/ermite-actions/workflow"
1315 )
@@ -83,13 +85,107 @@ func (j *job) displayName(st *workflow.Step, env *ordered) string {
8385 default:
8486 return ""
8587 }
86- name, err := j.evalString(tok, env)
88+ // JobExtension evaluates the display names once, when the job starts, with
89+ // the contexts the server sent (github.job is empty there) and without the
90+ // step functions; ActionRunner.GenerateDisplayName only does it when every
91+ // context and function the name uses is available (CheckHasRequiredContext).
92+ // A name that is fully evaluated then stays as it is (recorded:
93+ // ctx-github-names, "Run echo \"job=\""); the others are evaluated again,
94+ // with the contexts of the step, when the step starts.
95+ var name string
96+ var err error
97+ if serverEvaluable(tok) {
98+ j.mu.Lock()
99+ sc := j.state.ServerContexts()
100+ j.mu.Unlock()
101+ name, err = template.EvaluateString(context.Background(), tok, template.Env{Contexts: sc})
102+ } else {
103+ name, err = j.evalString(tok, env)
104+ }
87105 if err != nil {
88106 name = tokenSource(tok)
89107 }
90108 return formatStepName(prefix, name)
91109 }
92110
111+// serverContextNames are the contexts of the job message (ContextData) when
112+// display names are first evaluated.
113+var serverContextNames = map[string]bool{"github": true, "needs": true, "strategy": true, "matrix": true, "inputs": true, "vars": true}
114+
115+// wellKnownFuncs are the functions available without the step functions.
116+var wellKnownFuncs = map[string]bool{"case": true, "contains": true, "endswith": true, "format": true, "join": true, "startswith": true, "tojson": true, "fromjson": true}
117+
118+// serverEvaluable is TemplateTokenExtensions.CheckHasRequiredContext for the
119+// first evaluation of a display name: every expression of the token only
120+// reads contexts of the job message and calls well-known functions.
121+func serverEvaluable(tok template.Token) bool {
122+ ok := true
123+ template.Walk(tok, func(t template.Token) bool {
124+ if be, isExpr := t.(*template.BasicExpression); isExpr {
125+ ok = ok && nodeServerEvaluable(be.Node())
126+ }
127+ return ok
128+ })
129+ return ok
130+}
131+
132+func nodeServerEvaluable(n expr.Node) bool {
133+ switch n := n.(type) {
134+ case *expr.ContextAccess:
135+ return serverContextNames[strings.ToLower(n.Name.Lexeme)]
136+ case *expr.Unary:
137+ return nodeServerEvaluable(n.Expr)
138+ case *expr.Binary:
139+ return nodeServerEvaluable(n.Left) && nodeServerEvaluable(n.Right)
140+ case *expr.Grouping:
141+ return nodeServerEvaluable(n.Group)
142+ case *expr.IndexAccess:
143+ return nodeServerEvaluable(n.Expr) && nodeServerEvaluable(n.Index)
144+ case *expr.Logical:
145+ for _, a := range n.Args {
146+ if !nodeServerEvaluable(a) {
147+ return false
148+ }
149+ }
150+ case *expr.FunctionCall:
151+ if !wellKnownFuncs[strings.ToLower(n.Name.Lexeme)] {
152+ return false
153+ }
154+ for _, a := range n.Args {
155+ if !nodeServerEvaluable(a) {
156+ return false
157+ }
158+ }
159+ }
160+ return true
161+}
162+
163+// exceptionClasser is implemented by the errors of package expr that stand for
164+// a .NET exception (expr.JSONError).
165+type exceptionClasser interface{ ExceptionClass() string }
166+
167+// templateNotValid is the message of the TemplateValidationException the
168+// runner throws when the evaluation of a template fails
169+// (TemplateStrings.TemplateNotValidWithErrors): "The template is not valid. "
170+// and the errors. With a position the error reads "file (Line: l, Col: c):
171+// message" (recorded: expr-runtime-errors, step env and run). A step condition
172+// has no position: the error is the exception itself, "Class: message"
173+// (withLocation is false).
174+func templateNotValid(err error, withLocation bool) string {
175+ var te *template.Error
176+ if !errors.As(err, &te) {
177+ return err.Error()
178+ }
179+ if withLocation {
180+ return "The template is not valid. " + te.Error()
181+ }
182+ var c exceptionClasser
183+ if errors.As(err, &c) {
184+ return "The template is not valid. " + c.ExceptionClass() + ": " + te.Raw
185+ }
186+ return te.Error()
187+}
188+
93189 func tokenSource(t template.Token) string {
94190 if be, ok := t.(*template.BasicExpression); ok && be.Source != "" {
95191 return be.Source
@@ -165,7 +261,7 @@ func (j *job) runStep(num int, st *workflow.Step) {
165261 }
166262
167263 if envErr != nil {
168- j.errorf(num, "%s", envErr.Error())
264+ j.errorf(num, "%s", templateNotValid(envErr, true))
169265 complete(resFailed)
170266 return
171267 }
@@ -195,7 +291,7 @@ func (j *job) runStep(num int, st *workflow.Step) {
195291 ok, err := cond()
196292 switch {
197293 case err != nil:
198- j.errorf(num, "%s", err.Error())
294+ j.errorf(num, "%s", templateNotValid(err, false))
199295 complete(resFailed)
200296 return
201297 case !ok:
@@ -4,10 +4,12 @@ package executor
4 4
5 import (5 import (
6 "context"6 "context"
7+ "errors"
7 "fmt"8 "fmt"
8 "strings"9 "strings"
9 "sync"10 "sync"
10 11
12+ "bnema.dev/ermite-actions/expr"
11 "bnema.dev/ermite-actions/template"13 "bnema.dev/ermite-actions/template"
12 "bnema.dev/ermite-actions/workflow"14 "bnema.dev/ermite-actions/workflow"
13 )15 )
@@ -83,13 +85,107 @@ func (j *job) displayName(st *workflow.Step, env *ordered) string {
83 default:85 default:
84 return ""86 return ""
85 }87 }
86- name, err := j.evalString(tok, env)88+ // JobExtension evaluates the display names once, when the job starts, with
89+ // the contexts the server sent (github.job is empty there) and without the
90+ // step functions; ActionRunner.GenerateDisplayName only does it when every
91+ // context and function the name uses is available (CheckHasRequiredContext).
92+ // A name that is fully evaluated then stays as it is (recorded:
93+ // ctx-github-names, "Run echo \"job=\""); the others are evaluated again,
94+ // with the contexts of the step, when the step starts.
95+ var name string
96+ var err error
97+ if serverEvaluable(tok) {
98+ j.mu.Lock()
99+ sc := j.state.ServerContexts()
100+ j.mu.Unlock()
101+ name, err = template.EvaluateString(context.Background(), tok, template.Env{Contexts: sc})
102+ } else {
103+ name, err = j.evalString(tok, env)
104+ }
87 if err != nil {105 if err != nil {
88 name = tokenSource(tok)106 name = tokenSource(tok)
89 }107 }
90 return formatStepName(prefix, name)108 return formatStepName(prefix, name)
91 }109 }
92 110
111+// serverContextNames are the contexts of the job message (ContextData) when
112+// display names are first evaluated.
113+var serverContextNames = map[string]bool{"github": true, "needs": true, "strategy": true, "matrix": true, "inputs": true, "vars": true}
114+
115+// wellKnownFuncs are the functions available without the step functions.
116+var wellKnownFuncs = map[string]bool{"case": true, "contains": true, "endswith": true, "format": true, "join": true, "startswith": true, "tojson": true, "fromjson": true}
117+
118+// serverEvaluable is TemplateTokenExtensions.CheckHasRequiredContext for the
119+// first evaluation of a display name: every expression of the token only
120+// reads contexts of the job message and calls well-known functions.
121+func serverEvaluable(tok template.Token) bool {
122+ ok := true
123+ template.Walk(tok, func(t template.Token) bool {
124+ if be, isExpr := t.(*template.BasicExpression); isExpr {
125+ ok = ok && nodeServerEvaluable(be.Node())
126+ }
127+ return ok
128+ })
129+ return ok
130+}
131+
132+func nodeServerEvaluable(n expr.Node) bool {
133+ switch n := n.(type) {
134+ case *expr.ContextAccess:
135+ return serverContextNames[strings.ToLower(n.Name.Lexeme)]
136+ case *expr.Unary:
137+ return nodeServerEvaluable(n.Expr)
138+ case *expr.Binary:
139+ return nodeServerEvaluable(n.Left) && nodeServerEvaluable(n.Right)
140+ case *expr.Grouping:
141+ return nodeServerEvaluable(n.Group)
142+ case *expr.IndexAccess:
143+ return nodeServerEvaluable(n.Expr) && nodeServerEvaluable(n.Index)
144+ case *expr.Logical:
145+ for _, a := range n.Args {
146+ if !nodeServerEvaluable(a) {
147+ return false
148+ }
149+ }
150+ case *expr.FunctionCall:
151+ if !wellKnownFuncs[strings.ToLower(n.Name.Lexeme)] {
152+ return false
153+ }
154+ for _, a := range n.Args {
155+ if !nodeServerEvaluable(a) {
156+ return false
157+ }
158+ }
159+ }
160+ return true
161+}
162+
163+// exceptionClasser is implemented by the errors of package expr that stand for
164+// a .NET exception (expr.JSONError).
165+type exceptionClasser interface{ ExceptionClass() string }
166+
167+// templateNotValid is the message of the TemplateValidationException the
168+// runner throws when the evaluation of a template fails
169+// (TemplateStrings.TemplateNotValidWithErrors): "The template is not valid. "
170+// and the errors. With a position the error reads "file (Line: l, Col: c):
171+// message" (recorded: expr-runtime-errors, step env and run). A step condition
172+// has no position: the error is the exception itself, "Class: message"
173+// (withLocation is false).
174+func templateNotValid(err error, withLocation bool) string {
175+ var te *template.Error
176+ if !errors.As(err, &te) {
177+ return err.Error()
178+ }
179+ if withLocation {
180+ return "The template is not valid. " + te.Error()
181+ }
182+ var c exceptionClasser
183+ if errors.As(err, &c) {
184+ return "The template is not valid. " + c.ExceptionClass() + ": " + te.Raw
185+ }
186+ return te.Error()
187+}
188+
93 func tokenSource(t template.Token) string {189 func tokenSource(t template.Token) string {
94 if be, ok := t.(*template.BasicExpression); ok && be.Source != "" {190 if be, ok := t.(*template.BasicExpression); ok && be.Source != "" {
95 return be.Source191 return be.Source
@@ -165,7 +261,7 @@ func (j *job) runStep(num int, st *workflow.Step) {
165 }261 }
166 262
167 if envErr != nil {263 if envErr != nil {
168- j.errorf(num, "%s", envErr.Error())264+ j.errorf(num, "%s", templateNotValid(envErr, true))
169 complete(resFailed)265 complete(resFailed)
170 return266 return
171 }267 }
@@ -195,7 +291,7 @@ func (j *job) runStep(num int, st *workflow.Step) {
195 ok, err := cond()291 ok, err := cond()
196 switch {292 switch {
197 case err != nil:293 case err != nil:
198- j.errorf(num, "%s", err.Error())294+ j.errorf(num, "%s", templateNotValid(err, false))
199 complete(resFailed)295 complete(resFailed)
200 return296 return
201 case !ok:297 case !ok:
A internal/executor/token_test.go +57 -0
@@ -0,0 +1,57 @@
1+// SPDX-License-Identifier: MIT
2+
3+package executor
4+
5+import (
6+ "encoding/base64"
7+ "strings"
8+ "testing"
9+)
10+
11+// Recorded on github.com (ctx-inputs-vars): secrets.GITHUB_TOKEN is set and
12+// equals github.token. The token comes from the github context the server
13+// sends; it is masked in the log.
14+func TestSecretsHoldTheGithubToken(t *testing.T) {
15+ needTools(t, "bash")
16+ const tok = "ghs_testtoken123"
17+ yaml := `on: push
18+jobs:
19+ main:
20+ runs-on: ubuntu-24.04
21+ steps:
22+ - name: absent values
23+ env:
24+ V_SET: ${{ secrets.GITHUB_TOKEN != '' }}
25+ V_EQ: ${{ secrets.GITHUB_TOKEN == github.token }}
26+ V_NONE: ${{ secrets.NOT_DEFINED }}
27+ V_TOKEN: ${{ secrets.GITHUB_TOKEN }}
28+ run: |
29+ echo "V_SET=$V_SET V_EQ=$V_EQ V_NONE=$V_NONE"
30+ echo "token=$V_TOKEN b64=` + base64.StdEncoding.EncodeToString([]byte("x-access-token:"+tok)) + `"
31+`
32+ rec, res := runLocal(t, yaml, "main", func(c *Config) {
33+ c.ContextsJSON = strings.Replace(testGitHubContext("main"), `"github":{`, `"github":{"token":"`+tok+`",`, 1)
34+ })
35+ if res.Conclusion != ConclusionSuccess {
36+ t.Fatalf("%+v", res)
37+ }
38+ log := logOf(rec.step(t, "absent values"))
39+ if !strings.Contains(log, "V_SET=true V_EQ=true V_NONE=\n") {
40+ t.Errorf("log:\n%s", log)
41+ }
42+ if strings.Contains(log, tok) || strings.Contains(log, "eC1hY2Nlc3MtdG9rZW46") {
43+ t.Errorf("the token leaked into the log:\n%s", log)
44+ }
45+ if !strings.Contains(log, " V_TOKEN: ***") || !strings.Contains(log, "token=*** b64=***") {
46+ t.Errorf("the token must be masked:\n%s", log)
47+ }
48+}
49+
50+// Without a token from the server the secrets context is empty and nothing is
51+// masked.
52+func TestSecretsWithoutToken(t *testing.T) {
53+ s := newTestState(t)
54+ if d := s.Values(nil)["secrets"]; d == nil || len(s.Secrets()) != 0 {
55+ t.Fatalf("%v %v", d, s.Secrets())
56+ }
57+}
@@ -0,0 +1,57 @@
1+// SPDX-License-Identifier: MIT
2+
3+package executor
4+
5+import (
6+ "encoding/base64"
7+ "strings"
8+ "testing"
9+)
10+
11+// Recorded on github.com (ctx-inputs-vars): secrets.GITHUB_TOKEN is set and
12+// equals github.token. The token comes from the github context the server
13+// sends; it is masked in the log.
14+func TestSecretsHoldTheGithubToken(t *testing.T) {
15+ needTools(t, "bash")
16+ const tok = "ghs_testtoken123"
17+ yaml := `on: push
18+jobs:
19+ main:
20+ runs-on: ubuntu-24.04
21+ steps:
22+ - name: absent values
23+ env:
24+ V_SET: ${{ secrets.GITHUB_TOKEN != '' }}
25+ V_EQ: ${{ secrets.GITHUB_TOKEN == github.token }}
26+ V_NONE: ${{ secrets.NOT_DEFINED }}
27+ V_TOKEN: ${{ secrets.GITHUB_TOKEN }}
28+ run: |
29+ echo "V_SET=$V_SET V_EQ=$V_EQ V_NONE=$V_NONE"
30+ echo "token=$V_TOKEN b64=` + base64.StdEncoding.EncodeToString([]byte("x-access-token:"+tok)) + `"
31+`
32+ rec, res := runLocal(t, yaml, "main", func(c *Config) {
33+ c.ContextsJSON = strings.Replace(testGitHubContext("main"), `"github":{`, `"github":{"token":"`+tok+`",`, 1)
34+ })
35+ if res.Conclusion != ConclusionSuccess {
36+ t.Fatalf("%+v", res)
37+ }
38+ log := logOf(rec.step(t, "absent values"))
39+ if !strings.Contains(log, "V_SET=true V_EQ=true V_NONE=\n") {
40+ t.Errorf("log:\n%s", log)
41+ }
42+ if strings.Contains(log, tok) || strings.Contains(log, "eC1hY2Nlc3MtdG9rZW46") {
43+ t.Errorf("the token leaked into the log:\n%s", log)
44+ }
45+ if !strings.Contains(log, " V_TOKEN: ***") || !strings.Contains(log, "token=*** b64=***") {
46+ t.Errorf("the token must be masked:\n%s", log)
47+ }
48+}
49+
50+// Without a token from the server the secrets context is empty and nothing is
51+// masked.
52+func TestSecretsWithoutToken(t *testing.T) {
53+ s := newTestState(t)
54+ if d := s.Values(nil)["secrets"]; d == nil || len(s.Secrets()) != 0 {
55+ t.Fatalf("%v %v", d, s.Secrets())
56+ }
57+}

Keyboard shortcuts

Single-key shortcuts are on. ctrl k and esc work whatever this says.

ctrl k :
go to a section, branch, tag, file or repository
?
this list
j k
move the row cursor down / up
enter
open the row under the cursor; on Files, a directory opens or closes in place
↑ ↓ ← → home end
in the file tree (focus on a row): previous / next row, close / open a directory or go to its parent / first entry, first / last row. They never open a file
l h
on Files: open a directory or a file / close a directory or go up to its parent, then to the parent page. Elsewhere: into the side panel (on a merge request, its files) / back to the content; there j k move between its controls
g g G
first / last row
/
filter the list; on Files, find a file anywhere in the repository (esc clears, enter opens the best match)
b
switch branch or tag
g s g f g c g b g m g i g r g a
tabs: summary, files, commits, branches & tags, merge requests, issues, releases, actions (when available)
g h
home
w
wide mode: use the whole width of the screen, or not (remembered)
y
copy a permalink pinned to the commit
d u x
diff: split / unified / collapse or expand all
e E
diff: the whole file in view / every file, with the lines the hunks hide
n
merge request: next note in the diff
esc
close a dialog, leave a field

Branches 4

Tags 11

↑↓ moveenter openesc close