1// SPDX-License-Identifier: MIT
2
3package executor
4
5import (
6 "encoding/base64"
7 "fmt"
8 "path/filepath"
9 "runtime"
10 "slices"
11 "strings"
12
13 "bnema.dev/ermite-actions/expr"
14)
15
16// ordered is a string map that keeps the position of a key when it is set
17// again, like the Dictionary<string,string> the official runner uses for
18// Global.EnvironmentVariables and the step environment (the order shows in the
19// "env:" block of the log). Keys are case-sensitive, as on Linux
20// (VarUtil.EnvironmentVariableKeyComparer).
21type ordered struct {
22 keys []string
23 vals map[string]string
24}
25
26func newOrdered() *ordered { return &ordered{vals: map[string]string{}} }
27
28func (o *ordered) Set(k, v string) {
29 if _, ok := o.vals[k]; !ok {
30 o.keys = append(o.keys, k)
31 }
32 o.vals[k] = v
33}
34
35func (o *ordered) Get(k string) (string, bool) { v, ok := o.vals[k]; return v, ok }
36func (o *ordered) Remove(k string) {
37 if _, ok := o.vals[k]; !ok {
38 return
39 }
40 delete(o.vals, k)
41 o.keys = slices.DeleteFunc(o.keys, func(x string) bool { return x == k })
42}
43func (o *ordered) Len() int { return len(o.keys) }
44func (o *ordered) Keys() []string { return slices.Clone(o.keys) }
45
46func (o *ordered) Clone() *ordered {
47 c := newOrdered()
48 for _, k := range o.keys {
49 c.Set(k, o.vals[k])
50 }
51 return c
52}
53
54// Dict is the value of the env context. The context of the official runner is
55// case-sensitive on Linux (CaseSensitiveDictionaryContextData); package expr
56// has only case-insensitive dictionaries, so two names that differ by case
57// collapse to the first in expressions (they stay distinct in the process
58// environment). A known deviation.
59func (o *ordered) Dict() *expr.Dictionary {
60 d := expr.NewDictionary()
61 for _, k := range o.keys {
62 d.Add(k, expr.String(o.vals[k]))
63 }
64 return d
65}
66
67// setKey returns d with key set to v: the value is replaced in place when the
68// key (compared case-insensitively) exists, else appended.
69func setKey(d *expr.Dictionary, key string, v expr.Value) *expr.Dictionary {
70 out := expr.NewDictionary()
71 done := false
72 for _, p := range d.Pairs() {
73 if strings.EqualFold(p.Key, key) {
74 if !done {
75 out.Add(p.Key, v)
76 done = true
77 }
78 continue
79 }
80 out.Add(p.Key, p.Value)
81 }
82 if !done {
83 out.Add(key, v)
84 }
85 return out
86}
87
88// setKeySorted is setKey for the github context: a key that is not there yet
89// goes before the first key that sorts after it (bytewise; "token" stays first)
90// instead of at the end, so that the keys the runner adds (workspace,
91// secret_source, the file commands...) sit where the ermite-conformance
92// recordings list them. Note that the recordings are normalised with sorted
93// keys, so they show the sorted order, not necessarily the insertion order of
94// GitHub's runner (SetGitHubContext appends). Keys that exist are replaced in
95// place.
96func setKeySorted(d *expr.Dictionary, key string, v expr.Value) *expr.Dictionary {
97 for _, p := range d.Pairs() {
98 if strings.EqualFold(p.Key, key) {
99 return setKey(d, key, v)
100 }
101 }
102 out := expr.NewDictionary()
103 done := false
104 for _, p := range d.Pairs() {
105 if !done && p.Key > key && !strings.EqualFold(p.Key, "token") {
106 out.Add(key, v)
107 done = true
108 }
109 out.Add(p.Key, p.Value)
110 }
111 if !done {
112 out.Add(key, v)
113 }
114 return out
115}
116
117func str(d *expr.Dictionary, key string) string {
118 if v, ok := d.Get(key); ok {
119 if s, ok := v.(expr.String); ok {
120 return string(s)
121 }
122 }
123 return ""
124}
125
126// Layout is the directory layout of a job, like a hosted runner's
127// (/home/runner/work/<repo>/<repo>, RUNNER_TEMP, RUNNER_TOOL_CACHE).
128type Layout struct {
129 Home string // HOME of the runner user
130 // WorkRoot is the _work directory: <Home>/work.
131 WorkRoot string
132 // Temp is RUNNER_TEMP: <WorkRoot>/_temp.
133 Temp string
134 // ToolCache is RUNNER_TOOL_CACHE.
135 ToolCache string
136 // RunnerWorkspace is RUNNER_WORKSPACE: <WorkRoot>/<repo>.
137 RunnerWorkspace string
138 // Workspace is GITHUB_WORKSPACE: <WorkRoot>/<repo>/<repo>.
139 Workspace string
140 // Actions is where the actions of the job are downloaded: <WorkRoot>/_actions.
141 Actions string
142}
143
144// DefaultToolCache is RUNNER_TOOL_CACHE on the ermite image.
145const DefaultToolCache = "/opt/hostedtoolcache"
146
147// NewLayout lays out a job of the repository repo (its name, without owner)
148// under home. An empty toolCache means DefaultToolCache.
149func NewLayout(home, repo, toolCache string) Layout {
150 if toolCache == "" {
151 toolCache = DefaultToolCache
152 }
153 work := filepath.Join(home, "work")
154 return Layout{
155 Home: home, WorkRoot: work, Temp: filepath.Join(work, "_temp"), ToolCache: toolCache,
156 RunnerWorkspace: filepath.Join(work, repo), Workspace: filepath.Join(work, repo, repo),
157 Actions: filepath.Join(work, "_actions"),
158 }
159}
160
161// RunnerInfo describes the runner for the runner context.
162type RunnerInfo struct {
163 Name string
164 // Environment is "self-hosted" (the default) or "github-hosted".
165 Environment string
166 // Arch is "X64" or "ARM64"; empty means the architecture of this process.
167 Arch string
168}
169
170// githubHosted reports whether the runner.environment is "github-hosted". An
171// empty Environment is "self-hosted", the default of the official runner's
172// runner context.
173func (r RunnerInfo) githubHosted() bool { return r.Environment == "github-hosted" }
174
175func (r RunnerInfo) arch() string {
176 if r.Arch != "" {
177 return r.Arch
178 }
179 switch runtime.GOARCH {
180 case "amd64":
181 return "X64"
182 case "arm64":
183 return "ARM64"
184 case "arm":
185 return "ARM"
186 }
187 return "X86"
188}
189
190// repoName returns the repository name of "owner/name".
191func repoName(full string) string {
192 if i := strings.LastIndexByte(full, '/'); i >= 0 {
193 return full[i+1:]
194 }
195 return full
196}
197
198// stepCtx is one entry of the steps context.
199type stepCtx struct {
200 outputs *ordered
201 outcome string // "" until the step completes
202 concl string
203}
204
205// State holds the contexts of a job as the official runner does
206// (ExecutionContext.ExpressionValues): the contexts the server computed, and
207// github, runner, job, env and steps, which the runner owns.
208type State struct {
209 layout Layout
210 github *expr.Dictionary
211 // serverGitHub is the github context as the server sent it, before the
212 // runner added its keys (token, workspace, the file commands, secret_source,
213 // action...): the contexts that the server itself evaluates the job name
214 // with (see ServerContexts).
215 serverGitHub *expr.Dictionary
216 runner *expr.Dictionary
217 job *expr.Dictionary
218 // other holds needs, strategy, matrix, inputs, vars: copied from the server.
219 other map[string]expr.Value
220 // env is Global.EnvironmentVariables: the job-level environment and what
221 // GITHUB_ENV adds. A step's own env is not in it.
222 env *ordered
223 // root is the steps context of the job; cur is the one in use: a composite
224 // action's steps see their own (see PushScope).
225 root, cur *stepsScope
226 scopes map[string]*stepsScope
227 // prepend is Global.PrependPath in the order of the GITHUB_PATH entries.
228 prepend []string
229 // status is job.status: "success", "failure" or "cancelled".
230 status string
231}
232
233// serverContexts are the keys of the contexts JSON that the runner reads.
234var serverContexts = []string{"needs", "strategy", "matrix", "inputs", "vars"}
235
236// NewState builds the contexts of a job from the JSON the server sent (the
237// "contexts" field of the job) and the matrix JSON, with the context
238// initialisation of JobRunner.cs:142-177 and ExecutionContext.InitializeJob
239// (ExecutionContext.cs:1001-1056).
240func NewState(contextsJSON, matrixJSON, jobKey string, r RunnerInfo, repoFallback string) (*State, Layout, error) {
241 root := expr.NewDictionary()
242 if strings.TrimSpace(contextsJSON) != "" {
243 v, err := expr.FromJSON([]byte(contextsJSON))
244 if err != nil {
245 return nil, Layout{}, fmt.Errorf("job contexts: %w", err)
246 }
247 d, ok := v.(*expr.Dictionary)
248 if !ok {
249 return nil, Layout{}, fmt.Errorf("job contexts: want a JSON object")
250 }
251 root = d
252 }
253 s := &State{other: map[string]expr.Value{}, env: newOrdered(), status: "success", scopes: map[string]*stepsScope{}}
254 s.root = newScope()
255 s.cur = s.root
256
257 // github: token first, then what the server gave (ExecutionContext.cs:1023-1037).
258 // The token is the one the server sent in github.token (GitHub's
259 // system.github.token variable), empty when it sent none.
260 gh := expr.NewDictionary()
261 gh.Add("token", expr.String(""))
262 if v, ok := root.Get("github"); ok {
263 d, ok := v.(*expr.Dictionary)
264 if !ok {
265 return nil, Layout{}, fmt.Errorf("job contexts: github must be an object")
266 }
267 for _, p := range d.Pairs() {
268 if strings.EqualFold(p.Key, "token") {
269 if t, isStr := p.Value.(expr.String); isStr {
270 gh = setKey(gh, "token", t)
271 }
272 continue
273 }
274 gh.Add(p.Key, p.Value)
275 }
276 }
277 s.serverGitHub = expr.NewDictionary()
278 if v, ok := root.Get("github"); ok {
279 s.serverGitHub = v.(*expr.Dictionary)
280 // GitHub evaluates names with github.job empty (recorded:
281 // ctx-github-names): the job key is only added to the contexts the
282 // job runs with.
283 if _, has := s.serverGitHub.Get("job"); has {
284 s.serverGitHub = setKey(s.serverGitHub, "job", expr.String(""))
285 }
286 }
287 if str(gh, "job") == "" && jobKey != "" {
288 gh = setKeySorted(gh, "job", expr.String(jobKey))
289 }
290 repo := repoName(str(gh, "repository"))
291 if repo == "" {
292 repo = repoFallback
293 }
294 if repo == "" {
295 return nil, Layout{}, fmt.Errorf("job contexts: github.repository is missing")
296 }
297 home := "/home/runner"
298 s.layout = NewLayout(home, repo, "")
299 s.github = setKeySorted(gh, "workspace", expr.String(s.layout.Workspace))
300 return s, s.layout, s.initRest(root, matrixJSON, r)
301}
302
303// WithLayout replaces the directory layout (tests and non-default homes). It
304// must be called before the first step.
305func (s *State) WithLayout(l Layout) {
306 s.layout = l
307 s.github = setKeySorted(s.github, "workspace", expr.String(l.Workspace))
308 s.runner = setKey(s.runner, "temp", expr.String(l.Temp))
309 s.runner = setKey(s.runner, "tool_cache", expr.String(l.ToolCache))
310 s.runner = setKey(s.runner, "workspace", expr.String(l.RunnerWorkspace))
311}
312
313func (s *State) initRest(root *expr.Dictionary, matrixJSON string, r RunnerInfo) error {
314 for _, k := range serverContexts {
315 if v, ok := root.Get(k); ok {
316 s.other[k] = v
317 }
318 }
319 if matrixJSON != "" {
320 if v, err := expr.FromJSON([]byte(matrixJSON)); err == nil {
321 if d, ok := v.(*expr.Dictionary); ok && d.Len() > 0 {
322 s.other["matrix"] = d
323 }
324 } else {
325 return fmt.Errorf("job matrix: %w", err)
326 }
327 }
328 // A job without a matrix has the matrix context null on github.com, not an
329 // empty object (recordings: "matrix": null in the 176 of 221 jobs that have
330 // none, and toJSON(matrix) prints null). The server sends "{}" for such a
331 // job (runner.proto: Job.matrix), so an empty matrix becomes null here.
332 // matrix.x still works: a property of null is null, as for a missing key.
333 if d, ok := s.other["matrix"].(*expr.Dictionary); !ok || d.Len() == 0 {
334 s.other["matrix"] = expr.Null{}
335 }
336 // A context the job does not have is an empty object, like the server's
337 // ("needs": {}, "inputs": {}, "vars": {}); matrix and strategy are null
338 // when absent (recordings: "matrix": null).
339 for _, k := range []string{"needs", "inputs", "vars"} {
340 if _, ok := s.other[k]; !ok {
341 s.other[k] = expr.NewDictionary()
342 }
343 }
344
345 // runner: os, arch, name, environment, tool_cache, temp (JobRunner.cs:165-176,
346 // TempDirectoryManager.cs:31) and workspace (JobExtension.cs:215).
347 env := r.Environment
348 if env == "" {
349 env = "self-hosted"
350 }
351 rc := expr.NewDictionary()
352 rc.Add("os", expr.String("Linux"))
353 rc.Add("arch", expr.String(r.arch()))
354 rc.Add("name", expr.String(r.Name))
355 rc.Add("environment", expr.String(env))
356 rc.Add("tool_cache", expr.String(s.layout.ToolCache))
357 rc.Add("temp", expr.String(s.layout.Temp))
358 rc.Add("workspace", expr.String(s.layout.RunnerWorkspace))
359 s.runner = rc
360
361 // job: status first (StepsRunner.cs:53), then what the server gave.
362 jc := expr.NewDictionary()
363 jc.Add("status", expr.String("success"))
364 if v, ok := root.Get("job"); ok {
365 if d, ok := v.(*expr.Dictionary); ok {
366 for _, p := range d.Pairs() {
367 if !strings.EqualFold(p.Key, "status") {
368 jc.Add(p.Key, p.Value)
369 }
370 }
371 }
372 }
373 s.job = jc
374 return nil
375}
376
377// Layout returns the directory layout.
378func (s *State) Layout() Layout { return s.layout }
379
380// SetGitHub sets a key of the github context (SetGitHubContext).
381func (s *State) SetGitHub(k, v string) { s.github = setKeySorted(s.github, k, expr.String(v)) }
382
383// GitHub returns a string key of the github context ("" when absent).
384func (s *State) GitHub(k string) string { return str(s.github, k) }
385
386// Event returns github.event, or nil.
387func (s *State) Event() expr.Value { v, _ := s.github.Get("event"); return v }
388
389// SetEnv sets a variable of the job environment (Global.EnvironmentVariables
390// and the env context).
391func (s *State) SetEnv(k, v string) { s.env.Set(k, v) }
392
393// JobEnv returns a copy of the job environment.
394func (s *State) JobEnv() *ordered { return s.env.Clone() }
395
396// AddPath records a GITHUB_PATH entry: a repeated entry moves to the end
397// (AddPathFileCommand, FileCommandManager.cs:152).
398func (s *State) AddPath(p string) {
399 s.prepend = slices.DeleteFunc(s.prepend, func(x string) bool { return x == p })
400 s.prepend = append(s.prepend, p)
401}
402
403// Prepend returns the GITHUB_PATH entries in the order added.
404func (s *State) Prepend() []string { return slices.Clone(s.prepend) }
405
406// SetStatus sets job.status ("success", "failure" or "cancelled").
407func (s *State) SetStatus(status string) {
408 s.status = status
409 s.job = setKey(s.job, "status", expr.String(status))
410}
411
412// Status returns job.status.
413func (s *State) Status() string { return s.status }
414
415// stepIDVisible reports whether a step with this id appears in the steps
416// context: generated ids start with "__" and do not (ExecutionContext.cs:644).
417func stepIDVisible(id string) bool { return id != "" && !strings.HasPrefix(id, "__") }
418
419func (s *State) step(id string) *stepCtx {
420 c, ok := s.cur.steps[id]
421 if !ok {
422 c = &stepCtx{outputs: newOrdered()}
423 s.cur.steps[id] = c
424 s.cur.order = append(s.cur.order, id)
425 }
426 return c
427}
428
429// SetOutput sets an output of step id (StepsContext.SetOutput). Steps with a
430// generated id keep no outputs.
431func (s *State) SetOutput(id, name, value string) {
432 if !stepIDVisible(id) {
433 return
434 }
435 s.step(id).outputs.Set(name, value)
436}
437
438// SetStepResult records outcome and conclusion of a completed step
439// (ExecutionContext.UpdateGlobalStepsContext).
440func (s *State) SetStepResult(id, outcome, conclusion string) {
441 if !stepIDVisible(id) {
442 return
443 }
444 c := s.step(id)
445 c.outcome, c.concl = outcome, conclusion
446}
447
448// Output returns an output of a step ("" when absent).
449func (s *State) Output(id, name string) string {
450 if c, ok := s.cur.steps[id]; ok {
451 v, _ := c.outputs.Get(name)
452 return v
453 }
454 return ""
455}
456
457func (s *State) stepsValue() *expr.Dictionary {
458 d := expr.NewDictionary()
459 for _, id := range s.cur.order {
460 c := s.cur.steps[id]
461 sd := expr.NewDictionary()
462 sd.Add("outputs", c.outputs.Dict())
463 if c.outcome != "" {
464 sd.Add("outcome", expr.String(c.outcome))
465 sd.Add("conclusion", expr.String(c.concl))
466 }
467 d.Add(id, sd)
468 }
469 return d
470}
471
472// Values returns the contexts for expression evaluation. stepEnv, when not
473// nil, is the env context of a step (job environment with the step's own
474// variables); otherwise the env context is the job environment.
475func (s *State) Values(env *ordered) map[string]expr.Value {
476 if env == nil {
477 env = s.env
478 }
479 m := map[string]expr.Value{
480 "github": s.github,
481 "runner": s.runner,
482 "job": s.job,
483 "env": env.Dict(),
484 "steps": s.stepsValue(),
485 "secrets": s.secrets(),
486 }
487 for k, v := range s.other {
488 m[k] = v
489 }
490 if s.cur.inputs != nil {
491 m["inputs"] = s.cur.inputs
492 }
493 return m
494}
495
496// secrets is the secrets context. Today it holds one entry, GITHUB_TOKEN, as
497// on GitHub (Variables.ToSecretsContext lists the secret variables but the
498// access token; GitHub's message carries the token as the secret GITHUB_TOKEN,
499// recorded: ctx-inputs-vars, secrets.GITHUB_TOKEN == github.token).
500//
501// Where the value comes from: the "token" key of the github context that the
502// server sends in Job.contexts, the same value as github.token. It must be a
503// token of its own, scoped to the job (the permissions "Set up job" prints) and
504// short lived. It must never be the "ermitejob_" job token: that one
505// authenticates the runner's calls to ermite, stays in the runner host process
506// and must not reach the sandbox (runner.proto, "Secrets and trust"). ermite
507// does not send a token yet (planned for CI-3, with the secrets of the
508// repository); until it does, github.token is empty, so is secrets, and
509// a test of secrets.GITHUB_TOKEN against the empty string is false. Without a
510// token nothing is masked.
511func (s *State) secrets() *expr.Dictionary {
512 d := expr.NewDictionary()
513 if t := s.GitHub("token"); t != "" {
514 d.Add("GITHUB_TOKEN", expr.String(t))
515 }
516 return d
517}
518
519// Secrets returns the values that must never leave the job: the token and the
520// base64 of "x-access-token:<token>" (ExecutionContext.cs:1026-1028); newMasker
521// adds their other encodings.
522func (s *State) Secrets() []string {
523 t := s.GitHub("token")
524 if t == "" {
525 return nil
526 }
527 return []string{t, base64.StdEncoding.EncodeToString([]byte("x-access-token:" + t))}
528}
529
530// ServerContexts returns the contexts that the server has when it evaluates
531// the name of a job: github as received (without the keys the runner adds),
532// needs, strategy, matrix, vars and inputs.
533func (s *State) ServerContexts() map[string]expr.Value {
534 m := map[string]expr.Value{"github": s.serverGitHub}
535 for _, k := range []string{"needs", "strategy", "matrix", "vars", "inputs"} {
536 if v, ok := s.other[k]; ok {
537 m[k] = v
538 }
539 }
540 return m
541}
542
543// githubEnvAllowlist is GitHubContext._contextEnvAllowlist (GitHubContext.cs:9).
544var githubEnvAllowlist = map[string]bool{
545 "action_path": true, "action_ref": true, "action_repository": true, "action": true, "actor": true,
546 "actor_id": true, "api_url": true, "artifacts": true, "artifacts_list": true, "base_ref": true,
547 "env": true, "event_name": true, "event_path": true, "graphql_url": true, "head_ref": true,
548 "job": true, "output": true, "path": true, "ref_name": true, "ref_protected": true, "ref_type": true,
549 "ref": true, "repository": true, "repository_id": true, "repository_owner": true,
550 "repository_owner_id": true, "retention_days": true, "run_attempt": true, "run_id": true,
551 "run_number": true, "server_url": true, "sha": true, "state": true, "step_summary": true,
552 "triggering_actor": true, "workflow": true, "workflow_ref": true, "workflow_sha": true, "workspace": true,
553}
554
555// RuntimeEnv is the environment that the contexts add to every step: the
556// allow-listed keys of github as GITHUB_<KEY> (strings and booleans only, as in
557// GitHubContext.GetRuntimeEnvironmentVariables), and every key of runner as
558// RUNNER_<KEY> (RunnerContext.cs:11). Numbers and objects are skipped: the
559// server must send GitHub's string forms ("run_id": "123").
560func (s *State) RuntimeEnv() *ordered {
561 out := newOrdered()
562 for _, p := range s.github.Pairs() {
563 if !githubEnvAllowlist[strings.ToLower(p.Key)] {
564 continue
565 }
566 switch v := p.Value.(type) {
567 case expr.String:
568 out.Set("GITHUB_"+strings.ToUpper(p.Key), string(v))
569 case expr.Boolean:
570 out.Set("GITHUB_"+strings.ToUpper(p.Key), map[bool]string{true: "true", false: "false"}[bool(v)])
571 }
572 }
573 for _, p := range s.runner.Pairs() {
574 if v, ok := p.Value.(expr.String); ok {
575 out.Set("RUNNER_"+strings.ToUpper(p.Key), string(v))
576 }
577 }
578 return out
579}
580
581// EventJSON returns github.event as JSON for event.json, or "" without event.
582func (s *State) EventJSON() string {
583 v := s.Event()
584 if v == nil {
585 return ""
586 }
587 if _, isNull := v.(expr.Null); isNull {
588 return ""
589 }
590 return expr.ToJSON(v)
591}