1// SPDX-License-Identifier: MIT
2
3package executor
4
5import (
6 "encoding/base64"
7 "fmt"
8 "path/filepath"
9 "runtime"
10 "slices"
11 "strings"
12
13 "bnema.dev/ermite-actions/expr"
14)
15
16// ordered is a string map that keeps the position of a key when it is set
17// again, like the Dictionary<string,string> the official runner uses for
18// Global.EnvironmentVariables and the step environment (the order shows in the
19// "env:" block of the log). Keys are case-sensitive, as on Linux
20// (VarUtil.EnvironmentVariableKeyComparer).
21type ordered struct {
22 keys []string
23 vals map[string]string
24}
25
26func newOrdered() *ordered { return &ordered{vals: map[string]string{}} }
27
28func (o *ordered) Set(k, v string) {
29 if _, ok := o.vals[k]; !ok {
30 o.keys = append(o.keys, k)
31 }
32 o.vals[k] = v
33}
34
35func (o *ordered) Get(k string) (string, bool) { v, ok := o.vals[k]; return v, ok }
36func (o *ordered) Len() int { return len(o.keys) }
37func (o *ordered) Keys() []string { return slices.Clone(o.keys) }
38
39func (o *ordered) Clone() *ordered {
40 c := newOrdered()
41 for _, k := range o.keys {
42 c.Set(k, o.vals[k])
43 }
44 return c
45}
46
47// Dict is the value of the env context. The context of the official runner is
48// case-sensitive on Linux (CaseSensitiveDictionaryContextData); package expr
49// has only case-insensitive dictionaries, so two names that differ by case
50// collapse to the first in expressions (they stay distinct in the process
51// environment). A known deviation.
52func (o *ordered) Dict() *expr.Dictionary {
53 d := expr.NewDictionary()
54 for _, k := range o.keys {
55 d.Add(k, expr.String(o.vals[k]))
56 }
57 return d
58}
59
60// setKey returns d with key set to v: the value is replaced in place when the
61// key (compared case-insensitively) exists, else appended.
62func setKey(d *expr.Dictionary, key string, v expr.Value) *expr.Dictionary {
63 out := expr.NewDictionary()
64 done := false
65 for _, p := range d.Pairs() {
66 if strings.EqualFold(p.Key, key) {
67 if !done {
68 out.Add(p.Key, v)
69 done = true
70 }
71 continue
72 }
73 out.Add(p.Key, p.Value)
74 }
75 if !done {
76 out.Add(key, v)
77 }
78 return out
79}
80
81// setKeySorted is setKey for the github context: a key that is not there yet
82// goes before the first key that sorts after it (bytewise; "token" stays first)
83// instead of at the end, so that the keys the runner adds (workspace,
84// secret_source, the file commands...) sit where the ermite-conformance
85// recordings list them. Note that the recordings are normalised with sorted
86// keys, so they show the sorted order, not necessarily the insertion order of
87// GitHub's runner (SetGitHubContext appends). Keys that exist are replaced in
88// place.
89func setKeySorted(d *expr.Dictionary, key string, v expr.Value) *expr.Dictionary {
90 for _, p := range d.Pairs() {
91 if strings.EqualFold(p.Key, key) {
92 return setKey(d, key, v)
93 }
94 }
95 out := expr.NewDictionary()
96 done := false
97 for _, p := range d.Pairs() {
98 if !done && p.Key > key && !strings.EqualFold(p.Key, "token") {
99 out.Add(key, v)
100 done = true
101 }
102 out.Add(p.Key, p.Value)
103 }
104 if !done {
105 out.Add(key, v)
106 }
107 return out
108}
109
110func str(d *expr.Dictionary, key string) string {
111 if v, ok := d.Get(key); ok {
112 if s, ok := v.(expr.String); ok {
113 return string(s)
114 }
115 }
116 return ""
117}
118
119// Layout is the directory layout of a job, like a hosted runner's
120// (/home/runner/work/<repo>/<repo>, RUNNER_TEMP, RUNNER_TOOL_CACHE).
121type Layout struct {
122 Home string // HOME of the runner user
123 // WorkRoot is the _work directory: <Home>/work.
124 WorkRoot string
125 // Temp is RUNNER_TEMP: <WorkRoot>/_temp.
126 Temp string
127 // ToolCache is RUNNER_TOOL_CACHE.
128 ToolCache string
129 // RunnerWorkspace is RUNNER_WORKSPACE: <WorkRoot>/<repo>.
130 RunnerWorkspace string
131 // Workspace is GITHUB_WORKSPACE: <WorkRoot>/<repo>/<repo>.
132 Workspace string
133}
134
135// DefaultToolCache is RUNNER_TOOL_CACHE on the ermite image.
136const DefaultToolCache = "/opt/hostedtoolcache"
137
138// NewLayout lays out a job of the repository repo (its name, without owner)
139// under home. An empty toolCache means DefaultToolCache.
140func NewLayout(home, repo, toolCache string) Layout {
141 if toolCache == "" {
142 toolCache = DefaultToolCache
143 }
144 work := filepath.Join(home, "work")
145 return Layout{
146 Home: home, WorkRoot: work, Temp: filepath.Join(work, "_temp"), ToolCache: toolCache,
147 RunnerWorkspace: filepath.Join(work, repo), Workspace: filepath.Join(work, repo, repo),
148 }
149}
150
151// RunnerInfo describes the runner for the runner context.
152type RunnerInfo struct {
153 Name string
154 // Environment is "self-hosted" (the default) or "github-hosted".
155 Environment string
156 // Arch is "X64" or "ARM64"; empty means the architecture of this process.
157 Arch string
158}
159
160// githubHosted reports whether the runner.environment is "github-hosted". An
161// empty Environment is "self-hosted", the default of the official runner's
162// runner context.
163func (r RunnerInfo) githubHosted() bool { return r.Environment == "github-hosted" }
164
165func (r RunnerInfo) arch() string {
166 if r.Arch != "" {
167 return r.Arch
168 }
169 switch runtime.GOARCH {
170 case "amd64":
171 return "X64"
172 case "arm64":
173 return "ARM64"
174 case "arm":
175 return "ARM"
176 }
177 return "X86"
178}
179
180// repoName returns the repository name of "owner/name".
181func repoName(full string) string {
182 if i := strings.LastIndexByte(full, '/'); i >= 0 {
183 return full[i+1:]
184 }
185 return full
186}
187
188// stepCtx is one entry of the steps context.
189type stepCtx struct {
190 outputs *ordered
191 outcome string // "" until the step completes
192 concl string
193}
194
195// State holds the contexts of a job as the official runner does
196// (ExecutionContext.ExpressionValues): the contexts the server computed, and
197// github, runner, job, env and steps, which the runner owns.
198type State struct {
199 layout Layout
200 github *expr.Dictionary
201 // serverGitHub is the github context as the server sent it, before the
202 // runner added its keys (token, workspace, the file commands, secret_source,
203 // action...): the contexts that the server itself evaluates the job name
204 // with (see ServerContexts).
205 serverGitHub *expr.Dictionary
206 runner *expr.Dictionary
207 job *expr.Dictionary
208 // other holds needs, strategy, matrix, inputs, vars: copied from the server.
209 other map[string]expr.Value
210 // env is Global.EnvironmentVariables: the job-level environment and what
211 // GITHUB_ENV adds. A step's own env is not in it.
212 env *ordered
213 // steps is the steps context, in order of first mention.
214 stepOrder []string
215 steps map[string]*stepCtx
216 // prepend is Global.PrependPath in the order of the GITHUB_PATH entries.
217 prepend []string
218 // status is job.status: "success", "failure" or "cancelled".
219 status string
220}
221
222// serverContexts are the keys of the contexts JSON that the runner reads.
223var serverContexts = []string{"needs", "strategy", "matrix", "inputs", "vars"}
224
225// NewState builds the contexts of a job from the JSON the server sent (the
226// "contexts" field of the job) and the matrix JSON, with the context
227// initialisation of JobRunner.cs:142-177 and ExecutionContext.InitializeJob
228// (ExecutionContext.cs:1001-1056).
229func NewState(contextsJSON, matrixJSON, jobKey string, r RunnerInfo, repoFallback string) (*State, Layout, error) {
230 root := expr.NewDictionary()
231 if strings.TrimSpace(contextsJSON) != "" {
232 v, err := expr.FromJSON([]byte(contextsJSON))
233 if err != nil {
234 return nil, Layout{}, fmt.Errorf("job contexts: %w", err)
235 }
236 d, ok := v.(*expr.Dictionary)
237 if !ok {
238 return nil, Layout{}, fmt.Errorf("job contexts: want a JSON object")
239 }
240 root = d
241 }
242 s := &State{other: map[string]expr.Value{}, env: newOrdered(), steps: map[string]*stepCtx{}, status: "success"}
243
244 // github: token first, then what the server gave (ExecutionContext.cs:1023-1037).
245 // The token is the one the server sent in github.token (GitHub's
246 // system.github.token variable), empty when it sent none.
247 gh := expr.NewDictionary()
248 gh.Add("token", expr.String(""))
249 if v, ok := root.Get("github"); ok {
250 d, ok := v.(*expr.Dictionary)
251 if !ok {
252 return nil, Layout{}, fmt.Errorf("job contexts: github must be an object")
253 }
254 for _, p := range d.Pairs() {
255 if strings.EqualFold(p.Key, "token") {
256 if t, isStr := p.Value.(expr.String); isStr {
257 gh = setKey(gh, "token", t)
258 }
259 continue
260 }
261 gh.Add(p.Key, p.Value)
262 }
263 }
264 s.serverGitHub = expr.NewDictionary()
265 if v, ok := root.Get("github"); ok {
266 s.serverGitHub = v.(*expr.Dictionary)
267 // GitHub evaluates names with github.job empty (recorded:
268 // ctx-github-names): the job key is only added to the contexts the
269 // job runs with.
270 if _, has := s.serverGitHub.Get("job"); has {
271 s.serverGitHub = setKey(s.serverGitHub, "job", expr.String(""))
272 }
273 }
274 if str(gh, "job") == "" && jobKey != "" {
275 gh = setKeySorted(gh, "job", expr.String(jobKey))
276 }
277 repo := repoName(str(gh, "repository"))
278 if repo == "" {
279 repo = repoFallback
280 }
281 if repo == "" {
282 return nil, Layout{}, fmt.Errorf("job contexts: github.repository is missing")
283 }
284 home := "/home/runner"
285 s.layout = NewLayout(home, repo, "")
286 s.github = setKeySorted(gh, "workspace", expr.String(s.layout.Workspace))
287 return s, s.layout, s.initRest(root, matrixJSON, r)
288}
289
290// WithLayout replaces the directory layout (tests and non-default homes). It
291// must be called before the first step.
292func (s *State) WithLayout(l Layout) {
293 s.layout = l
294 s.github = setKeySorted(s.github, "workspace", expr.String(l.Workspace))
295 s.runner = setKey(s.runner, "temp", expr.String(l.Temp))
296 s.runner = setKey(s.runner, "tool_cache", expr.String(l.ToolCache))
297 s.runner = setKey(s.runner, "workspace", expr.String(l.RunnerWorkspace))
298}
299
300func (s *State) initRest(root *expr.Dictionary, matrixJSON string, r RunnerInfo) error {
301 for _, k := range serverContexts {
302 if v, ok := root.Get(k); ok {
303 s.other[k] = v
304 }
305 }
306 if matrixJSON != "" {
307 if v, err := expr.FromJSON([]byte(matrixJSON)); err == nil {
308 if d, ok := v.(*expr.Dictionary); ok && d.Len() > 0 {
309 s.other["matrix"] = d
310 }
311 } else {
312 return fmt.Errorf("job matrix: %w", err)
313 }
314 }
315 // A job without a matrix has the matrix context null on github.com, not an
316 // empty object (recordings: "matrix": null in the 176 of 221 jobs that have
317 // none, and toJSON(matrix) prints null). The server sends "{}" for such a
318 // job (runner.proto: Job.matrix), so an empty matrix becomes null here.
319 // matrix.x still works: a property of null is null, as for a missing key.
320 if d, ok := s.other["matrix"].(*expr.Dictionary); !ok || d.Len() == 0 {
321 s.other["matrix"] = expr.Null{}
322 }
323 // A context the job does not have is an empty object, like the server's
324 // ("needs": {}, "inputs": {}, "vars": {}); matrix and strategy are null
325 // when absent (recordings: "matrix": null).
326 for _, k := range []string{"needs", "inputs", "vars"} {
327 if _, ok := s.other[k]; !ok {
328 s.other[k] = expr.NewDictionary()
329 }
330 }
331
332 // runner: os, arch, name, environment, tool_cache, temp (JobRunner.cs:165-176,
333 // TempDirectoryManager.cs:31) and workspace (JobExtension.cs:215).
334 env := r.Environment
335 if env == "" {
336 env = "self-hosted"
337 }
338 rc := expr.NewDictionary()
339 rc.Add("os", expr.String("Linux"))
340 rc.Add("arch", expr.String(r.arch()))
341 rc.Add("name", expr.String(r.Name))
342 rc.Add("environment", expr.String(env))
343 rc.Add("tool_cache", expr.String(s.layout.ToolCache))
344 rc.Add("temp", expr.String(s.layout.Temp))
345 rc.Add("workspace", expr.String(s.layout.RunnerWorkspace))
346 s.runner = rc
347
348 // job: status first (StepsRunner.cs:53), then what the server gave.
349 jc := expr.NewDictionary()
350 jc.Add("status", expr.String("success"))
351 if v, ok := root.Get("job"); ok {
352 if d, ok := v.(*expr.Dictionary); ok {
353 for _, p := range d.Pairs() {
354 if !strings.EqualFold(p.Key, "status") {
355 jc.Add(p.Key, p.Value)
356 }
357 }
358 }
359 }
360 s.job = jc
361 return nil
362}
363
364// Layout returns the directory layout.
365func (s *State) Layout() Layout { return s.layout }
366
367// SetGitHub sets a key of the github context (SetGitHubContext).
368func (s *State) SetGitHub(k, v string) { s.github = setKeySorted(s.github, k, expr.String(v)) }
369
370// GitHub returns a string key of the github context ("" when absent).
371func (s *State) GitHub(k string) string { return str(s.github, k) }
372
373// Event returns github.event, or nil.
374func (s *State) Event() expr.Value { v, _ := s.github.Get("event"); return v }
375
376// SetEnv sets a variable of the job environment (Global.EnvironmentVariables
377// and the env context).
378func (s *State) SetEnv(k, v string) { s.env.Set(k, v) }
379
380// JobEnv returns a copy of the job environment.
381func (s *State) JobEnv() *ordered { return s.env.Clone() }
382
383// AddPath records a GITHUB_PATH entry: a repeated entry moves to the end
384// (AddPathFileCommand, FileCommandManager.cs:152).
385func (s *State) AddPath(p string) {
386 s.prepend = slices.DeleteFunc(s.prepend, func(x string) bool { return x == p })
387 s.prepend = append(s.prepend, p)
388}
389
390// Prepend returns the GITHUB_PATH entries in the order added.
391func (s *State) Prepend() []string { return slices.Clone(s.prepend) }
392
393// SetStatus sets job.status ("success", "failure" or "cancelled").
394func (s *State) SetStatus(status string) {
395 s.status = status
396 s.job = setKey(s.job, "status", expr.String(status))
397}
398
399// Status returns job.status.
400func (s *State) Status() string { return s.status }
401
402// stepIDVisible reports whether a step with this id appears in the steps
403// context: generated ids start with "__" and do not (ExecutionContext.cs:644).
404func stepIDVisible(id string) bool { return id != "" && !strings.HasPrefix(id, "__") }
405
406func (s *State) step(id string) *stepCtx {
407 c, ok := s.steps[id]
408 if !ok {
409 c = &stepCtx{outputs: newOrdered()}
410 s.steps[id] = c
411 s.stepOrder = append(s.stepOrder, id)
412 }
413 return c
414}
415
416// SetOutput sets an output of step id (StepsContext.SetOutput). Steps with a
417// generated id keep no outputs.
418func (s *State) SetOutput(id, name, value string) {
419 if !stepIDVisible(id) {
420 return
421 }
422 s.step(id).outputs.Set(name, value)
423}
424
425// SetStepResult records outcome and conclusion of a completed step
426// (ExecutionContext.UpdateGlobalStepsContext).
427func (s *State) SetStepResult(id, outcome, conclusion string) {
428 if !stepIDVisible(id) {
429 return
430 }
431 c := s.step(id)
432 c.outcome, c.concl = outcome, conclusion
433}
434
435// Output returns an output of a step ("" when absent).
436func (s *State) Output(id, name string) string {
437 if c, ok := s.steps[id]; ok {
438 v, _ := c.outputs.Get(name)
439 return v
440 }
441 return ""
442}
443
444func (s *State) stepsValue() *expr.Dictionary {
445 d := expr.NewDictionary()
446 for _, id := range s.stepOrder {
447 c := s.steps[id]
448 sd := expr.NewDictionary()
449 sd.Add("outputs", c.outputs.Dict())
450 if c.outcome != "" {
451 sd.Add("outcome", expr.String(c.outcome))
452 sd.Add("conclusion", expr.String(c.concl))
453 }
454 d.Add(id, sd)
455 }
456 return d
457}
458
459// Values returns the contexts for expression evaluation. stepEnv, when not
460// nil, is the env context of a step (job environment with the step's own
461// variables); otherwise the env context is the job environment.
462func (s *State) Values(env *ordered) map[string]expr.Value {
463 if env == nil {
464 env = s.env
465 }
466 m := map[string]expr.Value{
467 "github": s.github,
468 "runner": s.runner,
469 "job": s.job,
470 "env": env.Dict(),
471 "steps": s.stepsValue(),
472 "secrets": s.secrets(),
473 }
474 for k, v := range s.other {
475 m[k] = v
476 }
477 return m
478}
479
480// secrets is the secrets context. Today it holds one entry, GITHUB_TOKEN, as
481// on GitHub (Variables.ToSecretsContext lists the secret variables but the
482// access token; GitHub's message carries the token as the secret GITHUB_TOKEN,
483// recorded: ctx-inputs-vars, secrets.GITHUB_TOKEN == github.token).
484//
485// Where the value comes from: the "token" key of the github context that the
486// server sends in Job.contexts, the same value as github.token. It must be a
487// token of its own, scoped to the job (the permissions "Set up job" prints) and
488// short lived. It must never be the "ermitejob_" job token: that one
489// authenticates the runner's calls to ermite, stays in the runner host process
490// and must not reach the sandbox (runner.proto, "Secrets and trust"). ermite
491// does not send a token yet (planned for CI-3, with the secrets of the
492// repository); until it does, github.token is empty, so is secrets, and
493// a test of secrets.GITHUB_TOKEN against the empty string is false. Without a
494// token nothing is masked.
495func (s *State) secrets() *expr.Dictionary {
496 d := expr.NewDictionary()
497 if t := s.GitHub("token"); t != "" {
498 d.Add("GITHUB_TOKEN", expr.String(t))
499 }
500 return d
501}
502
503// Secrets returns the values that must never leave the job: the token and the
504// base64 of "x-access-token:<token>" (ExecutionContext.cs:1026-1028); newMasker
505// adds their other encodings.
506func (s *State) Secrets() []string {
507 t := s.GitHub("token")
508 if t == "" {
509 return nil
510 }
511 return []string{t, base64.StdEncoding.EncodeToString([]byte("x-access-token:" + t))}
512}
513
514// ServerContexts returns the contexts that the server has when it evaluates
515// the name of a job: github as received (without the keys the runner adds),
516// needs, strategy, matrix, vars and inputs.
517func (s *State) ServerContexts() map[string]expr.Value {
518 m := map[string]expr.Value{"github": s.serverGitHub}
519 for _, k := range []string{"needs", "strategy", "matrix", "vars", "inputs"} {
520 if v, ok := s.other[k]; ok {
521 m[k] = v
522 }
523 }
524 return m
525}
526
527// githubEnvAllowlist is GitHubContext._contextEnvAllowlist (GitHubContext.cs:9).
528var githubEnvAllowlist = map[string]bool{
529 "action_path": true, "action_ref": true, "action_repository": true, "action": true, "actor": true,
530 "actor_id": true, "api_url": true, "artifacts": true, "artifacts_list": true, "base_ref": true,
531 "env": true, "event_name": true, "event_path": true, "graphql_url": true, "head_ref": true,
532 "job": true, "output": true, "path": true, "ref_name": true, "ref_protected": true, "ref_type": true,
533 "ref": true, "repository": true, "repository_id": true, "repository_owner": true,
534 "repository_owner_id": true, "retention_days": true, "run_attempt": true, "run_id": true,
535 "run_number": true, "server_url": true, "sha": true, "state": true, "step_summary": true,
536 "triggering_actor": true, "workflow": true, "workflow_ref": true, "workflow_sha": true, "workspace": true,
537}
538
539// RuntimeEnv is the environment that the contexts add to every step: the
540// allow-listed keys of github as GITHUB_<KEY> (strings and booleans only, as in
541// GitHubContext.GetRuntimeEnvironmentVariables), and every key of runner as
542// RUNNER_<KEY> (RunnerContext.cs:11). Numbers and objects are skipped: the
543// server must send GitHub's string forms ("run_id": "123").
544func (s *State) RuntimeEnv() *ordered {
545 out := newOrdered()
546 for _, p := range s.github.Pairs() {
547 if !githubEnvAllowlist[strings.ToLower(p.Key)] {
548 continue
549 }
550 switch v := p.Value.(type) {
551 case expr.String:
552 out.Set("GITHUB_"+strings.ToUpper(p.Key), string(v))
553 case expr.Boolean:
554 out.Set("GITHUB_"+strings.ToUpper(p.Key), map[bool]string{true: "true", false: "false"}[bool(v)])
555 }
556 }
557 for _, p := range s.runner.Pairs() {
558 if v, ok := p.Value.(expr.String); ok {
559 out.Set("RUNNER_"+strings.ToUpper(p.Key), string(v))
560 }
561 }
562 return out
563}
564
565// EventJSON returns github.event as JSON for event.json, or "" without event.
566func (s *State) EventJSON() string {
567 v := s.Event()
568 if v == nil {
569 return ""
570 }
571 if _, isNull := v.(expr.Null); isNull {
572 return ""
573 }
574 return expr.ToJSON(v)
575}