1// SPDX-License-Identifier: MIT
2
3package executor
4
5import (
6 "context"
7 "errors"
8 "fmt"
9 "os"
10 "path/filepath"
11 "slices"
12 "strings"
13 "sync"
14 "sync/atomic"
15 "syscall"
16 "time"
17
18 "bnema.dev/ermite-actions/expr"
19 "bnema.dev/ermite-actions/template"
20 "bnema.dev/ermite-actions/workflow"
21 "github.com/bnema/zerowrap"
22 "github.com/google/uuid"
23)
24
25// Names of the implicit steps (JobExtension.cs:70 and :15).
26const (
27 SetUpJobName = "Set up job"
28 CompleteJobName = "Complete job"
29)
30
31// DefaultTimeoutMinutes is the timeout of a job without timeout-minutes.
32const DefaultTimeoutMinutes = 360
33
34// MaxOutputBytes is the limit of ermite on the job outputs, names and values
35// together (1 MiB).
36const MaxOutputBytes = 1 << 20
37
38// maxIssueMessageLength is ExecutionContext._maxIssueMessageLength.
39const maxIssueMessageLength = 4096
40
41// Config describes the job to run.
42type Config struct {
43 // WorkflowPath and WorkflowYAML are the workflow file, which the executor
44 // parses itself; JobKey selects the job.
45 WorkflowPath string
46 WorkflowYAML []byte
47 JobKey string
48 // JobName is the display name of the job, for "Complete job name: ...".
49 // Empty means the job key.
50 JobName string
51 // ContextsJSON and MatrixJSON are the fields of the job message.
52 ContextsJSON string
53 MatrixJSON string
54 // Repo is the repository name used when the github context lacks one.
55 Repo string
56 // TimeoutMinutes is the job timeout; 0 means DefaultTimeoutMinutes. It is
57 // used for the message and, without Deadline, for the timer.
58 TimeoutMinutes int
59 // Deadline is the time left of the job timeout when Run is called, when the
60 // timeout started counting earlier (at the receipt of the job by the host);
61 // 0 means TimeoutMinutes from now.
62 Deadline time.Duration
63
64 Runner RunnerInfo
65 RunnerVersion string
66 // Layout overrides the directory layout (tests); nil means the hosted layout
67 // under Home.
68 Layout *Layout
69 // Home is the home directory of the user of the steps; empty means
70 // /home/runner.
71 Home string
72 // ToolCache is RUNNER_TOOL_CACHE; empty means DefaultToolCache. The official
73 // runner reads it from its own environment (HostContext.cs:411).
74 ToolCache string
75 // BaseEnv is the environment steps inherit; nil means os.Environ().
76 BaseEnv []string
77 // CacheURL and CacheToken, when set, are ACTIONS_CACHE_URL and
78 // ACTIONS_RUNTIME_TOKEN of the node and docker actions of the job: the cache
79 // service of the runner. The token is masked in the log.
80 CacheURL, CacheToken string
81 // CacheMode is ACTIONS_CACHE_MODE ("read" or "write") next to the cache URL;
82 // empty sets nothing.
83 CacheMode string
84 // ContainerCacheURL is CacheURL for the steps that run in a container (a
85 // step in the job container, a docker action): the same relay on an address
86 // of the sandbox their network reaches. Empty: those steps get no runtime
87 // variables.
88 ContainerCacheURL string
89 // Results says that the service behind CacheURL also serves the results
90 // service (artifacts) and that CacheToken carries its scope: node and docker
91 // actions get ACTIONS_RESULTS_URL, and the node steps of the artifact actions
92 // get a GITHUB_SERVER_URL @actions/artifact takes for GitHub (see
93 // setArtifactsServerURL).
94 Results bool
95 // CacheOff says that the service behind CacheURL serves no cache (results
96 // only): ACTIONS_CACHE_URL and ACTIONS_CACHE_MODE are not set.
97 CacheOff bool
98 // Secrets and Vars are Job.secrets and Job.vars. Secrets are registered in
99 // the masker before the first step and form the secrets context; they are
100 // never logged. Vars are the fallback of the vars context.
101 Secrets, Vars map[string]string
102 // SecretsWithheld says that the server did not send secrets because the run
103 // is not trusted: "Set up job" says so.
104 SecretsWithheld bool
105 // TokenPermissions are the permissions of GITHUB_TOKEN as the server sent
106 // them (Job.permissions: key to "read", "write" or "none"), which "Set up
107 // job" prints as they are. Nil means the server sent none: they are then
108 // worked out from the workflow. An empty, non-nil map is "none at all".
109 TokenPermissions map[string]string
110 // IDTokenURL and IDTokenRequestToken are Job.id_token_url (its origin
111 // already rewritten for the sandbox) and Job.id_token_request_token:
112 // ACTIONS_ID_TOKEN_REQUEST_URL and ACTIONS_ID_TOKEN_REQUEST_TOKEN of the run
113 // steps, node actions and docker actions (see addIDTokenEnv). Nothing is set
114 // unless both are. The token is masked before anything is reported.
115 IDTokenURL, IDTokenRequestToken string
116 // Debug turns ##[debug] lines on (ACTIONS_STEP_DEBUG).
117 Debug bool
118 // Actions configures the actions of the job (resolution, downloads, Node.js,
119 // docker).
120 Actions ActionsConfig
121 // SendAnnotations and SendSummaries say that the server keeps annotations
122 // and step summaries (features ci.annotations and ci.summaries): the
123 // Reporter then gets them with StepResults.
124 SendAnnotations, SendSummaries bool
125 // Getenv reads the environment of the executor (the matcher timeout); nil
126 // means os.Getenv.
127 Getenv func(string) string
128 // StepTimeoutSkew is added to every step timeout (tests use a negative
129 // value to run a timeout of one minute in milliseconds).
130 StepTimeoutSkew time.Duration
131
132 Reporter Reporter
133 // Log receives the runner's own diagnostics (not the job log); the zero value
134 // discards them.
135 Log zerowrap.Logger
136
137 // Credential, when set, is the user the steps run as.
138 Credential *syscall.Credential
139 // Cancellation timings and test hooks, see procSpec.
140 TermAfter, KillAfter, WaitDelay time.Duration
141 Signal func(pid int, sig syscall.Signal)
142 // Now returns the current time; nil means time.Now.
143 Now func() time.Time
144}
145
146type stepResult int
147
148// Values are ordered like TaskResult so that merge picks the worst
149// (TaskResultUtil.MergeTaskResults).
150const (
151 resSucceeded stepResult = iota
152 resFailed
153 resCanceled
154 resSkipped
155)
156
157func (r stepResult) conclusion() Conclusion {
158 switch r {
159 case resFailed:
160 return ConclusionFailure
161 case resCanceled:
162 return ConclusionCancelled
163 case resSkipped:
164 return ConclusionSkipped
165 }
166 return ConclusionSuccess
167}
168
169// merge is TaskResultUtil.MergeTaskResults (Runner.Common/Util/TaskResultUtil.cs:40).
170func merge(cur *stepResult, coming stepResult) stepResult {
171 if cur == nil {
172 return coming
173 }
174 if *cur > resFailed {
175 return *cur
176 }
177 if coming >= *cur {
178 return coming
179 }
180 return *cur
181}
182
183type job struct {
184 cfg Config
185 cancel *Canceller
186 wf *workflow.Workflow
187 jb *workflow.Job
188 fc *FileCommands
189 // tempRoot is RUNNER_TEMP, opened before the first step (see FileCommands).
190 tempRoot *os.Root
191 trackID string
192
193 mu sync.Mutex // serialises the Reporter and guards state and result
194 state *State
195 result *stepResult // nil until a step fails or the job is cancelled (Global result)
196 cancelled bool
197 runDefaults struct{ shell, workingDirectory string }
198 baseEnv []string
199 lookupPath string
200 before map[string]bool
201 // masker hides the secrets in everything reported; Run sets it, with the
202 // secrets of the job, before the first report.
203 masker atomic.Pointer[masker]
204 // actions holds what the pre-job phase loaded (see prepareActions).
205 actions *actionState
206 // postStack is the post steps registered so far, last registered last.
207 postStack []qstep
208 // upgradedToNode24 are the actions that asked for node20 and got node24.
209 upgradedToNode24 []string
210 jobCtx context.Context
211 // cs is the containers of the job (nil container and no services: none).
212 cs *containerSet
213
214 // Results of the commands (see issues.go, matchers.go, results.go).
215 matchers *matcherRoot
216 issuesMu sync.Mutex
217 collectors map[int]*issueCollector // by step number
218 outcomes map[int]Conclusion
219 summaries map[int]string
220 logLines map[int]int // lines written to the log of each step, under mu
221 states map[string]*intraState // intra-action state, see saveState
222 // ghesSniff is, by script path, whether a node action's script bundles
223 // @actions/artifact v2 (see bundlesArtifactClient); under mu.
224 ghesSniff map[string]bool
225}
226
227func (j *job) mask(s string) string { return j.masker.Load().mask(s) }
228
229// The runner's own diagnostics (Config.Log) bypass the reporter, hence the
230// masker: a path, a docker message or an error can hold a secret. Every
231// diagnostic of the executor goes through these helpers, which mask the
232// rendered error, message and fields (TestDiagnosticsAreMasked checks that no
233// other code calls Config.Log).
234
235// diagWarn logs a warning with an optional error and string fields (key,
236// value, ...).
237func (j *job) diagWarn(err error, msg string, kv ...string) {
238 j.diag(false, err, msg, kv)
239}
240
241// diagDebug is diagWarn at debug level.
242func (j *job) diagDebug(err error, msg string, kv ...string) {
243 j.diag(true, err, msg, kv)
244}
245
246func (j *job) now() time.Time {
247 if j.cfg.Now != nil {
248 return j.cfg.Now()
249 }
250 return time.Now()
251}
252
253// Run executes one job and reports to cfg.Reporter. It always ends with a
254// JobCompleted call and returns the same Result. cancel may be nil.
255func Run(cancel *Canceller, cfg Config) Result {
256 if cancel == nil {
257 cancel = NewCanceller()
258 }
259 j := &job{cfg: cfg, cancel: cancel, trackID: "github_" + uuid.NewString(), matchers: &matcherRoot{}}
260 // The masker exists before anything is reported, with the secrets of the job
261 // registered (Worker.cs:158-171 does it before the job starts); the token and
262 // the cache token join it once the contexts are built. GitHub's SecretMasker
263 // has no minimum length (SecretMasker.AddValue ignores only null or empty
264 // values), so a one-character secret is masked too.
265 j.masker.Store(newMasker())
266 registerSecrets(j.masker.Load(), cfg.Secrets)
267 j.masker.Load().add(cfg.IDTokenRequestToken)
268 j.cfg.Reporter = &maskReporter{j: j, next: cfg.Reporter}
269 defer func() {
270 if j.fc != nil {
271 _ = j.fc.Close()
272 }
273 if j.tempRoot != nil {
274 _ = j.tempRoot.Close()
275 }
276 }()
277 res := j.run()
278 res.Error = j.mask(res.Error)
279 j.cfg.Reporter.JobCompleted(res)
280 return res
281}
282
283func (j *job) run() Result {
284 // Whatever happens below (a panic, a shutdown that skips the post steps),
285 // the containers, the network and the credentials directories are removed.
286 defer j.fallbackCleanup()
287 limit := j.cfg.Deadline
288 if limit <= 0 {
289 limit = time.Duration(j.timeoutMinutes()) * time.Minute
290 }
291 timer := time.AfterFunc(limit, func() { j.cancel.Cancel(CancelTimeout) })
292 defer timer.Stop()
293
294 // ---- Set up job (JobExtension.InitializeJob) ----
295 rep := j.cfg.Reporter
296 rep.StepStarted(1, SetUpJobName, j.now())
297 var cancelCtx context.CancelFunc
298 j.jobCtx, cancelCtx = context.WithCancel(context.Background())
299 defer cancelCtx()
300 go func() {
301 select {
302 case <-j.cancel.Done():
303 cancelCtx()
304 case <-j.jobCtx.Done():
305 }
306 }()
307 steps, err := j.setUp()
308 if err != nil {
309 j.errorf(1, "%s", err.Error())
310 rep.StepCompleted(1, SetUpJobName, ConclusionFailure, j.now())
311 // The job ends here, without "Complete job" (JobRunner.cs:107).
312 return Result{Conclusion: ConclusionFailure, Error: err.Error()}
313 }
314 rep.StepCompleted(1, SetUpJobName, ConclusionSuccess, j.now())
315
316 // ---- the steps (StepsRunner) ----
317 // The queue is the pre-job steps then the steps of the job; the post steps
318 // registered while they run follow, last registered first
319 // (StepsRunner.cs:62-75).
320 num := 2
321 for _, q := range steps {
322 j.runStep(num, q)
323 num++
324 }
325 for {
326 j.mu.Lock()
327 if len(j.postStack) == 0 {
328 j.mu.Unlock()
329 break
330 }
331 q := j.postStack[len(j.postStack)-1]
332 j.postStack = j.postStack[:len(j.postStack)-1]
333 j.mu.Unlock()
334 j.runStep(num, q)
335 num++
336 }
337
338 // ---- Complete job (JobExtension.FinalizeJob) ----
339 outputs := j.completeJob(num)
340
341 return j.finish(outputs)
342}
343
344func (j *job) finish(outputs map[string]string) Result {
345 res := Result{Outputs: outputs}
346 j.mu.Lock()
347 defer j.mu.Unlock()
348 r := resSucceeded
349 if j.result != nil {
350 r = *j.result
351 }
352 switch {
353 case j.cancel.Reason() == CancelTimeout && j.cancelled:
354 res.Conclusion = ConclusionTimedOut
355 res.Error = fmt.Sprintf("The job has exceeded the maximum execution time of %d minutes.", j.timeoutMinutes())
356 case r == resCanceled:
357 res.Conclusion = ConclusionCancelled
358 res.RunnerShutdown = j.cancel.Reason() == CancelShutdown
359 case r == resFailed:
360 res.Conclusion = ConclusionFailure
361 default:
362 res.Conclusion = ConclusionSuccess
363 }
364 return res
365}
366
367func (j *job) timeoutMinutes() int {
368 if j.cfg.TimeoutMinutes > 0 {
369 return j.cfg.TimeoutMinutes
370 }
371 return DefaultTimeoutMinutes
372}
373
374// ---------------------------------------------------------------- logging
375
376// write sends one log line to the Reporter, in order. It returns the 1-based
377// number of the line in the log of the step num, what the runner calls
378// logFileLineNumber (_logger.TotalLines + 1 in ExecutionContext.Write). The
379// steps inside a composite action write to the log of the step that uses it,
380// so they count in the same sequence.
381func (j *job) write(num int, line string) int {
382 j.mu.Lock()
383 defer j.mu.Unlock()
384 if j.logLines == nil {
385 j.logLines = map[int]int{}
386 }
387 n := j.logLines[num] + 1
388 // A line with a line feed in it is several lines of the log (Logging.cs:
389 // 90-101); the host splits the text the same way.
390 j.logLines[num] += 1 + strings.Count(line, "\n")
391 j.cfg.Reporter.Log(num, j.now(), line)
392 return n
393}
394
395// out writes a message that may hold several lines (ExecutionContext.Output).
396func (j *job) out(num int, text string) {
397 for _, l := range strings.Split(strings.ReplaceAll(text, "\r\n", "\n"), "\n") {
398 j.write(num, l)
399 }
400}
401
402// tagged writes tag+message; only the first line carries the tag. The tags of
403// issues (##[error], ##[warning], ##[notice]) go through AddIssue: they are
404// logged and become annotations, like the runner's IExecutionContext.Error
405// and Warning.
406func (j *job) tagged(num int, tag, msg string) {
407 if level, ok := levelOfTag(tag); ok {
408 j.addIssue(num, nil, false, issue{Level: level, Message: msg})
409 return
410 }
411 j.logTagged(num, tag, j.mask(msg))
412}
413
414func (j *job) errorf(num int, format string, a ...any) {
415 j.tagged(num, "##[error]", fmt.Sprintf(format, a...))
416}
417
418func (j *job) debug(num int, msg string) {
419 if j.cfg.Debug {
420 for _, l := range strings.Split(strings.ReplaceAll(msg, "\r\n", "\n"), "\n") {
421 j.write(num, "##[debug]"+l)
422 }
423 }
424}
425
426// ---------------------------------------------------------------- set up
427
428// evalEnv is the environment for expressions of the job and step tokens.
429func (j *job) evalEnv(env *ordered) template.Env {
430 return template.Env{
431 Contexts: j.state.Values(env),
432 Funcs: map[string]expr.Func{"hashFiles": j.hashFilesFunc()},
433 }
434}
435
436func (j *job) evalString(tok template.Token, env *ordered) (string, error) {
437 if tok == nil {
438 return "", nil
439 }
440 return template.EvaluateString(context.Background(), tok, j.evalEnv(env))
441}
442
443// evalMap evaluates a mapping of strings (EvaluateStepEnvironment): values
444// are converted like GitHub converts them (null is the empty string).
445func (j *job) evalMap(m *template.Mapping, env *ordered) ([]Pair, error) {
446 if m == nil {
447 return nil, nil
448 }
449 v, err := template.Evaluate(context.Background(), m, j.evalEnv(env))
450 if err != nil {
451 return nil, err
452 }
453 d, ok := v.(*expr.Dictionary)
454 if !ok {
455 return nil, nil
456 }
457 var out []Pair
458 for _, p := range d.Pairs() {
459 if k := p.Value.Kind(); k == expr.KindArray || k == expr.KindDictionary {
460 return nil, ghErrorf("Unexpected type '%s' encountered while reading 'environment value'. The type 'StringToken' was expected.", k)
461 }
462 out = append(out, Pair{p.Key, expr.CoerceString(p.Value)})
463 }
464 return out, nil
465}
466
467// setUp parses the workflow and prepares the job: JobExtension.InitializeJob
468// (JobExtension.cs:63-182) and the job-start code of JobRunner.cs:142-177. Its
469// log lines belong to step 1.
470func (j *job) setUp() ([]qstep, error) {
471 cfg := j.cfg
472 j.out(1, fmt.Sprintf("Current runner version: '%s'", cfg.RunnerVersion))
473 // Only a self-hosted runner prints its name and machine (JobExtension.cs:
474 // 88-98: the runner is identified by its OAuth client id); the recordings
475 // of github-hosted runners have neither line. The runner group name
476 // (system.runnerGroupName) is not known to ermite.
477 if !cfg.Runner.githubHosted() {
478 if cfg.Runner.Name != "" {
479 j.out(1, fmt.Sprintf("Runner name: '%s'", cfg.Runner.Name))
480 }
481 if host, err := os.Hostname(); err == nil {
482 j.out(1, fmt.Sprintf("Machine name: '%s'", host))
483 }
484 }
485
486 wf, errs := workflow.Parse(cfg.WorkflowPath, cfg.WorkflowYAML)
487 if len(errs) > 0 {
488 var msgs []string
489 for _, e := range errs {
490 msgs = append(msgs, e.Error())
491 }
492 return nil, ghErrorf("The workflow is not valid. %s", strings.Join(msgs, "\n"))
493 }
494 jb := wf.Job(cfg.JobKey)
495 if jb == nil {
496 return nil, ghErrorf("The workflow has no job '%s'.", cfg.JobKey)
497 }
498 j.wf, j.jb = wf, jb
499
500 state, layout, err := NewState(cfg.ContextsJSON, cfg.MatrixJSON, cfg.JobKey, cfg.Runner, cfg.Repo)
501 if err != nil {
502 return nil, err
503 }
504 switch {
505 case cfg.Layout != nil:
506 state.WithLayout(*cfg.Layout)
507 layout = *cfg.Layout
508 case (cfg.Home != "" && cfg.Home != layout.Home) || cfg.ToolCache != "":
509 home := cfg.Home
510 if home == "" {
511 home = layout.Home
512 }
513 layout = NewLayout(home, filepath.Base(layout.Workspace), cfg.ToolCache)
514 state.WithLayout(layout)
515 }
516 state.WithJobData(cfg.Secrets, cfg.Vars)
517 j.state = state
518 for _, v := range state.Secrets() {
519 j.masker.Load().add(v)
520 }
521 j.masker.Load().add(cfg.CacheToken)
522
523 // Features that arrive in later phases fail the job with a clear message.
524 switch {
525 case jb.IsReusable():
526 return nil, errors.New("not supported yet: reusable workflows (jobs.<id>.uses)")
527 }
528
529 j.printTokenInfo()
530
531 // Directories (TempDirectoryManager, PipelineDirectoryManager).
532 j.out(1, "Prepare workflow directory")
533 for _, d := range []string{layout.Temp, layout.Workspace, layout.ToolCache} {
534 if err := os.MkdirAll(d, 0o755); err != nil {
535 return nil, fmt.Errorf("create %s: %w", d, err)
536 }
537 }
538 clearDir(layout.Temp)
539 if err := os.MkdirAll(layout.Temp, 0o755); err != nil {
540 return nil, err
541 }
542 fc, err := NewFileCommands(filepath.Join(layout.Temp, "_runner_file_commands"))
543 if err != nil {
544 return nil, err
545 }
546 if c := cfg.Credential; c != nil {
547 fc.SetOwner(int(c.Uid), int(c.Gid))
548 }
549 j.fc = fc
550 // Everything the executor writes below the temp directory later goes through
551 // this root (see FileCommands): the user of the steps owns the directory.
552 if j.tempRoot, err = os.OpenRoot(layout.Temp); err != nil {
553 return nil, err
554 }
555 for _, d := range []string{layout.WorkRoot, layout.RunnerWorkspace, layout.Workspace, layout.Temp, fc.dir} {
556 j.own(d)
557 }
558
559 j.baseEnv = cfg.BaseEnv
560 if j.baseEnv == nil {
561 j.baseEnv = os.Environ()
562 }
563 if !hasEnv(j.baseEnv, "HOME") {
564 j.baseEnv = append(append([]string(nil), j.baseEnv...), "HOME="+layout.Home)
565 }
566 j.lookupPath = envValue(j.baseEnv, "PATH")
567
568 // Job-level environment: workflow env, then job env, each evaluated with an
569 // empty env context (JobExtension.cs:38-49, ExecutionContext.cs:1048).
570 for _, m := range []*template.Mapping{wf.Env, jb.Env} {
571 pairs, err := j.evalMap(m, newOrdered())
572 if err != nil {
573 return nil, err
574 }
575 for _, p := range pairs {
576 j.state.SetEnv(p.Key, p.Value)
577 }
578 }
579
580 // defaults.run: a job-level "run" replaces the workflow's whole (JobExtension.cs:78-96).
581 for _, d := range []*workflow.Defaults{wf.Defaults, jb.Defaults} {
582 if d == nil || d.Run == nil {
583 continue
584 }
585 shell, err := j.evalString(d.Run.Shell, newOrdered())
586 if err != nil {
587 return nil, err
588 }
589 wd, err := j.evalString(d.Run.WorkingDirectory, newOrdered())
590 if err != nil {
591 return nil, err
592 }
593 j.runDefaults.shell, j.runDefaults.workingDirectory = shell, wd
594 }
595
596 // JobExtension.cs:250-275: the job container and the services are
597 // evaluated after the job environment, before the actions are prepared.
598 j.debug(1, "Evaluating job container")
599 j.debug(1, "Evaluating job service containers")
600 jc, svcs, err := j.evalContainers()
601 if err != nil {
602 return nil, err
603 }
604 if jc != nil || len(svcs) > 0 {
605 j.cs = &containerSet{job: jc, services: svcs}
606 }
607
608 j.out(1, "Prepare all required actions")
609 queue, err := j.prepareJob()
610 if err != nil {
611 return nil, err
612 }
613 // JobExtension.cs:418-422: the display name of the job, which the server
614 // computes there (message.JobDisplayName). ermite's Job message has no such
615 // field, so the executor evaluates it (see jobDisplayName).
616 name := cfg.JobName
617 if name == "" {
618 name = j.jobDisplayName()
619 }
620 if strings.TrimSpace(name) != "" {
621 j.out(1, "Complete job name: "+name)
622 }
623
624 // Process tracking for the orphan cleanup of "Complete job".
625 j.before = map[string]bool{}
626 for _, p := range snapshotProcesses() {
627 j.before[p.key()] = true
628 }
629 return queue, nil
630}
631
632// own gives a file or directory created by the executor to the user the steps
633// run as, so that scripts can write to RUNNER_TEMP, the workspace and the file
634// command files. Best effort: without Credential nothing is done.
635func (j *job) own(path string) {
636 if c := j.cfg.Credential; c != nil {
637 _ = os.Lchown(path, int(c.Uid), int(c.Gid))
638 }
639}
640
641func clearDir(dir string) {
642 ents, err := os.ReadDir(dir)
643 if err != nil {
644 return
645 }
646 for _, e := range ents {
647 _ = os.RemoveAll(filepath.Join(dir, e.Name()))
648 }
649}
650
651func hasEnv(env []string, key string) bool {
652 for _, kv := range env {
653 if strings.HasPrefix(kv, key+"=") {
654 return true
655 }
656 }
657 return false
658}
659
660func envValue(env []string, key string) string {
661 v := ""
662 for _, kv := range env {
663 if k, val, ok := strings.Cut(kv, "="); ok && k == key {
664 v = val
665 }
666 }
667 return v
668}
669
670// ---------------------------------------------------------------- complete
671
672// completeJob is JobExtension.FinalizeJob (JobExtension.cs:9-190): job outputs
673// and the cleanup of orphan processes.
674func (j *job) completeJob(num int) map[string]string {
675 rep := j.cfg.Reporter
676 rep.StepStarted(num, CompleteJobName, j.now())
677 result := resSucceeded
678 outputs := map[string]string{}
679
680 if j.jb.Outputs != nil {
681 // The runner of the official service prints nothing here: the recordings
682 // of github.com show only "Cleaning up orphan processes" in "Complete
683 // job", although the jobs have outputs (the log lines of
684 // JobExtension.cs:48,68 belong to the legacy message that carries
685 // JobOutputs).
686 j.mu.Lock()
687 env := j.state.JobEnv()
688 j.mu.Unlock()
689 pairs, err := j.evalMap(j.jb.Outputs, env)
690 if err != nil {
691 result = resFailed
692 j.errorf(num, "Fail to evaluate job outputs")
693 j.errorf(num, "%s", err.Error())
694 j.mu.Lock()
695 m := merge(j.result, resFailed)
696 j.result = &m
697 j.mu.Unlock()
698 }
699 total := 0
700 for _, p := range pairs {
701 if p.Value == "" {
702 j.debug(num, fmt.Sprintf("Skip output '%s' since it's empty", p.Key))
703 continue
704 }
705 // An output that holds a secret is dropped (JobExtension.cs:819-824).
706 if j.masker.Load().has(p.Value) {
707 j.tagged(num, "##[warning]", fmt.Sprintf("Skip output '%s' since it may contain secret.", p.Key))
708 continue
709 }
710 // ermite refuses outputs above 1 MiB in total (runner.proto,
711 // CompleteJobRequest.outputs), and they would not fit a frame of the
712 // stream either: the job fails without outputs.
713 if total += len(p.Key) + len(p.Value); total > MaxOutputBytes {
714 result = resFailed
715 j.errorf(num, "Fail to evaluate job outputs")
716 j.errorf(num, "The job outputs are larger than 1 MiB in total.")
717 j.mu.Lock()
718 m := merge(j.result, resFailed)
719 j.result = &m
720 j.mu.Unlock()
721 outputs = map[string]string{}
722 break
723 }
724 outputs[p.Key] = p.Value
725 }
726 }
727
728 j.out(num, "Cleaning up orphan processes")
729 killOrphans(j.trackID, j.before, func(l string) { j.out(num, l) })
730
731 // The actions that asked for node20 and got node24 (JobExtension.cs:1043-1049).
732 j.mu.Lock()
733 up := slices.SortedFunc(slices.Values(j.upgradedToNode24), func(a, b string) int { return strings.Compare(strings.ToLower(a), strings.ToLower(b)) })
734 j.mu.Unlock()
735 if len(up) > 0 {
736 j.tagged(num, "##[warning]", "Node.js 20 is deprecated. The following actions target Node.js 20 but are being forced to run on Node.js 24: "+
737 strings.Join(up, ", ")+". For more information see: "+node20DeprecationURL)
738 }
739
740 rep.StepCompleted(num, CompleteJobName, result.conclusion(), j.now())
741 return outputs
742}
743
744// Defaults of the lines "Set up job" prints between the runner version and
745// "Prepare workflow directory" (JobExtension.cs:150-182). The server gives
746// GitHub's runner these as the variables system.github.token.permissions and
747// actions_cache_mode and the github context key secret_source; ermite's Job
748// message carries none of them, so the values recorded on github.com for a
749// push to a public repository are used. Every one of the 221 recorded jobs
750// has exactly these lines.
751const (
752 defaultSecretSource = "Actions"
753 defaultCacheMode = "write"
754)
755
756// SecretsWithheldLine is the line of "Set up job" for a run whose secrets the
757// server withheld (an ermite text: GitHub has none for fork pull requests).
758const SecretsWithheldLine = "Secrets are not available to this job: the run was started by a limited token."
759
760// printTokenInfo prints the GITHUB_TOKEN permissions group, the secret source
761// and the cache mode.
762//
763// GitHub's runner prints what the server sends as
764// system.github.token.permissions (JobExtension.cs:154-166). ermite's server
765// sends them in Job.permissions, already merged and lowered by its trust rule:
766// they are printed as they are (serverPermissionLines). From a server that
767// sends none the group is worked out from the workflow with the rules recorded
768// on github.com (tokenPermissionLines). It is always printed: Metadata is listed
769// even for "permissions: {}".
770func (j *job) printTokenInfo() {
771 j.out(1, "##[group]GITHUB_TOKEN Permissions")
772 var lines []permissionLine
773 if j.cfg.TokenPermissions != nil {
774 lines = serverPermissionLines(j.cfg.TokenPermissions)
775 } else {
776 lines = tokenPermissionLines(j.wf, j.jb)
777 }
778 for _, p := range lines {
779 j.out(1, p.name+": "+p.level)
780 }
781 j.out(1, "##[endgroup]")
782 // secret_source is a github context key (JobExtension.cs:172); the context
783 // of a job from ermite lacks it, so it is added with the recorded value.
784 src := j.state.GitHub("secret_source")
785 if src == "" {
786 src = defaultSecretSource
787 j.state.SetGitHub("secret_source", src)
788 }
789 j.out(1, "Secret source: "+src)
790 if j.cfg.SecretsWithheld {
791 j.out(1, SecretsWithheldLine)
792 }
793 j.out(1, "Cache mode: "+defaultCacheMode)
794}
795
796// jobDisplayName is the name of the job as GitHub shows it and the runner
797// prints it after "Complete job name:": the "name" of the job (the job key when
798// it has none) evaluated with github, needs, strategy, matrix, vars and inputs
799// as the server has them (State.ServerContexts: github without the keys the
800// runner adds), followed by the matrix values in parentheses unless the name
801// reads the matrix (recorded: matrix-job-name, see workflow.Job.DisplayName).
802// When the name cannot be worked out the job key is used and a warning is
803// logged: the name is not worth failing the job for.
804func (j *job) jobDisplayName() string {
805 ctx := context.Background()
806 fallback := func(why string, err error) string {
807 j.diagWarn(err, "job display name: "+why+", using the job key", "job_key", j.cfg.JobKey)
808 return j.cfg.JobKey
809 }
810 j.mu.Lock()
811 all := j.state.ServerContexts()
812 j.mu.Unlock()
813 env := j.evalEnv(newOrdered())
814 env.Contexts = all
815 // The strategy itself sees neither the strategy nor the matrix context.
816 senv := env
817 senv.Contexts = map[string]expr.Value{}
818 for _, k := range []string{"github", "needs", "vars", "inputs"} {
819 senv.Contexts[k] = all[k]
820 }
821 st, err := j.jb.EvaluateStrategy(ctx, senv)
822 if err != nil {
823 return fallback("the strategy does not evaluate", err)
824 }
825 idx, ok := comboIndex(st, all["strategy"], all["matrix"])
826 if !ok {
827 return fallback("the matrix of the job is not one of the combinations of its strategy", nil)
828 }
829 name, err := j.jb.DisplayName(ctx, env, st, idx)
830 if err != nil {
831 return fallback("the name does not evaluate", err)
832 }
833 // GitHub shows "Display name of", not "Display name of " (recorded:
834 // matrix-include-exclude, an empty last matrix value). The recordings only
835 // show trailing spaces; both ends are trimmed, as ermite's server does.
836 return strings.TrimSpace(name)
837}
838
839// comboIndex finds the combination of the strategy that is this job. Without a
840// matrix it is the only one. The index of the strategy context is trusted only
841// when that combination has the matrix of the job; otherwise the first
842// combination with equal values is taken. ok is false when there is none.
843func comboIndex(st *workflow.Strategy, strategyCtx, matrix expr.Value) (idx int, ok bool) {
844 if !st.HasMatrix {
845 return 0, true
846 }
847 m, isDict := matrix.(*expr.Dictionary)
848 if !isDict {
849 return 0, false
850 }
851 if d, isDict := strategyCtx.(*expr.Dictionary); isDict {
852 if n, found := d.Get("job-index"); found {
853 if f, isNum := n.(expr.Number); isNum && f >= 0 && int(f) < len(st.Combos) && sameMatrix(st.Combos[int(f)].Values, m) {
854 return int(f), true
855 }
856 }
857 }
858 for i, c := range st.Combos {
859 if sameMatrix(c.Values, m) {
860 return i, true
861 }
862 }
863 return 0, false
864}
865
866// sameMatrix compares two matrix values: the same keys (ignoring case and
867// order) with equal values.
868func sameMatrix(a, b *expr.Dictionary) bool {
869 if a.Len() != b.Len() {
870 return false
871 }
872 for _, p := range a.Pairs() {
873 v, found := b.Get(p.Key)
874 if !found || expr.ToJSON(v) != expr.ToJSON(p.Value) {
875 return false
876 }
877 }
878 return true
879}
880
881func (j *job) diag(debug bool, err error, msg string, kv []string) {
882 ev := j.cfg.Log.Warn()
883 if debug {
884 ev = j.cfg.Log.Debug()
885 }
886 if err != nil {
887 ev = ev.Str("error", j.mask(err.Error()))
888 }
889 for i := 0; i+1 < len(kv); i += 2 {
890 ev = ev.Str(kv[i], j.mask(kv[i+1]))
891 }
892 ev.Msg(j.mask(msg))
893}