1// SPDX-License-Identifier: MIT
2
3// Package externals provides the tools that jobs expect to find outside the
4// image, today the Node.js builds of actions/runner (node20, node24 and the
5// alpine variants for job containers), so that a runner upgrade or a new Node.js
6// version never needs an image rebuild.
7//
8// The host downloads the pinned builds (table.go) into
9// <state dir>/externals/<name>-<version>-<arch>/ at start, verifying the
10// SHA-256 of each archive before it is unpacked, and the sandbox drivers attach
11// them read-only at /home/runner/externals/<name> (GitHub's layout:
12// externals/node24/bin/node). Only the pinned versions are kept.
13//
14// A tool that cannot be provided (offline, checksum mismatch, no build for this
15// architecture, checksum not recorded) never stops the runner: the sandboxes get
16// a directory with the file UNAVAILABLE and a bin/node stub that print why, and
17// a job that needs the tool fails with that message. Failed downloads are retried in the
18// background.
19package externals
20
21import (
22 "context"
23 "crypto/sha256"
24 "encoding/hex"
25 "errors"
26 "fmt"
27 "io"
28 "net/http"
29 "net/url"
30 "os"
31 "path/filepath"
32 "strings"
33 "sync"
34 "time"
35
36 "github.com/bnema/zerowrap"
37
38 "bnema.dev/ermite-runner/internal/sandbox"
39)
40
41// maxDownload bounds one archive (the largest build is about 60 MiB).
42const maxDownload = 512 << 20
43
44// Manager downloads, verifies and keeps the externals of one architecture.
45type Manager struct {
46 // Dir is <state dir>/externals.
47 Dir string
48 // Arch is GitHub's architecture name of the host (x64, arm64).
49 Arch string
50 // Table is the set of known artifacts (Pinned in production).
51 Table []Artifact
52 // Client downloads the archives; nil means a client with a 15 minute limit
53 // that follows redirects to https locations only.
54 Client *http.Client
55 // RetryEvery is the interval of the background retries of the downloads that
56 // failed for a reason that may pass (default 5 minutes).
57 RetryEvery time.Duration
58
59 mu sync.Mutex
60 state map[string]entry // by artifact name
61 syncing bool
62
63 // stubMu serialises the creation of the directories of unavailable tools;
64 // stubs remembers the ones built (or checked) by this process.
65 stubMu sync.Mutex
66 stubs map[string]bool
67}
68
69type entry struct {
70 ext sandbox.External
71 retry bool // a failure worth trying again (network), not a verdict
72}
73
74// New returns a Manager for the host architecture; ok is false when the runner
75// has no builds for it.
76func New(stateDir string) (m *Manager, ok bool) {
77 arch := HostArch()
78 return &Manager{Dir: filepath.Join(stateDir, "externals"), Arch: arch, Table: Pinned}, arch != ""
79}
80
81// Start runs Sync in the background and waits for it at most wait, so that a
82// runner with a working network normally takes its first job with everything in
83// place, and one without never waits long. When the wait ends the download goes
84// on; jobs that start meanwhile see the tools that are not done yet as
85// unavailable ("still being downloaded"). Downloads that failed for a reason
86// that may pass (network, HTTP status) are retried every RetryEvery, jobs or
87// not, until ctx ends.
88func (m *Manager) Start(ctx context.Context, wait time.Duration) {
89 log := zerowrap.FromCtx(ctx)
90 done := make(chan struct{})
91 go func() {
92 defer close(done)
93 if err := m.Sync(ctx); err != nil && ctx.Err() == nil {
94 log.Warn().Err(err).Msg("some external tools are not available: jobs that need them fail with the reason; the download is retried")
95 }
96 }()
97 go m.retryLoop(ctx)
98 select {
99 case <-done:
100 case <-time.After(wait):
101 log.Warn().Dur("waited", wait).Msg("external tools are still downloading: the runner takes jobs, those that need them fail until they are ready")
102 case <-ctx.Done():
103 }
104}
105
106// retryLoop calls Sync every RetryEvery while a tool is missing for a reason
107// that may pass.
108func (m *Manager) retryLoop(ctx context.Context) {
109 every := m.RetryEvery
110 if every == 0 {
111 every = 5 * time.Minute
112 }
113 t := time.NewTicker(every)
114 defer t.Stop()
115 for {
116 select {
117 case <-ctx.Done():
118 return
119 case <-t.C:
120 }
121 if m.wantsRetry() {
122 _ = m.Sync(ctx)
123 }
124 }
125}
126
127func (m *Manager) wantsRetry() bool {
128 m.mu.Lock()
129 defer m.mu.Unlock()
130 for _, e := range m.state {
131 if e.retry {
132 return true
133 }
134 }
135 return false
136}
137
138// Sync makes the externals match the table: it downloads what is missing,
139// removes everything else from Dir, and records the outcome per tool as it goes.
140// It returns an error describing the artifacts that could not be provided; the
141// runner goes on with them marked unavailable.
142func (m *Manager) Sync(ctx context.Context) error {
143 m.mu.Lock()
144 if m.syncing {
145 m.mu.Unlock()
146 return nil
147 }
148 m.syncing = true
149 if m.state == nil {
150 m.state = map[string]entry{}
151 }
152 m.mu.Unlock()
153 defer func() { m.mu.Lock(); m.syncing = false; m.mu.Unlock() }()
154
155 log := zerowrap.FromCtx(ctx)
156 var errs []error
157 keep := map[string]bool{}
158 if err := os.MkdirAll(m.Dir, 0o755); err != nil {
159 return err
160 }
161 for _, name := range m.names() {
162 a, ok := m.artifact(name)
163 var e entry
164 switch {
165 case !ok:
166 e = m.unavailable(name, fmt.Sprintf("there is no %s build for linux-%s", name, m.Arch), false)
167 default:
168 var err error
169 if e, err = m.ensure(ctx, a); err != nil {
170 errs = append(errs, fmt.Errorf("%s %s: %w", a.Name, a.Version, err))
171 log.Warn().Err(err).Str("external", a.Name).Str("version", a.Version).Msg("external tool not available")
172 } else {
173 log.Info().Str("external", a.Name).Str("version", a.Version).Str("arch", a.Arch).Msg("external tool ready")
174 }
175 }
176 m.mu.Lock()
177 m.state[name] = e
178 m.mu.Unlock()
179 keep[filepath.Base(e.ext.Dir)] = true
180 if e.ext.Unavailable {
181 keep[unavailableDir] = true
182 }
183 }
184 m.prune(log, keep)
185 return errors.Join(errs...)
186}
187
188// Externals implements sandbox.ExternalsSource. A tool that has no outcome yet
189// is reported as being downloaded. It has no side effect on the downloads (the
190// retries run on a timer, see Start); the directories of unavailable tools are
191// built at most once and only touched here.
192func (m *Manager) Externals() []sandbox.External {
193 m.mu.Lock()
194 names := m.names()
195 got := make([]entry, len(names))
196 have := make([]bool, len(names))
197 for i, n := range names {
198 got[i], have[i] = m.state[n]
199 }
200 m.mu.Unlock()
201
202 // Outside the lock: the first call for a reason writes files.
203 out := make([]sandbox.External, len(names))
204 for i, n := range names {
205 switch {
206 case !have[i]:
207 got[i] = m.unavailable(n, n+" is still being downloaded on this runner: run the job again in a few minutes", false)
208 case got[i].ext.Unavailable:
209 touch(got[i].ext.Dir) // handed out again: not stale
210 }
211 out[i] = got[i].ext
212 }
213 return out
214}
215
216// names is every artifact name in the table, in table order, once.
217func (m *Manager) names() []string {
218 seen := map[string]bool{}
219 var out []string
220 for _, a := range m.Table {
221 if !seen[a.Name] {
222 seen[a.Name] = true
223 out = append(out, a.Name)
224 }
225 }
226 return out
227}
228
229func (m *Manager) artifact(name string) (Artifact, bool) {
230 for _, a := range m.Table {
231 if a.Name == name && a.Arch == m.Arch {
232 return a, true
233 }
234 }
235 return Artifact{}, false
236}
237
238const unavailableDir = ".unavailable"
239
240// unavailableStub is the bin/node of a tool that is missing: it prints the
241// reason (the file UNAVAILABLE next to bin/) on stderr and fails, so that a job
242// which runs the tool anyway (a node action, a run step) gets the reason rather
243// than "No such file or directory". It does not contain the reason.
244const unavailableStub = "#!/bin/sh\ncat \"$(dirname \"$0\")/../" + sandbox.ExternalUnavailableFile + "\" >&2\nexit 1\n"
245
246// unavailable returns the entry of a tool that is missing: a directory with the
247// reason in ExternalUnavailableFile and the bin/node stub. Its name carries a
248// hash of the reason, so that the ID (and so a VM's disk) changes only when the
249// reason does. The directory is built once per process, complete, under a
250// temporary name and renamed into place; files that exist with the right
251// content are never rewritten, and a missing or different one is replaced
252// through a rename, so a reader (the build of a VM's externals disk, a job that
253// runs the stub) never sees a file half written. Each call touches the
254// directory: the pruning keeps what was handed out in the last hour.
255func (m *Manager) unavailable(name, reason string, retry bool) entry {
256 sum := sha256.Sum256([]byte(name + "\x00" + reason))
257 id := "unavailable-" + hex.EncodeToString(sum[:8])
258 dir := filepath.Join(m.Dir, unavailableDir, name+"-"+id)
259 e := entry{retry: retry, ext: sandbox.External{Name: name, ID: id, Dir: dir, Unavailable: true}}
260
261 m.stubMu.Lock()
262 defer m.stubMu.Unlock()
263 if _, err := os.Lstat(filepath.Join(dir, "bin", "node")); err != nil || !m.stubs[dir] {
264 // Best effort: on failure the entry stays marked unavailable and the
265 // sandbox sees whatever there is.
266 if err := buildStub(dir, reason); err == nil {
267 if m.stubs == nil {
268 m.stubs = map[string]bool{}
269 }
270 m.stubs[dir] = true
271 }
272 }
273 touch(dir)
274 return e
275}
276
277func touch(dir string) {
278 now := time.Now()
279 _ = os.Chtimes(dir, now, now)
280}
281
282// buildStub makes dir hold the UNAVAILABLE file and the bin/node stub, with
283// explicit modes (the umask of the runner must not hide them from the user of
284// the sandbox).
285func buildStub(dir, reason string) error {
286 parent := filepath.Dir(dir)
287 if err := os.MkdirAll(parent, 0o755); err != nil {
288 return err
289 }
290 if err := os.Chmod(parent, 0o755); err != nil {
291 return err
292 }
293 files := []stubFile{
294 {sandbox.ExternalUnavailableFile, reason + "\n", 0o644},
295 {"bin/node", unavailableStub, 0o755},
296 }
297 if _, err := os.Lstat(dir); errors.Is(err, os.ErrNotExist) {
298 tmp, err := os.MkdirTemp(parent, ".tmp-")
299 if err != nil {
300 return err
301 }
302 defer func() { _ = os.RemoveAll(tmp) }()
303 if err := os.Mkdir(filepath.Join(tmp, "bin"), 0o755); err != nil {
304 return err
305 }
306 for _, d := range []string{tmp, filepath.Join(tmp, "bin")} {
307 if err := os.Chmod(d, 0o755); err != nil {
308 return err
309 }
310 }
311 for _, f := range files {
312 if err := f.write(tmp); err != nil {
313 return err
314 }
315 }
316 if err := os.Rename(tmp, dir); err == nil {
317 return nil
318 }
319 // Lost a race with another process, or the directory appeared: repair it.
320 }
321 if err := os.MkdirAll(filepath.Join(dir, "bin"), 0o755); err != nil {
322 return err
323 }
324 for _, f := range files {
325 if f.current(dir) {
326 continue
327 }
328 if err := f.write(dir); err != nil {
329 return err
330 }
331 }
332 return nil
333}
334
335type stubFile struct {
336 name, content string
337 mode os.FileMode
338}
339
340// current reports whether the file exists with this content and mode.
341func (f stubFile) current(dir string) bool {
342 p := filepath.Join(dir, f.name)
343 st, err := os.Lstat(p)
344 if err != nil || !st.Mode().IsRegular() || st.Mode().Perm() != f.mode {
345 return false
346 }
347 b, err := os.ReadFile(p)
348 return err == nil && string(b) == f.content
349}
350
351// write creates the file through a temporary file in the same directory and a
352// rename, never in place.
353func (f stubFile) write(dir string) error {
354 p := filepath.Join(dir, f.name)
355 tmp, err := os.CreateTemp(filepath.Dir(p), ".tmp-")
356 if err != nil {
357 return err
358 }
359 ok := false
360 defer func() {
361 if !ok {
362 _ = os.Remove(tmp.Name())
363 }
364 }()
365 if _, err := tmp.WriteString(f.content); err != nil {
366 _ = tmp.Close()
367 return err
368 }
369 if err := tmp.Chmod(f.mode); err != nil {
370 _ = tmp.Close()
371 return err
372 }
373 if err := tmp.Close(); err != nil {
374 return err
375 }
376 if err := os.Rename(tmp.Name(), p); err != nil {
377 return err
378 }
379 ok = true
380 return nil
381}
382
383// ensure returns the ready entry of a, downloading it when needed. On failure it
384// returns the unavailable entry together with the error.
385func (m *Manager) ensure(ctx context.Context, a Artifact) (entry, error) {
386 final := filepath.Join(m.Dir, a.dirName())
387 if isReady(final) {
388 return readyEntry(a, final), nil
389 }
390 err := m.fetch(ctx, a, final)
391 if err != nil {
392 reason := fmt.Sprintf("%s (Node.js %s) is not available on this runner: %v", a.Name, a.Version, err)
393 var v *verdictError
394 return m.unavailable(a.Name, reason, !errors.As(err, &v)), err
395 }
396 return readyEntry(a, final), nil
397}
398
399func readyEntry(a Artifact, dir string) entry {
400 return entry{ext: sandbox.External{Name: a.Name, ID: a.dirName() + "-" + a.SHA256[:16], Dir: dir}}
401}
402
403// isReady reports whether dir holds a complete tool: a regular executable
404// bin/node. Directories only appear under their final name by atomic rename
405// after a verified extraction.
406func isReady(dir string) bool {
407 st, err := os.Stat(filepath.Join(dir, "bin", "node"))
408 return err == nil && st.Mode().IsRegular() && st.Mode().Perm()&0o111 != 0
409}
410
411// verdictError is a failure that another attempt will not change (the checksum
412// differs, the archive is unsafe).
413type verdictError struct{ error }
414
415func (e *verdictError) Unwrap() error { return e.error }
416
417// fetch downloads a, verifies its checksum, unpacks it next to final and
418// renames it into place.
419func (m *Manager) fetch(ctx context.Context, a Artifact, final string) error {
420 u, err := url.Parse(a.URL)
421 if err != nil || u.Scheme != "https" {
422 return &verdictError{fmt.Errorf("%q is not an https URL", a.URL)}
423 }
424 tmp, err := os.MkdirTemp(m.Dir, ".tmp-")
425 if err != nil {
426 return err
427 }
428 defer func() { _ = os.RemoveAll(tmp) }()
429
430 archive := filepath.Join(tmp, "archive.tar.gz")
431 if err := m.download(ctx, a, archive); err != nil {
432 return err
433 }
434 tree := filepath.Join(tmp, "tree")
435 if err := os.Mkdir(tree, 0o755); err != nil {
436 return err
437 }
438 if err := extractTarGz(archive, tree, a.StripTop); err != nil {
439 return &verdictError{fmt.Errorf("unpack: %w", err)}
440 }
441 if !isReady(tree) {
442 return &verdictError{errors.New("the archive has no executable bin/node")}
443 }
444 if err := os.RemoveAll(final); err != nil { // an incomplete directory of an older run
445 return err
446 }
447 return os.Rename(tree, final)
448}
449
450// download fetches a.URL to dest and verifies its SHA-256 as it arrives.
451func (m *Manager) download(ctx context.Context, a Artifact, dest string) error {
452 client := m.Client
453 if client == nil {
454 client = newClient()
455 }
456 req, err := http.NewRequestWithContext(ctx, http.MethodGet, a.URL, nil)
457 if err != nil {
458 return err
459 }
460 resp, err := client.Do(req)
461 if err != nil {
462 return fmt.Errorf("download %s: %w", a.URL, err)
463 }
464 defer func() { _ = resp.Body.Close() }()
465 if resp.StatusCode != http.StatusOK {
466 return fmt.Errorf("download %s: %s", a.URL, resp.Status)
467 }
468 f, err := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
469 if err != nil {
470 return err
471 }
472 h := sha256.New()
473 n, err := io.Copy(io.MultiWriter(f, h), io.LimitReader(resp.Body, maxDownload+1))
474 if cerr := f.Close(); err == nil {
475 err = cerr
476 }
477 if err != nil {
478 return fmt.Errorf("download %s: %w", a.URL, err)
479 }
480 if n > maxDownload {
481 return &verdictError{fmt.Errorf("download %s: more than %d bytes", a.URL, int64(maxDownload))}
482 }
483 if err := VerifySum(hex.EncodeToString(h.Sum(nil)), a.SHA256); err != nil {
484 return &verdictError{fmt.Errorf("download %s: %w", a.URL, err)}
485 }
486 return nil
487}
488
489// newClient is the default downloader: a time limit, and redirects only to
490// https locations (nodejs.org does not redirect; GitHub releases do).
491func newClient() *http.Client {
492 return &http.Client{
493 Timeout: 15 * time.Minute,
494 CheckRedirect: func(req *http.Request, via []*http.Request) error {
495 if req.URL.Scheme != "https" {
496 return fmt.Errorf("redirect to a non-https URL %s", req.URL.Redacted())
497 }
498 if len(via) >= 5 {
499 return errors.New("too many redirects")
500 }
501 return nil
502 },
503 }
504}
505
506// VerifySum compares two hex SHA-256 values.
507func VerifySum(got, want string) error {
508 if !strings.EqualFold(got, want) {
509 return fmt.Errorf("SHA-256 mismatch: got %s, want %s", got, want)
510 }
511 return nil
512}
513
514// prune removes everything in Dir that is not pinned: other versions, other
515// architectures, leftovers of an interrupted download. The reasons of
516// unavailable tools (a few bytes each) may be bind-mounted into a running
517// container, so those are only removed once they are an hour old.
518func (m *Manager) prune(log zerowrap.Logger, keep map[string]bool) {
519 ents, err := os.ReadDir(m.Dir)
520 if err != nil {
521 return
522 }
523 for _, e := range ents {
524 switch name := e.Name(); {
525 case keep[name]:
526 case name == unavailableDir:
527 m.pruneUnavailable(keep)
528 default:
529 if err := os.RemoveAll(filepath.Join(m.Dir, name)); err != nil {
530 log.Warn().Err(err).Str("path", name).Msg("removing an unpinned external tool")
531 }
532 }
533 }
534}
535
536func (m *Manager) pruneUnavailable(keep map[string]bool) {
537 dir := filepath.Join(m.Dir, unavailableDir)
538 ents, _ := os.ReadDir(dir)
539 for _, e := range ents {
540 fi, err := e.Info()
541 if keep[e.Name()] || err != nil || time.Since(fi.ModTime()) < time.Hour {
542 continue
543 }
544 _ = os.RemoveAll(filepath.Join(dir, e.Name()))
545 }
546}