1// SPDX-License-Identifier: MIT
2
3package main
4
5import (
6 "context"
7 "errors"
8 "flag"
9 "fmt"
10 "net"
11 "net/http"
12 "os"
13 "path/filepath"
14 "strings"
15 "time"
16
17 "github.com/bnema/zerowrap"
18
19 "bnema.dev/ermite-api/gen/ermite/ermiteconnect"
20
21 "bnema.dev/ermite-runner/internal/actionscache"
22 "bnema.dev/ermite-runner/internal/cachesvc"
23 "bnema.dev/ermite-runner/internal/externals"
24 "bnema.dev/ermite-runner/internal/host"
25 "bnema.dev/ermite-runner/internal/registryproxy"
26 "bnema.dev/ermite-runner/internal/resultssvc"
27 "bnema.dev/ermite-runner/internal/sandbox"
28 "bnema.dev/ermite-runner/internal/sandbox/drivers"
29)
30
31// loadHostConfig reads the sandbox settings and the host settings together, so
32// that the state directory and the capacity are the same for both.
33func loadHostConfig() (sandbox.Config, host.Config, error) {
34 scfg, err := sandbox.LoadConfig(os.Getenv)
35 if err != nil {
36 return scfg, host.Config{}, err
37 }
38 name, _ := os.Hostname()
39 hcfg, err := host.LoadConfig(os.Getenv, name, scfg.StateDir, scfg.Capacity, version)
40 return scfg, hcfg, err
41}
42
43// setupCache starts the cache service of the runner (README, "Cache") and hands
44// it to the sandbox drivers through scfg. ERMITE_RUNNER_CACHE=0 turns it off;
45// ERMITE_RUNNER_CACHE_MAX is the size cap (default 10 GiB).
46func setupCache(ctx context.Context, scfg *sandbox.Config) (*cachesvc.Service, *actionscache.Cache, error) {
47 log := zerowrap.FromCtx(ctx)
48 switch strings.ToLower(strings.TrimSpace(os.Getenv(envCache))) {
49 case "", "1", "true", "yes":
50 case "0", "false", "no":
51 log.Info().Msg("the cache service and registry proxy are off (" + envCache + "=0)")
52 return nil, nil, nil
53 default:
54 return nil, nil, fmt.Errorf("%s: %q is not 0 or 1", envCache, os.Getenv(envCache))
55 }
56 // The cache API cannot be turned off by its size: 0 is refused.
57 max, err := cachesvc.ParseSize(os.Getenv(cachesvc.EnvMax), cachesvc.DefaultMax)
58 if err == nil && max == 0 {
59 err = errors.New("0 is not a size cap; " + envCache + "=0 turns the cache off")
60 }
61 if err != nil {
62 return nil, nil, fmt.Errorf("%s: %w", cachesvc.EnvMax, err)
63 }
64 archives, err := setupArchives(ctx, scfg.StateDir)
65 if err != nil {
66 return nil, nil, err
67 }
68 registry, err := setupRegistry(ctx, scfg.StateDir)
69 if err != nil {
70 return nil, nil, err
71 }
72 svc, err := cachesvc.New(cachesvc.Config{Dir: filepath.Join(scfg.StateDir, "cache"), Max: max, Log: log, Archives: archives, Registry: registry})
73 if err != nil {
74 return nil, nil, err
75 }
76 scfg.Cache = svc
77 log.Info().Int64("max_bytes", max).Msg("cache service on (guest loopback only, no host port)")
78 return svc, archives, nil
79}
80
81// setupArchives opens the cache of action archives (ERMITE_RUNNER_ACTIONS_CACHE_MAX,
82// default 2 GiB, 0 turns it off). It is served on the channel of the cache
83// service, so it is only used when that is on.
84func setupArchives(ctx context.Context, stateDir string) (*actionscache.Cache, error) {
85 log := zerowrap.FromCtx(ctx)
86 max, err := cachesvc.ParseSize(os.Getenv(actionscache.EnvMax), actionscache.DefaultMax)
87 if err != nil {
88 return nil, fmt.Errorf("%s: %w", actionscache.EnvMax, err)
89 }
90 if max == 0 {
91 log.Info().Msg("the action archive cache is off (" + actionscache.EnvMax + "=0)")
92 return nil, nil
93 }
94 c, err := actionscache.New(actionscache.Config{Dir: filepath.Join(stateDir, "actions-cache"), Max: max, Log: log})
95 if err != nil {
96 return nil, err
97 }
98 log.Info().Int64("max_bytes", max).Msg("action archive cache on")
99 return c, nil
100}
101
102// envRegistryMax is the size cap of the Docker Hub mirror (default
103// defaultRegistryMax); 0 turns it off.
104const envRegistryMax = "ERMITE_RUNNER_REGISTRY_CACHE_MAX"
105
106const defaultRegistryMax = 10 << 30
107
108// setupRegistry opens the Docker Hub mirror. The caller only calls this when
109// the cache channel is enabled; its capacity is independent of the cache API.
110func setupRegistry(ctx context.Context, stateDir string) (http.Handler, error) {
111 log := zerowrap.FromCtx(ctx)
112 max, err := cachesvc.ParseSize(os.Getenv(envRegistryMax), defaultRegistryMax)
113 if err != nil {
114 return nil, fmt.Errorf("%s: %w", envRegistryMax, err)
115 }
116 if max == 0 {
117 log.Info().Msg("registry proxy off (" + envRegistryMax + "=0)")
118 return nil, nil
119 }
120 p, err := registryproxy.New(registryproxy.Config{Dir: filepath.Join(stateDir, "registry"), Max: max, Log: log})
121 if err != nil {
122 return nil, err
123 }
124 log.Info().Int64("max_bytes", max).Msg("registry proxy on (Docker Hub pull-through)")
125 return p, nil
126}
127
128// envCache=0 turns the cache service off.
129const envCache = "ERMITE_RUNNER_CACHE"
130
131// envArtifacts=0 turns the results service (artifacts v4) off.
132const envArtifacts = "ERMITE_RUNNER_ARTIFACTS"
133
134// setupResults makes the results service of the runner (README, "Artifacts"):
135// on by default, off with ERMITE_RUNNER_ARTIFACTS=0. It is served on the cache
136// channel of each sandbox and checks the cache's runtime tokens, so it needs
137// the cache service; a job gets it only when the server has ci.artifacts.
138func setupResults(ctx context.Context, cache *cachesvc.Service) (*resultssvc.Service, error) {
139 log := zerowrap.FromCtx(ctx)
140 switch strings.ToLower(strings.TrimSpace(os.Getenv(envArtifacts))) {
141 case "", "1", "true", "yes":
142 case "0", "false", "no":
143 log.Info().Msg("the results service is off (" + envArtifacts + "=0): artifact actions fail")
144 return nil, nil
145 default:
146 return nil, fmt.Errorf("%s: %q is not 0 or 1", envArtifacts, os.Getenv(envArtifacts))
147 }
148 if cache == nil {
149 log.Warn().Msg("the results service needs the cache service (" + envCache + "=0): artifact actions fail")
150 return nil, nil
151 }
152 log.Info().Msg("results service on (artifacts v4, on the cache channel; stored by ermite)")
153 return resultssvc.New(resultssvc.Config{Auth: cache, Log: log}), nil
154}
155
156// externalsWait is how long the start of the runner waits for the Node.js
157// downloads. A runner with a working network is then ready for its first job; one
158// without goes on after the first failure (nothing waits for a dead network), and
159// the downloads that are still running when the time is up finish in the
160// background.
161const externalsWait = 2 * time.Minute
162
163// setupExternals downloads the Node.js externals (README, "Externals") and hands
164// them to the sandbox drivers through scfg. It never fails the start: whatever
165// cannot be provided is reported to the jobs that need it.
166func setupExternals(ctx context.Context, scfg *sandbox.Config) {
167 log := zerowrap.FromCtx(ctx)
168 if !scfg.NodeExternals {
169 log.Info().Msg("Node.js externals are off (" + sandbox.EnvNode + "=0)")
170 return
171 }
172 m, ok := externals.New(scfg.StateDir)
173 if !ok {
174 log.Warn().Msg("no Node.js builds for this architecture: jobs get no externals")
175 return
176 }
177 m.Start(ctx, externalsWait)
178 scfg.Externals = m
179}
180
181func buildDriver(ctx context.Context, scfg sandbox.Config) (sandbox.Driver, error) {
182 return sandbox.Select(ctx, scfg, drivers.Factories(), sandbox.Prober{})
183}
184
185// registerCmd is `ermite-runner register [--token-stdin] [--token T] [--replace]`:
186// it chooses the isolation mode like `run` will, registers the runner with it,
187// and stores the runner token (mode 0600) in the state directory. The register
188// token is read from standard input (--token-stdin) or from
189// ERMITE_RUNNER_REGISTER_TOKEN; --token is accepted but the arguments of a
190// process are visible to every user of the host.
191func registerCmd(ctx context.Context, args []string) int {
192 fs := flag.NewFlagSet("register", flag.ContinueOnError)
193 token := fs.String("token", "", "token with the runner:register scope (visible in the process list: prefer $"+host.EnvRegisterToken+" or --token-stdin)")
194 fromStdin := fs.Bool("token-stdin", false, "read the token from standard input")
195 replace := fs.Bool("replace", false, "replace a runner of the same name")
196 if err := fs.Parse(args); err != nil {
197 return 2
198 }
199 tok, err := host.ResolveRegisterToken(*token, *fromStdin, os.Stdin, os.Getenv)
200 if err != nil {
201 fmt.Fprintln(os.Stderr, "ermite-runner:", err)
202 return 2
203 }
204 *token = tok
205 scfg, hcfg, err := loadHostConfig()
206 if err != nil {
207 fmt.Fprintln(os.Stderr, "ermite-runner:", err)
208 return 2
209 }
210 if hcfg.URL == "" {
211 fmt.Fprintf(os.Stderr, "ermite-runner: %s is not set\n", host.EnvURL)
212 return 2
213 }
214 drv, err := buildDriver(ctx, scfg)
215 if err != nil {
216 fmt.Fprintln(os.Stderr, "ermite-runner:", err)
217 return 1
218 }
219 iso := drv.Isolation()
220 _ = drv.Close()
221
222 client := hcfg.Client(hcfg.URL, *token)
223 st, err := host.Register(ctx, hcfg, client, iso, *replace)
224 if err != nil {
225 fmt.Fprintln(os.Stderr, "ermite-runner:", err)
226 return 1
227 }
228 fmt.Printf("registered runner %q (%s) with isolation %s; token stored in %s\n", st.Name, st.RunnerID, iso, host.StateFile(hcfg.StateDir))
229 return 0
230}
231
232// registerOnFirstRun registers the runner when `run` finds no registration and
233// a register token is given in ERMITE_RUNNER_REGISTER_TOKEN or in the file
234// named by ERMITE_RUNNER_REGISTER_TOKEN_FILE. Without a token it returns
235// host.ErrNotRegistered, as before.
236func registerOnFirstRun(ctx context.Context, scfg sandbox.Config, hcfg host.Config) (host.State, error) {
237 tok, err := firstRunToken(os.Getenv)
238 if err != nil || tok == "" {
239 return host.State{}, errors.Join(host.ErrNotRegistered, err)
240 }
241 if hcfg.URL == "" {
242 return host.State{}, fmt.Errorf("%s is not set", host.EnvURL)
243 }
244 drv, err := buildDriver(ctx, scfg)
245 if err != nil {
246 return host.State{}, err
247 }
248 iso := drv.Isolation()
249 _ = drv.Close()
250 st, err := host.Register(ctx, hcfg, hcfg.Client(hcfg.URL, tok), iso, false)
251 if err != nil {
252 return host.State{}, fmt.Errorf("first-run registration: %w", err)
253 }
254 log := zerowrap.FromCtx(ctx)
255 log.Info().Str("runner", st.Name).Str("isolation", string(iso)).Msg("registered on first run; the register token is no longer needed")
256 return st, nil
257}
258
259// envRegisterTokenFile names a file holding the register token (a container
260// secret delivered as a file).
261const envRegisterTokenFile = "ERMITE_RUNNER_REGISTER_TOKEN_FILE"
262
263// firstRunToken reads the register token of a first run: the file of
264// envRegisterTokenFile when set, else host.EnvRegisterToken. "" means none.
265func firstRunToken(getenv func(string) string) (string, error) {
266 if p := strings.TrimSpace(getenv(envRegisterTokenFile)); p != "" {
267 b, err := os.ReadFile(p)
268 if err != nil {
269 return "", fmt.Errorf("%s: %w", envRegisterTokenFile, err)
270 }
271 return strings.TrimSpace(string(b)), nil
272 }
273 return strings.TrimSpace(getenv(host.EnvRegisterToken)), nil
274}
275
276// runCmd is `ermite-runner run`: poll ermite for jobs and run each one in its
277// own sandbox until SIGINT or SIGTERM, then cancel the running jobs with
278// runner_shutdown and report them.
279func runCmd(ctx context.Context) int {
280 log := zerowrap.FromCtx(ctx)
281 scfg, hcfg, err := loadHostConfig()
282 if err != nil {
283 fmt.Fprintln(os.Stderr, "ermite-runner:", err)
284 return 2
285 }
286 st, err := host.LoadState(hcfg.StateDir)
287 if errors.Is(err, host.ErrNotRegistered) {
288 // A container service has no separate install step: it registers on its
289 // first start when it is given a register token.
290 st, err = registerOnFirstRun(ctx, scfg, hcfg)
291 }
292 if err != nil {
293 fmt.Fprintln(os.Stderr, "ermite-runner:", err)
294 return 2
295 }
296 url := hcfg.URL
297 if url == "" {
298 url = st.URL
299 }
300 if url == "" {
301 fmt.Fprintf(os.Stderr, "ermite-runner: %s is not set\n", host.EnvURL)
302 return 2
303 }
304 if scfg.GuestForgeEnabled {
305 // The sandboxes reach the forge the way the runner does: by the host name
306 // and port of the URL the runner itself uses.
307 gf, err := sandbox.ResolveGuestForge(ctx, url, os.Getenv(sandbox.EnvGuestForgeTarget), hcfg.InsecureHTTP, net.DefaultResolver.LookupHost)
308 if err != nil {
309 fmt.Fprintln(os.Stderr, "ermite-runner:", err)
310 return 2
311 }
312 scfg.GuestForge = gf
313 hcfg.GuestForgeOrigin = gf.Origin
314 }
315 setupExternals(ctx, &scfg)
316 cache, archives, err := setupCache(ctx, &scfg)
317 if err != nil {
318 fmt.Fprintln(os.Stderr, "ermite-runner:", err)
319 return 2
320 }
321 if cache != nil {
322 defer func() { _ = cache.CloseAll() }()
323 }
324 results, err := setupResults(ctx, cache)
325 if err != nil {
326 fmt.Fprintln(os.Stderr, "ermite-runner:", err)
327 return 2
328 }
329 drv, err := buildDriver(ctx, scfg)
330 if err != nil {
331 fmt.Fprintln(os.Stderr, "ermite-runner:", err)
332 return 1
333 }
334 defer func() { _ = drv.Close() }()
335 log.Info().Str("runner", st.Name).Str("isolation", string(drv.Isolation())).Int("capacity", scfg.Capacity).Msg("runner started")
336
337 r := &host.Runner{
338 Cfg: hcfg,
339 Driver: drv,
340 Poll: hcfg.Client(url, st.RunnerToken),
341 Job: func(tok string) ermiteconnect.RunnerServiceClient {
342 return hcfg.Client(url, tok)
343 },
344 Features: hcfg.ServerFeaturesFunc(url),
345 Cache: cache,
346 Archives: archives,
347 Results: results,
348 Artifacts: func(tok string) ermiteconnect.JobArtifactServiceClient {
349 return hcfg.ArtifactClient(url, tok)
350 },
351 }
352 r.Cfg.URL = url
353 if err := r.Run(ctx); err != nil {
354 if errors.Is(err, host.ErrRemoved) {
355 fmt.Fprintln(os.Stderr, "ermite-runner:", err)
356 return 3
357 }
358 fmt.Fprintln(os.Stderr, "ermite-runner:", err)
359 return 1
360 }
361 log.Info().Msg("runner stopped")
362 return 0
363}