1// SPDX-License-Identifier: MIT
2
3package main
4
5import (
6 "context"
7 "errors"
8 "flag"
9 "fmt"
10 "net"
11 "os"
12 "path/filepath"
13 "strings"
14 "time"
15
16 "github.com/bnema/zerowrap"
17
18 "bnema.dev/ermite-api/gen/ermite/ermiteconnect"
19
20 "bnema.dev/ermite-runner/internal/actionscache"
21 "bnema.dev/ermite-runner/internal/cachesvc"
22 "bnema.dev/ermite-runner/internal/externals"
23 "bnema.dev/ermite-runner/internal/host"
24 "bnema.dev/ermite-runner/internal/resultssvc"
25 "bnema.dev/ermite-runner/internal/sandbox"
26 "bnema.dev/ermite-runner/internal/sandbox/drivers"
27)
28
29// loadHostConfig reads the sandbox settings and the host settings together, so
30// that the state directory and the capacity are the same for both.
31func loadHostConfig() (sandbox.Config, host.Config, error) {
32 scfg, err := sandbox.LoadConfig(os.Getenv)
33 if err != nil {
34 return scfg, host.Config{}, err
35 }
36 name, _ := os.Hostname()
37 hcfg, err := host.LoadConfig(os.Getenv, name, scfg.StateDir, scfg.Capacity, version)
38 return scfg, hcfg, err
39}
40
41// setupCache starts the cache service of the runner (README, "Cache") and hands
42// it to the sandbox drivers through scfg. ERMITE_RUNNER_CACHE=0 turns it off;
43// ERMITE_RUNNER_CACHE_MAX is the size cap (default 10 GiB).
44func setupCache(ctx context.Context, scfg *sandbox.Config) (*cachesvc.Service, *actionscache.Cache, error) {
45 log := zerowrap.FromCtx(ctx)
46 switch strings.ToLower(strings.TrimSpace(os.Getenv(envCache))) {
47 case "", "1", "true", "yes":
48 case "0", "false", "no":
49 log.Info().Msg("the cache service is off (" + envCache + "=0)")
50 return nil, nil, nil
51 default:
52 return nil, nil, fmt.Errorf("%s: %q is not 0 or 1", envCache, os.Getenv(envCache))
53 }
54 max, err := cachesvc.ParseMax(os.Getenv(cachesvc.EnvMax))
55 if err != nil {
56 return nil, nil, err
57 }
58 archives, err := setupArchives(ctx, scfg.StateDir)
59 if err != nil {
60 return nil, nil, err
61 }
62 svc, err := cachesvc.New(cachesvc.Config{Dir: filepath.Join(scfg.StateDir, "cache"), Max: max, Log: log, Archives: archives})
63 if err != nil {
64 return nil, nil, err
65 }
66 scfg.Cache = svc
67 log.Info().Int64("max_bytes", max).Msg("cache service on (guest loopback only, no host port)")
68 return svc, archives, nil
69}
70
71// setupArchives opens the cache of action archives (ERMITE_RUNNER_ACTIONS_CACHE_MAX,
72// default 2 GiB, 0 turns it off). It is served on the channel of the cache
73// service, so it is only used when that is on.
74func setupArchives(ctx context.Context, stateDir string) (*actionscache.Cache, error) {
75 log := zerowrap.FromCtx(ctx)
76 max, err := actionscache.ParseMax(os.Getenv(actionscache.EnvMax))
77 if err != nil {
78 return nil, err
79 }
80 if max == 0 {
81 log.Info().Msg("the action archive cache is off (" + actionscache.EnvMax + "=0)")
82 return nil, nil
83 }
84 c, err := actionscache.New(actionscache.Config{Dir: filepath.Join(stateDir, "actions-cache"), Max: max, Log: log})
85 if err != nil {
86 return nil, err
87 }
88 log.Info().Int64("max_bytes", max).Msg("action archive cache on")
89 return c, nil
90}
91
92// envCache=0 turns the cache service off.
93const envCache = "ERMITE_RUNNER_CACHE"
94
95// envArtifacts=0 turns the results service (artifacts v4) off.
96const envArtifacts = "ERMITE_RUNNER_ARTIFACTS"
97
98// setupResults makes the results service of the runner (README, "Artifacts"):
99// on by default, off with ERMITE_RUNNER_ARTIFACTS=0. It is served on the cache
100// channel of each sandbox and checks the cache's runtime tokens, so it needs
101// the cache service; a job gets it only when the server has ci.artifacts.
102func setupResults(ctx context.Context, cache *cachesvc.Service) (*resultssvc.Service, error) {
103 log := zerowrap.FromCtx(ctx)
104 switch strings.ToLower(strings.TrimSpace(os.Getenv(envArtifacts))) {
105 case "", "1", "true", "yes":
106 case "0", "false", "no":
107 log.Info().Msg("the results service is off (" + envArtifacts + "=0): artifact actions fail")
108 return nil, nil
109 default:
110 return nil, fmt.Errorf("%s: %q is not 0 or 1", envArtifacts, os.Getenv(envArtifacts))
111 }
112 if cache == nil {
113 log.Warn().Msg("the results service needs the cache service (" + envCache + "=0): artifact actions fail")
114 return nil, nil
115 }
116 log.Info().Msg("results service on (artifacts v4, on the cache channel; stored by ermite)")
117 return resultssvc.New(resultssvc.Config{Auth: cache, Log: log}), nil
118}
119
120// externalsWait is how long the start of the runner waits for the Node.js
121// downloads. A runner with a working network is then ready for its first job; one
122// without goes on after the first failure (nothing waits for a dead network), and
123// the downloads that are still running when the time is up finish in the
124// background.
125const externalsWait = 2 * time.Minute
126
127// setupExternals downloads the Node.js externals (README, "Externals") and hands
128// them to the sandbox drivers through scfg. It never fails the start: whatever
129// cannot be provided is reported to the jobs that need it.
130func setupExternals(ctx context.Context, scfg *sandbox.Config) {
131 log := zerowrap.FromCtx(ctx)
132 if !scfg.NodeExternals {
133 log.Info().Msg("Node.js externals are off (" + sandbox.EnvNode + "=0)")
134 return
135 }
136 m, ok := externals.New(scfg.StateDir)
137 if !ok {
138 log.Warn().Msg("no Node.js builds for this architecture: jobs get no externals")
139 return
140 }
141 m.Start(ctx, externalsWait)
142 scfg.Externals = m
143}
144
145func buildDriver(ctx context.Context, scfg sandbox.Config) (sandbox.Driver, error) {
146 return sandbox.Select(ctx, scfg, drivers.Factories(), sandbox.Prober{})
147}
148
149// registerCmd is `ermite-runner register [--token-stdin] [--token T] [--replace]`:
150// it chooses the isolation mode like `run` will, registers the runner with it,
151// and stores the runner token (mode 0600) in the state directory. The register
152// token is read from standard input (--token-stdin) or from
153// ERMITE_RUNNER_REGISTER_TOKEN; --token is accepted but the arguments of a
154// process are visible to every user of the host.
155func registerCmd(ctx context.Context, args []string) int {
156 fs := flag.NewFlagSet("register", flag.ContinueOnError)
157 token := fs.String("token", "", "token with the runner:register scope (visible in the process list: prefer $"+host.EnvRegisterToken+" or --token-stdin)")
158 fromStdin := fs.Bool("token-stdin", false, "read the token from standard input")
159 replace := fs.Bool("replace", false, "replace a runner of the same name")
160 if err := fs.Parse(args); err != nil {
161 return 2
162 }
163 tok, err := host.ResolveRegisterToken(*token, *fromStdin, os.Stdin, os.Getenv)
164 if err != nil {
165 fmt.Fprintln(os.Stderr, "ermite-runner:", err)
166 return 2
167 }
168 *token = tok
169 scfg, hcfg, err := loadHostConfig()
170 if err != nil {
171 fmt.Fprintln(os.Stderr, "ermite-runner:", err)
172 return 2
173 }
174 if hcfg.URL == "" {
175 fmt.Fprintf(os.Stderr, "ermite-runner: %s is not set\n", host.EnvURL)
176 return 2
177 }
178 drv, err := buildDriver(ctx, scfg)
179 if err != nil {
180 fmt.Fprintln(os.Stderr, "ermite-runner:", err)
181 return 1
182 }
183 iso := drv.Isolation()
184 _ = drv.Close()
185
186 client := hcfg.Client(hcfg.URL, *token)
187 st, err := host.Register(ctx, hcfg, client, iso, *replace)
188 if err != nil {
189 fmt.Fprintln(os.Stderr, "ermite-runner:", err)
190 return 1
191 }
192 fmt.Printf("registered runner %q (%s) with isolation %s; token stored in %s\n", st.Name, st.RunnerID, iso, host.StateFile(hcfg.StateDir))
193 return 0
194}
195
196// registerOnFirstRun registers the runner when `run` finds no registration and
197// a register token is given in ERMITE_RUNNER_REGISTER_TOKEN or in the file
198// named by ERMITE_RUNNER_REGISTER_TOKEN_FILE. Without a token it returns
199// host.ErrNotRegistered, as before.
200func registerOnFirstRun(ctx context.Context, scfg sandbox.Config, hcfg host.Config) (host.State, error) {
201 tok, err := firstRunToken(os.Getenv)
202 if err != nil || tok == "" {
203 return host.State{}, errors.Join(host.ErrNotRegistered, err)
204 }
205 if hcfg.URL == "" {
206 return host.State{}, fmt.Errorf("%s is not set", host.EnvURL)
207 }
208 drv, err := buildDriver(ctx, scfg)
209 if err != nil {
210 return host.State{}, err
211 }
212 iso := drv.Isolation()
213 _ = drv.Close()
214 st, err := host.Register(ctx, hcfg, hcfg.Client(hcfg.URL, tok), iso, false)
215 if err != nil {
216 return host.State{}, fmt.Errorf("first-run registration: %w", err)
217 }
218 log := zerowrap.FromCtx(ctx)
219 log.Info().Str("runner", st.Name).Str("isolation", string(iso)).Msg("registered on first run; the register token is no longer needed")
220 return st, nil
221}
222
223// envRegisterTokenFile names a file holding the register token (a container
224// secret delivered as a file).
225const envRegisterTokenFile = "ERMITE_RUNNER_REGISTER_TOKEN_FILE"
226
227// firstRunToken reads the register token of a first run: the file of
228// envRegisterTokenFile when set, else host.EnvRegisterToken. "" means none.
229func firstRunToken(getenv func(string) string) (string, error) {
230 if p := strings.TrimSpace(getenv(envRegisterTokenFile)); p != "" {
231 b, err := os.ReadFile(p)
232 if err != nil {
233 return "", fmt.Errorf("%s: %w", envRegisterTokenFile, err)
234 }
235 return strings.TrimSpace(string(b)), nil
236 }
237 return strings.TrimSpace(getenv(host.EnvRegisterToken)), nil
238}
239
240// runCmd is `ermite-runner run`: poll ermite for jobs and run each one in its
241// own sandbox until SIGINT or SIGTERM, then cancel the running jobs with
242// runner_shutdown and report them.
243func runCmd(ctx context.Context) int {
244 log := zerowrap.FromCtx(ctx)
245 scfg, hcfg, err := loadHostConfig()
246 if err != nil {
247 fmt.Fprintln(os.Stderr, "ermite-runner:", err)
248 return 2
249 }
250 st, err := host.LoadState(hcfg.StateDir)
251 if errors.Is(err, host.ErrNotRegistered) {
252 // A container service has no separate install step: it registers on its
253 // first start when it is given a register token.
254 st, err = registerOnFirstRun(ctx, scfg, hcfg)
255 }
256 if err != nil {
257 fmt.Fprintln(os.Stderr, "ermite-runner:", err)
258 return 2
259 }
260 url := hcfg.URL
261 if url == "" {
262 url = st.URL
263 }
264 if url == "" {
265 fmt.Fprintf(os.Stderr, "ermite-runner: %s is not set\n", host.EnvURL)
266 return 2
267 }
268 if scfg.GuestForgeEnabled {
269 // The sandboxes reach the forge the way the runner does: by the host name
270 // and port of the URL the runner itself uses.
271 gf, err := sandbox.ResolveGuestForge(ctx, url, os.Getenv(sandbox.EnvGuestForgeTarget), hcfg.InsecureHTTP, net.DefaultResolver.LookupHost)
272 if err != nil {
273 fmt.Fprintln(os.Stderr, "ermite-runner:", err)
274 return 2
275 }
276 scfg.GuestForge = gf
277 hcfg.GuestForgeOrigin = gf.Origin
278 }
279 setupExternals(ctx, &scfg)
280 cache, archives, err := setupCache(ctx, &scfg)
281 if err != nil {
282 fmt.Fprintln(os.Stderr, "ermite-runner:", err)
283 return 2
284 }
285 if cache != nil {
286 defer func() { _ = cache.CloseAll() }()
287 }
288 results, err := setupResults(ctx, cache)
289 if err != nil {
290 fmt.Fprintln(os.Stderr, "ermite-runner:", err)
291 return 2
292 }
293 drv, err := buildDriver(ctx, scfg)
294 if err != nil {
295 fmt.Fprintln(os.Stderr, "ermite-runner:", err)
296 return 1
297 }
298 defer func() { _ = drv.Close() }()
299 log.Info().Str("runner", st.Name).Str("isolation", string(drv.Isolation())).Int("capacity", scfg.Capacity).Msg("runner started")
300
301 r := &host.Runner{
302 Cfg: hcfg,
303 Driver: drv,
304 Poll: hcfg.Client(url, st.RunnerToken),
305 Job: func(tok string) ermiteconnect.RunnerServiceClient {
306 return hcfg.Client(url, tok)
307 },
308 Features: hcfg.ServerFeaturesFunc(url),
309 Cache: cache,
310 Archives: archives,
311 Results: results,
312 Artifacts: func(tok string) ermiteconnect.JobArtifactServiceClient {
313 return hcfg.ArtifactClient(url, tok)
314 },
315 }
316 r.Cfg.URL = url
317 if err := r.Run(ctx); err != nil {
318 if errors.Is(err, host.ErrRemoved) {
319 fmt.Fprintln(os.Stderr, "ermite-runner:", err)
320 return 3
321 }
322 fmt.Fprintln(os.Stderr, "ermite-runner:", err)
323 return 1
324 }
325 log.Info().Msg("runner stopped")
326 return 0
327}