1// SPDX-License-Identifier: MIT
2
3package guest
4
5import (
6 "context"
7 "errors"
8 "fmt"
9 "io"
10 "os"
11 "os/exec"
12 "sync"
13 "syscall"
14 "time"
15
16 "bnema.dev/ermite-runner/internal/execstream"
17 "bnema.dev/ermite-runner/internal/execstream/execpb"
18)
19
20// DefaultScript is what the stub executor runs when the host sends no script.
21const DefaultScript = "uname -a; id; cat /etc/os-release; docker info"
22
23// Cancellation timings, like the official runner's ProcessInvoker: SIGINT, then
24// SIGTERM after 7.5 s, then SIGKILL 2.5 s later.
25const (
26 DefaultTermAfter = 7500 * time.Millisecond
27 DefaultKillAfter = 2500 * time.Millisecond
28 // DefaultWaitDelay bounds how long the executor waits for the output pipes
29 // to close after the script's shell exited (background children may hold
30 // them open).
31 DefaultWaitDelay = 500 * time.Millisecond
32)
33
34// ExecConfig configures the executor.
35type ExecConfig struct {
36 Target Target
37 Sandbox string // "vm" or "container", reported in Hello
38 Version string
39 // Shell runs the script as `Shell -c script`; default /bin/sh.
40 Shell string
41 // StepUser is the account the steps of a job run as when the executor is
42 // root; default "runner".
43 StepUser string
44 // AllowRootSteps lets the steps run as root when StepUser does not exist.
45 // For test images only; set by ERMITE_RUNNER_ALLOW_ROOT_STEPS=1 in the
46 // executor's environment.
47 AllowRootSteps bool
48 // ConnectTimeout bounds the connection attempts (default 30 s).
49 ConnectTimeout time.Duration
50 // RunTimeout bounds the wait for the host's Run frame (default 60 s).
51 RunTimeout time.Duration
52 // TermAfter and KillAfter are the escalation delays after a Cancel (or after
53 // the executor's own context ends): SIGINT at once, SIGTERM after TermAfter,
54 // SIGKILL KillAfter later. WaitDelay is the pipe-drain grace after the shell
55 // exits. Zero means the defaults above.
56 TermAfter, KillAfter, WaitDelay time.Duration
57
58 // signal sends a signal to a process group; tests replace it to observe the
59 // escalation. Default: syscall.Kill(-pgid, sig).
60 signal func(pgid int, sig syscall.Signal)
61}
62
63func (c *ExecConfig) defaults() {
64 if c.Shell == "" {
65 c.Shell = "/bin/sh"
66 }
67 if c.StepUser == "" {
68 c.StepUser = "runner"
69 }
70 if c.ConnectTimeout == 0 {
71 c.ConnectTimeout = 30 * time.Second
72 }
73 if c.RunTimeout == 0 {
74 c.RunTimeout = 60 * time.Second
75 }
76 if c.TermAfter == 0 {
77 c.TermAfter = DefaultTermAfter
78 }
79 if c.KillAfter == 0 {
80 c.KillAfter = DefaultKillAfter
81 }
82 if c.WaitDelay == 0 {
83 c.WaitDelay = DefaultWaitDelay
84 }
85 if c.signal == nil {
86 c.signal = func(pgid int, sig syscall.Signal) { _ = syscall.Kill(-pgid, sig) }
87 }
88}
89
90// Exec is `ermite-runner exec`: it connects back to the host and receives one
91// Run. A Run with a job is executed by package executor (see runJob); a Run
92// without one is the stub of sandbox-check, which runs the script with the shell, streams stdout and
93// stderr as Output frames, and ends with an Exit frame. It returns the exit code
94// it reported. A Cancel frame, or the end of ctx, stops the script's process
95// group with escalating signals (see ExecConfig).
96func Exec(ctx context.Context, cfg ExecConfig) (int, error) {
97 cfg.defaults()
98 // The steps inherit the executor's umask: GitHub's 022, whatever started
99 // the executor (Init sets it too; a container's entrypoint may not).
100 syscall.Umask(StepUmask)
101 raw, err := cfg.Target.Dial(ctx, cfg.ConnectTimeout)
102 if err != nil {
103 return 1, err
104 }
105 defer func() { _ = raw.Close() }()
106 return serve(ctx, raw, cfg)
107}
108
109// frameWriter turns writes into Output frames of at most ChunkSize bytes. It is
110// the command's Stdout/Stderr, so no goroutine of ours sits between the pipe
111// and the stream.
112type frameWriter struct {
113 conn *execstream.Conn
114 kind execpb.StreamKind
115}
116
117func (w frameWriter) Write(p []byte) (int, error) {
118 n := 0
119 for len(p) > 0 {
120 chunk := p
121 if len(chunk) > execstream.ChunkSize {
122 chunk = chunk[:execstream.ChunkSize]
123 }
124 err := w.conn.Send(&execpb.Frame{Body: &execpb.Frame_Output{Output: &execpb.Output{
125 Stream: w.kind, Data: append([]byte(nil), chunk...), UnixNano: time.Now().UnixNano(),
126 }}})
127 if err != nil {
128 return n, err
129 }
130 n += len(chunk)
131 p = p[len(chunk):]
132 }
133 return n, nil
134}
135
136// escalator signals a process group and escalates after a cancel. Once finish
137// has been called (the shell was reaped) it never signals again: the group id
138// could have been reused by then.
139type escalator struct {
140 cfg ExecConfig
141 pgid int
142
143 mu sync.Mutex
144 finished bool
145 started bool
146 timers []*time.Timer
147}
148
149func (e *escalator) send(sig syscall.Signal) {
150 e.mu.Lock()
151 defer e.mu.Unlock()
152 if !e.finished {
153 e.cfg.signal(e.pgid, sig)
154 }
155}
156
157// cancel starts the escalation; only the first call does anything.
158func (e *escalator) cancel() {
159 e.mu.Lock()
160 if e.finished || e.started {
161 e.mu.Unlock()
162 return
163 }
164 e.started = true
165 e.cfg.signal(e.pgid, syscall.SIGINT)
166 e.timers = append(e.timers,
167 time.AfterFunc(e.cfg.TermAfter, func() { e.send(syscall.SIGTERM) }),
168 time.AfterFunc(e.cfg.TermAfter+e.cfg.KillAfter, func() { e.send(syscall.SIGKILL) }),
169 )
170 e.mu.Unlock()
171}
172
173// finish is called right after the shell was reaped: it kills what is left of
174// the group (background children), stops the timers and forbids further signals.
175func (e *escalator) finish() {
176 e.mu.Lock()
177 defer e.mu.Unlock()
178 if e.finished {
179 return
180 }
181 e.cfg.signal(e.pgid, syscall.SIGKILL)
182 e.finished = true
183 for _, t := range e.timers {
184 t.Stop()
185 }
186}
187
188func serve(ctx context.Context, rw io.ReadWriteCloser, cfg ExecConfig) (int, error) {
189 conn := execstream.NewConn(rw)
190 host, _ := os.Hostname()
191 if err := conn.Send(&execpb.Frame{Body: &execpb.Frame_Hello{Hello: &execpb.Hello{
192 Version: cfg.Version, Sandbox: cfg.Sandbox, Hostname: host,
193 }}}); err != nil {
194 return 1, fmt.Errorf("send hello: %w", err)
195 }
196 run, vals, err := waitForRun(ctx, conn, cfg.RunTimeout)
197 if err != nil {
198 return 1, err
199 }
200 if run.GetJob() != nil {
201 return sendExit(conn, runJob(ctx, conn, cfg, run.GetJob(), vals), nil)
202 }
203 return runStub(ctx, conn, cfg, run)
204}
205
206// jobValues are the secrets and variables of the job that the host sent in
207// NamedValue frames before Run. They stay in memory.
208type jobValues struct{ secrets, vars map[string]string }
209
210// maxJobValues bounds what a host can make the executor keep before Run.
211const maxJobValues = 64 << 20
212
213// waitForRun waits for the Run frame of the host, bounded: a host that never
214// answers must not pin the sandbox. The NamedValue frames that come before it
215// are collected.
216func waitForRun(ctx context.Context, conn *execstream.Conn, timeout time.Duration) (*execpb.Run, jobValues, error) {
217 type recv struct {
218 run *execpb.Run
219 vals jobValues
220 err error
221 }
222 first := make(chan recv, 1)
223 go func() {
224 vals := jobValues{secrets: map[string]string{}, vars: map[string]string{}}
225 size := 0
226 for {
227 f, err := conn.Recv()
228 if err != nil {
229 first <- recv{err: fmt.Errorf("waiting for run: %w", err)}
230 return
231 }
232 switch b := f.Body.(type) {
233 case *execpb.Frame_Run:
234 first <- recv{run: b.Run, vals: vals}
235 return
236 case *execpb.Frame_Secret:
237 size += len(b.Secret.GetName()) + len(b.Secret.GetValue())
238 vals.secrets[b.Secret.GetName()] = b.Secret.GetValue()
239 case *execpb.Frame_Var:
240 size += len(b.Var.GetName()) + len(b.Var.GetValue())
241 vals.vars[b.Var.GetName()] = b.Var.GetValue()
242 default:
243 first <- recv{err: errors.New("first frame from host is not run")}
244 return
245 }
246 if size > maxJobValues {
247 first <- recv{err: errors.New("the host sent too many secrets and variables before run")}
248 return
249 }
250 }
251 }()
252 timer := time.NewTimer(timeout)
253 defer timer.Stop()
254 select {
255 case r := <-first:
256 return r.run, r.vals, r.err
257 case <-timer.C:
258 return nil, jobValues{}, errors.New("timed out waiting for the host's run frame")
259 case <-ctx.Done():
260 return nil, jobValues{}, ctx.Err()
261 }
262}
263
264// runStub is the stub executor of sandbox-check: it runs the script of a Run
265// without a job with the shell and streams its output.
266func runStub(ctx context.Context, conn *execstream.Conn, cfg ExecConfig, run *execpb.Run) (int, error) {
267 script := run.GetScript()
268 if script == "" {
269 script = DefaultScript
270 }
271 cmd := exec.Command(cfg.Shell, "-c", script)
272 cmd.Env = os.Environ()
273 for k, v := range run.GetEnv() {
274 cmd.Env = append(cmd.Env, k+"="+v)
275 }
276 cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
277 // Frame writers as stdout/stderr: exec copies the pipes for us, and
278 // WaitDelay bounds the wait when a background child keeps a pipe open.
279 cmd.Stdout = frameWriter{conn, execpb.StreamKind_STREAM_KIND_STDOUT}
280 cmd.Stderr = frameWriter{conn, execpb.StreamKind_STREAM_KIND_STDERR}
281 cmd.WaitDelay = cfg.WaitDelay
282 if err := cmd.Start(); err != nil {
283 return sendExit(conn, 127, fmt.Errorf("start %s: %w", cfg.Shell, err))
284 }
285 esc := &escalator{cfg: cfg, pgid: cmd.Process.Pid}
286
287 // Frames from the host while the script runs: a Cancel starts the escalation;
288 // a lost host kills the script rather than let it run unobserved.
289 go func() {
290 for {
291 f, err := conn.Recv()
292 if err != nil {
293 esc.send(syscall.SIGKILL)
294 return
295 }
296 if f.GetCancel() != nil {
297 esc.cancel()
298 }
299 }
300 }()
301 // The executor's own context (SIGTERM forwarded by init) cancels the same way.
302 stopCtx := context.AfterFunc(ctx, esc.cancel)
303 defer stopCtx()
304
305 waitErr := cmd.Wait()
306 esc.finish()
307 code, err := exitCode(waitErr)
308 return sendExit(conn, code, err)
309}
310
311// exitCode is the exit code to report for the error of cmd.Wait: 0 for a clean
312// exit (also when a background child kept the pipes open past WaitDelay: the
313// group was killed by esc.finish), the code of the process (128+signal when it
314// was killed, like a shell), or 1 with the error for anything else.
315func exitCode(waitErr error) (int, error) {
316 var ee *exec.ExitError
317 switch {
318 case waitErr == nil, errors.Is(waitErr, exec.ErrWaitDelay):
319 return 0, nil
320 case errors.As(waitErr, &ee):
321 code := ee.ExitCode()
322 if code < 0 {
323 if ws, ok := ee.Sys().(syscall.WaitStatus); ok && ws.Signaled() {
324 return 128 + int(ws.Signal()), nil
325 }
326 return 1, nil
327 }
328 return code, nil
329 }
330 return 1, waitErr
331}
332
333func sendExit(c *execstream.Conn, code int, cause error) (int, error) {
334 e := &execpb.Exit{Code: int32(code)}
335 if cause != nil {
336 e.Error = cause.Error()
337 }
338 if err := c.Send(&execpb.Frame{Body: &execpb.Frame_Exit{Exit: e}}); err != nil {
339 return code, fmt.Errorf("send exit: %w", err)
340 }
341 return code, nil
342}