detached d85c6ab

feat(guest): add init (guest PID 1) and the stub executor

commit
d85c6abd05abbd537a7fb7615839c9d1cc719d0b
author
brice · · 2026-10-12 15:35 UTC
parent
a743879
signature
GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email
7 files · +1040 -0
Unified
UnifiedSplit
A internal/guest/cmdline.go +43 -0
@@ -0,0 +1,43 @@
1+// SPDX-License-Identifier: MIT
2+
3+// Package guest is the code that runs inside a sandbox: `ermite-runner init`
4+// (the guest PID 1) and `ermite-runner exec` (the executor).
5+package guest
6+
7+import "strings"
8+
9+// ParseCmdline splits a kernel command line into key=value pairs; a bare word
10+// has the value "". The last occurrence of a key wins. Values may be quoted
11+// with double quotes, as the kernel allows.
12+func ParseCmdline(s string) map[string]string {
13+ out := map[string]string{}
14+ for _, f := range splitCmdline(strings.TrimSpace(s)) {
15+ k, v, _ := strings.Cut(f, "=")
16+ out[k] = strings.Trim(v, `"`)
17+ }
18+ return out
19+}
20+
21+func splitCmdline(s string) []string {
22+ var out []string
23+ var b strings.Builder
24+ inQuote := false
25+ for _, r := range s {
26+ switch {
27+ case r == '"':
28+ inQuote = !inQuote
29+ b.WriteRune(r)
30+ case (r == ' ' || r == '\t' || r == '\n') && !inQuote:
31+ if b.Len() > 0 {
32+ out = append(out, b.String())
33+ b.Reset()
34+ }
35+ default:
36+ b.WriteRune(r)
37+ }
38+ }
39+ if b.Len() > 0 {
40+ out = append(out, b.String())
41+ }
42+ return out
43+}
@@ -0,0 +1,43 @@
1+// SPDX-License-Identifier: MIT
2+
3+// Package guest is the code that runs inside a sandbox: `ermite-runner init`
4+// (the guest PID 1) and `ermite-runner exec` (the executor).
5+package guest
6+
7+import "strings"
8+
9+// ParseCmdline splits a kernel command line into key=value pairs; a bare word
10+// has the value "". The last occurrence of a key wins. Values may be quoted
11+// with double quotes, as the kernel allows.
12+func ParseCmdline(s string) map[string]string {
13+ out := map[string]string{}
14+ for _, f := range splitCmdline(strings.TrimSpace(s)) {
15+ k, v, _ := strings.Cut(f, "=")
16+ out[k] = strings.Trim(v, `"`)
17+ }
18+ return out
19+}
20+
21+func splitCmdline(s string) []string {
22+ var out []string
23+ var b strings.Builder
24+ inQuote := false
25+ for _, r := range s {
26+ switch {
27+ case r == '"':
28+ inQuote = !inQuote
29+ b.WriteRune(r)
30+ case (r == ' ' || r == '\t' || r == '\n') && !inQuote:
31+ if b.Len() > 0 {
32+ out = append(out, b.String())
33+ b.Reset()
34+ }
35+ default:
36+ b.WriteRune(r)
37+ }
38+ }
39+ if b.Len() > 0 {
40+ out = append(out, b.String())
41+ }
42+ return out
43+}
A internal/guest/connect.go +87 -0
@@ -0,0 +1,87 @@
1+// SPDX-License-Identifier: MIT
2+
3+package guest
4+
5+import (
6+ "context"
7+ "fmt"
8+ "io"
9+ "net"
10+ "strconv"
11+ "strings"
12+ "time"
13+
14+ "golang.org/x/sys/unix"
15+)
16+
17+// Target is where the executor connects back to: "unix:/path" (container) or
18+// "vsock:PORT" (vm; the host is CID 2).
19+type Target struct {
20+ Network string // "unix" or "vsock"
21+ Addr string // socket path, or the vsock port
22+}
23+
24+// ParseTarget parses "unix:/path" or "vsock:port".
25+func ParseTarget(s string) (Target, error) {
26+ kind, rest, ok := strings.Cut(s, ":")
27+ if !ok || rest == "" {
28+ return Target{}, fmt.Errorf("connect target %q: want unix:PATH or vsock:PORT", s)
29+ }
30+ switch kind {
31+ case "unix":
32+ return Target{Network: "unix", Addr: rest}, nil
33+ case "vsock":
34+ if p, err := strconv.ParseUint(rest, 10, 32); err != nil || p == 0 {
35+ return Target{}, fmt.Errorf("connect target %q: bad vsock port", s)
36+ }
37+ return Target{Network: "vsock", Addr: rest}, nil
38+ }
39+ return Target{}, fmt.Errorf("connect target %q: unknown kind %q", s, kind)
40+}
41+
42+func (t Target) String() string { return t.Network + ":" + t.Addr }
43+
44+// Dial connects to the host, retrying until ctx ends or timeout elapses: the
45+// host listens before the sandbox starts, but a vsock device may take a moment.
46+func (t Target) Dial(ctx context.Context, timeout time.Duration) (io.ReadWriteCloser, error) {
47+ ctx, cancel := context.WithTimeout(ctx, timeout)
48+ defer cancel()
49+ var last error
50+ for {
51+ var c io.ReadWriteCloser
52+ var err error
53+ if t.Network == "unix" {
54+ var d net.Dialer
55+ c, err = d.DialContext(ctx, "unix", t.Addr)
56+ } else {
57+ c, err = dialVsock(t.Addr)
58+ }
59+ if err == nil {
60+ return c, nil
61+ }
62+ last = err
63+ select {
64+ case <-ctx.Done():
65+ return nil, fmt.Errorf("connect to %s: %w (last error: %v)", t, ctx.Err(), last)
66+ case <-time.After(100 * time.Millisecond):
67+ }
68+ }
69+}
70+
71+// vsockHostCID is the well-known CID of the host.
72+const vsockHostCID = 2
73+
74+func dialVsock(port string) (io.ReadWriteCloser, error) {
75+ p, _ := strconv.ParseUint(port, 10, 32)
76+ fd, err := unix.Socket(unix.AF_VSOCK, unix.SOCK_STREAM|unix.SOCK_CLOEXEC, 0)
77+ if err != nil {
78+ return nil, err
79+ }
80+ // A blocking connect is fine: a Firecracker vsock either connects to the
81+ // host listener at once or fails.
82+ if err := unix.Connect(fd, &unix.SockaddrVM{CID: vsockHostCID, Port: uint32(p)}); err != nil {
83+ _ = unix.Close(fd)
84+ return nil, err
85+ }
86+ return newFDConn(fd)
87+}
@@ -0,0 +1,87 @@
1+// SPDX-License-Identifier: MIT
2+
3+package guest
4+
5+import (
6+ "context"
7+ "fmt"
8+ "io"
9+ "net"
10+ "strconv"
11+ "strings"
12+ "time"
13+
14+ "golang.org/x/sys/unix"
15+)
16+
17+// Target is where the executor connects back to: "unix:/path" (container) or
18+// "vsock:PORT" (vm; the host is CID 2).
19+type Target struct {
20+ Network string // "unix" or "vsock"
21+ Addr string // socket path, or the vsock port
22+}
23+
24+// ParseTarget parses "unix:/path" or "vsock:port".
25+func ParseTarget(s string) (Target, error) {
26+ kind, rest, ok := strings.Cut(s, ":")
27+ if !ok || rest == "" {
28+ return Target{}, fmt.Errorf("connect target %q: want unix:PATH or vsock:PORT", s)
29+ }
30+ switch kind {
31+ case "unix":
32+ return Target{Network: "unix", Addr: rest}, nil
33+ case "vsock":
34+ if p, err := strconv.ParseUint(rest, 10, 32); err != nil || p == 0 {
35+ return Target{}, fmt.Errorf("connect target %q: bad vsock port", s)
36+ }
37+ return Target{Network: "vsock", Addr: rest}, nil
38+ }
39+ return Target{}, fmt.Errorf("connect target %q: unknown kind %q", s, kind)
40+}
41+
42+func (t Target) String() string { return t.Network + ":" + t.Addr }
43+
44+// Dial connects to the host, retrying until ctx ends or timeout elapses: the
45+// host listens before the sandbox starts, but a vsock device may take a moment.
46+func (t Target) Dial(ctx context.Context, timeout time.Duration) (io.ReadWriteCloser, error) {
47+ ctx, cancel := context.WithTimeout(ctx, timeout)
48+ defer cancel()
49+ var last error
50+ for {
51+ var c io.ReadWriteCloser
52+ var err error
53+ if t.Network == "unix" {
54+ var d net.Dialer
55+ c, err = d.DialContext(ctx, "unix", t.Addr)
56+ } else {
57+ c, err = dialVsock(t.Addr)
58+ }
59+ if err == nil {
60+ return c, nil
61+ }
62+ last = err
63+ select {
64+ case <-ctx.Done():
65+ return nil, fmt.Errorf("connect to %s: %w (last error: %v)", t, ctx.Err(), last)
66+ case <-time.After(100 * time.Millisecond):
67+ }
68+ }
69+}
70+
71+// vsockHostCID is the well-known CID of the host.
72+const vsockHostCID = 2
73+
74+func dialVsock(port string) (io.ReadWriteCloser, error) {
75+ p, _ := strconv.ParseUint(port, 10, 32)
76+ fd, err := unix.Socket(unix.AF_VSOCK, unix.SOCK_STREAM|unix.SOCK_CLOEXEC, 0)
77+ if err != nil {
78+ return nil, err
79+ }
80+ // A blocking connect is fine: a Firecracker vsock either connects to the
81+ // host listener at once or fails.
82+ if err := unix.Connect(fd, &unix.SockaddrVM{CID: vsockHostCID, Port: uint32(p)}); err != nil {
83+ _ = unix.Close(fd)
84+ return nil, err
85+ }
86+ return newFDConn(fd)
87+}
A internal/guest/exec.go +175 -0
@@ -0,0 +1,175 @@
1+// SPDX-License-Identifier: MIT
2+
3+package guest
4+
5+import (
6+ "context"
7+ "errors"
8+ "fmt"
9+ "io"
10+ "os"
11+ "os/exec"
12+ "sync"
13+ "syscall"
14+ "time"
15+
16+ "bnema.dev/ermite-runner/internal/execstream"
17+ "bnema.dev/ermite-runner/internal/execstream/execpb"
18+)
19+
20+// DefaultScript is what the stub executor runs when the host sends no script.
21+const DefaultScript = "uname -a; id; cat /etc/os-release; docker info"
22+
23+// ExecConfig configures the executor.
24+type ExecConfig struct {
25+ Target Target
26+ Sandbox string // "vm" or "container", reported in Hello
27+ Version string
28+ // Shell runs the script as `Shell -c script`; default /bin/sh.
29+ Shell string
30+ // ConnectTimeout bounds the connection attempts (default 30 s).
31+ ConnectTimeout time.Duration
32+ // RunTimeout bounds the wait for the host's Run frame (default 60 s).
33+ RunTimeout time.Duration
34+}
35+
36+// Exec is `ermite-runner exec`, the CI-1 stub executor: it connects back to the
37+// host, receives one Run, runs the script with the shell, streams stdout and
38+// stderr as Output frames, and ends with an Exit frame. It returns the exit code
39+// it reported. A Cancel frame sends SIGINT to the script's process group.
40+func Exec(ctx context.Context, cfg ExecConfig) (int, error) {
41+ if cfg.Shell == "" {
42+ cfg.Shell = "/bin/sh"
43+ }
44+ if cfg.ConnectTimeout == 0 {
45+ cfg.ConnectTimeout = 30 * time.Second
46+ }
47+ if cfg.RunTimeout == 0 {
48+ cfg.RunTimeout = 60 * time.Second
49+ }
50+ raw, err := cfg.Target.Dial(ctx, cfg.ConnectTimeout)
51+ if err != nil {
52+ return 1, err
53+ }
54+ defer func() { _ = raw.Close() }()
55+ return serve(ctx, raw, cfg)
56+}
57+
58+func serve(ctx context.Context, rw io.ReadWriteCloser, cfg ExecConfig) (int, error) {
59+ conn := execstream.NewConn(rw)
60+ host, _ := os.Hostname()
61+ if err := conn.Send(&execpb.Frame{Body: &execpb.Frame_Hello{Hello: &execpb.Hello{
62+ Version: cfg.Version, Sandbox: cfg.Sandbox, Hostname: host,
63+ }}}); err != nil {
64+ return 1, fmt.Errorf("send hello: %w", err)
65+ }
66+
67+ // Wait for Run, bounded: a host that never answers must not pin the sandbox.
68+ type recv struct {
69+ f *execpb.Frame
70+ err error
71+ }
72+ first := make(chan recv, 1)
73+ go func() { f, err := conn.Recv(); first <- recv{f, err} }()
74+ var run *execpb.Run
75+ select {
76+ case r := <-first:
77+ if r.err != nil {
78+ return 1, fmt.Errorf("waiting for run: %w", r.err)
79+ }
80+ if run = r.f.GetRun(); run == nil {
81+ return 1, errors.New("first frame from host is not run")
82+ }
83+ case <-time.After(cfg.RunTimeout):
84+ return 1, errors.New("timed out waiting for the host's run frame")
85+ case <-ctx.Done():
86+ return 1, ctx.Err()
87+ }
88+
89+ script := run.GetScript()
90+ if script == "" {
91+ script = DefaultScript
92+ }
93+ cmd := exec.Command(cfg.Shell, "-c", script)
94+ cmd.Env = os.Environ()
95+ for k, v := range run.GetEnv() {
96+ cmd.Env = append(cmd.Env, k+"="+v)
97+ }
98+ cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
99+ stdout, err := cmd.StdoutPipe()
100+ if err != nil {
101+ return sendExit(conn, 1, err)
102+ }
103+ stderr, err := cmd.StderrPipe()
104+ if err != nil {
105+ return sendExit(conn, 1, err)
106+ }
107+ if err := cmd.Start(); err != nil {
108+ return sendExit(conn, 127, fmt.Errorf("start %s: %w", cfg.Shell, err))
109+ }
110+
111+ // Cancel frames arrive while the script runs.
112+ go func() {
113+ for {
114+ f, err := conn.Recv()
115+ if err != nil {
116+ // Host went away: stop the script rather than run unobserved.
117+ _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
118+ return
119+ }
120+ if f.GetCancel() != nil {
121+ _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGINT)
122+ }
123+ }
124+ }()
125+
126+ var wg sync.WaitGroup
127+ pump := func(r io.Reader, kind execpb.StreamKind) {
128+ defer wg.Done()
129+ buf := make([]byte, execstream.ChunkSize)
130+ for {
131+ n, err := r.Read(buf)
132+ if n > 0 {
133+ _ = conn.Send(&execpb.Frame{Body: &execpb.Frame_Output{Output: &execpb.Output{
134+ Stream: kind, Data: append([]byte(nil), buf[:n]...), UnixNano: time.Now().UnixNano(),
135+ }}})
136+ }
137+ if err != nil {
138+ return
139+ }
140+ }
141+ }
142+ wg.Add(2)
143+ go pump(stdout, execpb.StreamKind_STREAM_KIND_STDOUT)
144+ go pump(stderr, execpb.StreamKind_STREAM_KIND_STDERR)
145+ wg.Wait() // pipes must be drained before Wait closes them
146+ waitErr := cmd.Wait()
147+ code := 0
148+ var ee *exec.ExitError
149+ switch {
150+ case waitErr == nil:
151+ case errors.As(waitErr, &ee):
152+ code = ee.ExitCode()
153+ if code < 0 { // killed by a signal: 128+signal like a shell
154+ if ws, ok := ee.Sys().(syscall.WaitStatus); ok && ws.Signaled() {
155+ code = 128 + int(ws.Signal())
156+ } else {
157+ code = 1
158+ }
159+ }
160+ default:
161+ return sendExit(conn, 1, waitErr)
162+ }
163+ return sendExit(conn, code, nil)
164+}
165+
166+func sendExit(c *execstream.Conn, code int, cause error) (int, error) {
167+ e := &execpb.Exit{Code: int32(code)}
168+ if cause != nil {
169+ e.Error = cause.Error()
170+ }
171+ if err := c.Send(&execpb.Frame{Body: &execpb.Frame_Exit{Exit: e}}); err != nil {
172+ return code, fmt.Errorf("send exit: %w", err)
173+ }
174+ return code, nil
175+}
@@ -0,0 +1,175 @@
1+// SPDX-License-Identifier: MIT
2+
3+package guest
4+
5+import (
6+ "context"
7+ "errors"
8+ "fmt"
9+ "io"
10+ "os"
11+ "os/exec"
12+ "sync"
13+ "syscall"
14+ "time"
15+
16+ "bnema.dev/ermite-runner/internal/execstream"
17+ "bnema.dev/ermite-runner/internal/execstream/execpb"
18+)
19+
20+// DefaultScript is what the stub executor runs when the host sends no script.
21+const DefaultScript = "uname -a; id; cat /etc/os-release; docker info"
22+
23+// ExecConfig configures the executor.
24+type ExecConfig struct {
25+ Target Target
26+ Sandbox string // "vm" or "container", reported in Hello
27+ Version string
28+ // Shell runs the script as `Shell -c script`; default /bin/sh.
29+ Shell string
30+ // ConnectTimeout bounds the connection attempts (default 30 s).
31+ ConnectTimeout time.Duration
32+ // RunTimeout bounds the wait for the host's Run frame (default 60 s).
33+ RunTimeout time.Duration
34+}
35+
36+// Exec is `ermite-runner exec`, the CI-1 stub executor: it connects back to the
37+// host, receives one Run, runs the script with the shell, streams stdout and
38+// stderr as Output frames, and ends with an Exit frame. It returns the exit code
39+// it reported. A Cancel frame sends SIGINT to the script's process group.
40+func Exec(ctx context.Context, cfg ExecConfig) (int, error) {
41+ if cfg.Shell == "" {
42+ cfg.Shell = "/bin/sh"
43+ }
44+ if cfg.ConnectTimeout == 0 {
45+ cfg.ConnectTimeout = 30 * time.Second
46+ }
47+ if cfg.RunTimeout == 0 {
48+ cfg.RunTimeout = 60 * time.Second
49+ }
50+ raw, err := cfg.Target.Dial(ctx, cfg.ConnectTimeout)
51+ if err != nil {
52+ return 1, err
53+ }
54+ defer func() { _ = raw.Close() }()
55+ return serve(ctx, raw, cfg)
56+}
57+
58+func serve(ctx context.Context, rw io.ReadWriteCloser, cfg ExecConfig) (int, error) {
59+ conn := execstream.NewConn(rw)
60+ host, _ := os.Hostname()
61+ if err := conn.Send(&execpb.Frame{Body: &execpb.Frame_Hello{Hello: &execpb.Hello{
62+ Version: cfg.Version, Sandbox: cfg.Sandbox, Hostname: host,
63+ }}}); err != nil {
64+ return 1, fmt.Errorf("send hello: %w", err)
65+ }
66+
67+ // Wait for Run, bounded: a host that never answers must not pin the sandbox.
68+ type recv struct {
69+ f *execpb.Frame
70+ err error
71+ }
72+ first := make(chan recv, 1)
73+ go func() { f, err := conn.Recv(); first <- recv{f, err} }()
74+ var run *execpb.Run
75+ select {
76+ case r := <-first:
77+ if r.err != nil {
78+ return 1, fmt.Errorf("waiting for run: %w", r.err)
79+ }
80+ if run = r.f.GetRun(); run == nil {
81+ return 1, errors.New("first frame from host is not run")
82+ }
83+ case <-time.After(cfg.RunTimeout):
84+ return 1, errors.New("timed out waiting for the host's run frame")
85+ case <-ctx.Done():
86+ return 1, ctx.Err()
87+ }
88+
89+ script := run.GetScript()
90+ if script == "" {
91+ script = DefaultScript
92+ }
93+ cmd := exec.Command(cfg.Shell, "-c", script)
94+ cmd.Env = os.Environ()
95+ for k, v := range run.GetEnv() {
96+ cmd.Env = append(cmd.Env, k+"="+v)
97+ }
98+ cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
99+ stdout, err := cmd.StdoutPipe()
100+ if err != nil {
101+ return sendExit(conn, 1, err)
102+ }
103+ stderr, err := cmd.StderrPipe()
104+ if err != nil {
105+ return sendExit(conn, 1, err)
106+ }
107+ if err := cmd.Start(); err != nil {
108+ return sendExit(conn, 127, fmt.Errorf("start %s: %w", cfg.Shell, err))
109+ }
110+
111+ // Cancel frames arrive while the script runs.
112+ go func() {
113+ for {
114+ f, err := conn.Recv()
115+ if err != nil {
116+ // Host went away: stop the script rather than run unobserved.
117+ _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
118+ return
119+ }
120+ if f.GetCancel() != nil {
121+ _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGINT)
122+ }
123+ }
124+ }()
125+
126+ var wg sync.WaitGroup
127+ pump := func(r io.Reader, kind execpb.StreamKind) {
128+ defer wg.Done()
129+ buf := make([]byte, execstream.ChunkSize)
130+ for {
131+ n, err := r.Read(buf)
132+ if n > 0 {
133+ _ = conn.Send(&execpb.Frame{Body: &execpb.Frame_Output{Output: &execpb.Output{
134+ Stream: kind, Data: append([]byte(nil), buf[:n]...), UnixNano: time.Now().UnixNano(),
135+ }}})
136+ }
137+ if err != nil {
138+ return
139+ }
140+ }
141+ }
142+ wg.Add(2)
143+ go pump(stdout, execpb.StreamKind_STREAM_KIND_STDOUT)
144+ go pump(stderr, execpb.StreamKind_STREAM_KIND_STDERR)
145+ wg.Wait() // pipes must be drained before Wait closes them
146+ waitErr := cmd.Wait()
147+ code := 0
148+ var ee *exec.ExitError
149+ switch {
150+ case waitErr == nil:
151+ case errors.As(waitErr, &ee):
152+ code = ee.ExitCode()
153+ if code < 0 { // killed by a signal: 128+signal like a shell
154+ if ws, ok := ee.Sys().(syscall.WaitStatus); ok && ws.Signaled() {
155+ code = 128 + int(ws.Signal())
156+ } else {
157+ code = 1
158+ }
159+ }
160+ default:
161+ return sendExit(conn, 1, waitErr)
162+ }
163+ return sendExit(conn, code, nil)
164+}
165+
166+func sendExit(c *execstream.Conn, code int, cause error) (int, error) {
167+ e := &execpb.Exit{Code: int32(code)}
168+ if cause != nil {
169+ e.Error = cause.Error()
170+ }
171+ if err := c.Send(&execpb.Frame{Body: &execpb.Frame_Exit{Exit: e}}); err != nil {
172+ return code, fmt.Errorf("send exit: %w", err)
173+ }
174+ return code, nil
175+}
A internal/guest/fdconn.go +20 -0
@@ -0,0 +1,20 @@
1+// SPDX-License-Identifier: MIT
2+
3+package guest
4+
5+import (
6+ "io"
7+ "os"
8+
9+ "golang.org/x/sys/unix"
10+)
11+
12+// newFDConn wraps a connected stream socket (net.FileConn does not know
13+// AF_VSOCK) as a ReadWriteCloser registered with the runtime poller.
14+func newFDConn(fd int) (io.ReadWriteCloser, error) {
15+ if err := unix.SetNonblock(fd, true); err != nil {
16+ _ = unix.Close(fd)
17+ return nil, err
18+ }
19+ return os.NewFile(uintptr(fd), "vsock"), nil
20+}
@@ -0,0 +1,20 @@
1+// SPDX-License-Identifier: MIT
2+
3+package guest
4+
5+import (
6+ "io"
7+ "os"
8+
9+ "golang.org/x/sys/unix"
10+)
11+
12+// newFDConn wraps a connected stream socket (net.FileConn does not know
13+// AF_VSOCK) as a ReadWriteCloser registered with the runtime poller.
14+func newFDConn(fd int) (io.ReadWriteCloser, error) {
15+ if err := unix.SetNonblock(fd, true); err != nil {
16+ _ = unix.Close(fd)
17+ return nil, err
18+ }
19+ return os.NewFile(uintptr(fd), "vsock"), nil
20+}
A internal/guest/guest_test.go +261 -0
@@ -0,0 +1,261 @@
1+// SPDX-License-Identifier: MIT
2+
3+package guest
4+
5+import (
6+ "context"
7+ "net"
8+ "path/filepath"
9+ "reflect"
10+ "strings"
11+ "syscall"
12+ "testing"
13+ "time"
14+
15+ "bnema.dev/ermite-runner/internal/execstream"
16+ "bnema.dev/ermite-runner/internal/execstream/execpb"
17+)
18+
19+func TestParseCmdline(t *testing.T) {
20+ got := ParseCmdline("console=ttyS0 ro ermite.connect=vsock:1024 x=\"a b\" ermite.gw=10.0.0.1 ermite.gw=10.0.0.9 -- init\n")
21+ want := map[string]string{"console": "ttyS0", "ro": "", "ermite.connect": "vsock:1024", "x": "a b", "ermite.gw": "10.0.0.9", "--": "", "init": ""}
22+ if !reflect.DeepEqual(got, want) {
23+ t.Fatalf("got %v", got)
24+ }
25+ if len(ParseCmdline("")) != 0 {
26+ t.Fatal("empty command line")
27+ }
28+}
29+
30+func TestParseTarget(t *testing.T) {
31+ for in, want := range map[string]Target{
32+ "unix:/run/ermite/exec.sock": {"unix", "/run/ermite/exec.sock"},
33+ "vsock:1024": {"vsock", "1024"},
34+ } {
35+ got, err := ParseTarget(in)
36+ if err != nil || got != want || got.String() != in {
37+ t.Errorf("%q: %v %v", in, got, err)
38+ }
39+ }
40+ for _, in := range []string{"", "unix", "unix:", "vsock:0", "vsock:abc", "tcp:1.2.3.4:5", "vsock:99999999999"} {
41+ if _, err := ParseTarget(in); err == nil {
42+ t.Errorf("%q accepted", in)
43+ }
44+ }
45+}
46+
47+// host plays the host process on a unix socket and runs one session.
48+func host(t *testing.T, script string, ctxFn func(context.Context) context.Context) (out, errOut string, exit *execpb.Exit, hello *execpb.Hello, code int, execErr error) {
49+ t.Helper()
50+ sock := filepath.Join(t.TempDir(), "s")
51+ ln, err := net.Listen("unix", sock)
52+ if err != nil {
53+ t.Skip(err)
54+ }
55+ defer func() { _ = ln.Close() }()
56+ type res struct {
57+ code int
58+ err error
59+ }
60+ done := make(chan res, 1)
61+ go func() {
62+ c, err := Exec(context.Background(), ExecConfig{Target: Target{"unix", sock}, Sandbox: "container", Version: "test", Shell: "/bin/sh", RunTimeout: 5 * time.Second})
63+ done <- res{c, err}
64+ }()
65+ conn, err := ln.Accept()
66+ if err != nil {
67+ t.Fatal(err)
68+ }
69+ defer func() { _ = conn.Close() }()
70+ ctx := context.Background()
71+ if ctxFn != nil {
72+ ctx = ctxFn(ctx)
73+ }
74+ var so, se strings.Builder
75+ hello, exit, err = execstream.Drive(ctx, conn, &execpb.Run{Script: script, Env: map[string]string{"STUB_VAR": "from-host"}}, func(o *execpb.Output) {
76+ if o.Stream == execpb.StreamKind_STREAM_KIND_STDERR {
77+ se.Write(o.Data)
78+ } else {
79+ so.Write(o.Data)
80+ }
81+ })
82+ if err != nil {
83+ t.Fatal(err)
84+ }
85+ r := <-done
86+ return so.String(), se.String(), exit, hello, r.code, r.err
87+}
88+
89+func TestExecStreamsOutputAndExitCode(t *testing.T) {
90+ out, errOut, exit, hello, code, err := host(t, `echo out-line; echo err-line >&2; echo "$STUB_VAR"; exit 3`, nil)
91+ if err != nil {
92+ t.Fatal(err)
93+ }
94+ if out != "out-line\nfrom-host\n" || errOut != "err-line\n" {
95+ t.Fatalf("out=%q err=%q", out, errOut)
96+ }
97+ if exit.Code != 3 || code != 3 || exit.Error != "" {
98+ t.Fatalf("exit=%v code=%d", exit, code)
99+ }
100+ if hello.Sandbox != "container" || hello.Version != "test" || hello.Hostname == "" {
101+ t.Fatalf("hello=%v", hello)
102+ }
103+}
104+
105+func TestExecDefaultScriptRunsDiagnostics(t *testing.T) {
106+ // The default script is the CI-1 stub: uname, id, os-release, docker info.
107+ // docker may be absent here: only the layout of the output is checked.
108+ out, errOut, exit, _, _, _ := host(t, "", nil)
109+ if !strings.Contains(out, "Linux") || !strings.Contains(out, "uid=") {
110+ t.Fatalf("uname/id missing: out=%q err=%q", out, errOut)
111+ }
112+ if exit == nil {
113+ t.Fatal("no exit frame")
114+ }
115+ if !strings.Contains(DefaultScript, "docker info") || !strings.Contains(DefaultScript, "os-release") {
116+ t.Fatal("default script must run what the plan says")
117+ }
118+}
119+
120+func TestExecLargeOutputIsChunkedAndComplete(t *testing.T) {
121+ out, _, exit, _, _, _ := host(t, `head -c 300000 /dev/zero | tr '\0' 'x'`, nil)
122+ if len(out) != 300000 || exit.Code != 0 {
123+ t.Fatalf("len=%d exit=%v", len(out), exit)
124+ }
125+}
126+
127+func TestExecCancelSendsSIGINT(t *testing.T) {
128+ start := time.Now()
129+ out, _, exit, _, _, _ := host(t, `trap 'echo got-int; exit 130' INT; echo ready; while :; do sleep 0.05; done`, func(ctx context.Context) context.Context {
130+ c, cancel := context.WithCancel(ctx)
131+ time.AfterFunc(500*time.Millisecond, cancel)
132+ return c
133+ })
134+ if !strings.Contains(out, "got-int") || exit.Code != 130 {
135+ t.Fatalf("out=%q exit=%v", out, exit)
136+ }
137+ if time.Since(start) > 5*time.Second {
138+ t.Fatal("cancellation too slow")
139+ }
140+}
141+
142+func TestExecKilledBySignalReports128PlusSignal(t *testing.T) {
143+ _, _, exit, _, _, _ := host(t, `kill -TERM $$`, nil)
144+ if exit.Code != 128+int32(syscall.SIGTERM) {
145+ t.Fatalf("exit=%v", exit)
146+ }
147+}
148+
149+func TestExecMissingShell(t *testing.T) {
150+ sock := filepath.Join(t.TempDir(), "s")
151+ ln, err := net.Listen("unix", sock)
152+ if err != nil {
153+ t.Skip(err)
154+ }
155+ defer func() { _ = ln.Close() }()
156+ go func() {
157+ c, _ := ln.Accept()
158+ defer func() { _ = c.Close() }()
159+ _, exit, _ := execstream.Drive(context.Background(), c, &execpb.Run{Script: "true"}, nil)
160+ if exit == nil || exit.Code != 127 || exit.Error == "" {
161+ t.Errorf("want 127 with an error message, got %v", exit)
162+ }
163+ }()
164+ code, _ := Exec(context.Background(), ExecConfig{Target: Target{"unix", sock}, Shell: "/nonexistent/sh", RunTimeout: 3 * time.Second})
165+ if code != 127 {
166+ t.Fatalf("code=%d", code)
167+ }
168+}
169+
170+func TestExecHostGoneKillsTheScript(t *testing.T) {
171+ sock := filepath.Join(t.TempDir(), "s")
172+ ln, err := net.Listen("unix", sock)
173+ if err != nil {
174+ t.Skip(err)
175+ }
176+ defer func() { _ = ln.Close() }()
177+ done := make(chan int, 1)
178+ go func() {
179+ c, _ := Exec(context.Background(), ExecConfig{Target: Target{"unix", sock}, Shell: "/bin/sh", RunTimeout: 3 * time.Second})
180+ done <- c
181+ }()
182+ conn, _ := ln.Accept()
183+ g := execstream.NewConn(conn)
184+ _, _ = g.Recv() // hello
185+ _ = g.Send(&execpb.Frame{Body: &execpb.Frame_Run{Run: &execpb.Run{Script: "sleep 60"}}})
186+ time.Sleep(200 * time.Millisecond)
187+ _ = conn.Close() // the host died
188+ select {
189+ case code := <-done:
190+ if code == 0 {
191+ t.Fatalf("code=%d", code)
192+ }
193+ case <-time.After(10 * time.Second):
194+ t.Fatal("the executor kept running after the host went away")
195+ }
196+}
197+
198+func TestExecConnectTimeout(t *testing.T) {
199+ start := time.Now()
200+ code, err := Exec(context.Background(), ExecConfig{Target: Target{"unix", filepath.Join(t.TempDir(), "nobody")}, ConnectTimeout: 300 * time.Millisecond})
201+ if code == 0 || err == nil || time.Since(start) > 3*time.Second {
202+ t.Fatalf("code=%d err=%v after %v", code, err, time.Since(start))
203+ }
204+}
205+
206+func TestExecRunTimeout(t *testing.T) {
207+ sock := filepath.Join(t.TempDir(), "s")
208+ ln, err := net.Listen("unix", sock)
209+ if err != nil {
210+ t.Skip(err)
211+ }
212+ defer func() { _ = ln.Close() }()
213+ go func() { c, _ := ln.Accept(); time.Sleep(2 * time.Second); _ = c.Close() }()
214+ code, err := Exec(context.Background(), ExecConfig{Target: Target{"unix", sock}, RunTimeout: 200 * time.Millisecond})
215+ if code == 0 || err == nil || !strings.Contains(err.Error(), "run frame") {
216+ t.Fatalf("code=%d err=%v", code, err)
217+ }
218+}
219+
220+func TestInitInContainerSkipsVMSetup(t *testing.T) {
221+ // In a container (not a VM's PID 1) Init must not mount anything: with a
222+ // command line that says "vm" but a pid that is not 1 it behaves as a
223+ // container, runs the payload and returns the executor's code.
224+ sock := filepath.Join(t.TempDir(), "s")
225+ ln, err := net.Listen("unix", sock)
226+ if err != nil {
227+ t.Skip(err)
228+ }
229+ defer func() { _ = ln.Close() }()
230+ go func() {
231+ c, err := ln.Accept()
232+ if err != nil {
233+ return
234+ }
235+ defer func() { _ = c.Close() }()
236+ _, _, _ = execstream.Drive(context.Background(), c, &execpb.Run{Script: "exit 0"}, nil)
237+ }()
238+ code := Init(context.Background(), InitConfig{
239+ Cmdline: "ermite.sandbox=vm ermite.connect=unix:" + sock,
240+ Getpid: func() int { return 4242 },
241+ ExecArgs: []string{"/bin/sh", "-c", "exit 5"},
242+ DockerTimeout: time.Millisecond,
243+ SkipDocker: true,
244+ })
245+ // ExecArgs is a stand-in for the executor: its exit code is Init's.
246+ if code != 5 {
247+ t.Fatalf("code=%d", code)
248+ }
249+}
250+
251+func TestInitWithoutTargetFails(t *testing.T) {
252+ if code := Init(context.Background(), InitConfig{Cmdline: "ermite.sandbox=container", Getpid: func() int { return 77 }, ExecArgs: []string{"/bin/true"}, SkipDocker: true}); code == 0 {
253+ t.Fatal("init without a connect target must fail")
254+ }
255+}
256+
257+func TestWriteResolvConfNeedsGateway(t *testing.T) {
258+ if err := writeResolvConf(""); err == nil {
259+ t.Fatal("empty gateway accepted")
260+ }
261+}
@@ -0,0 +1,261 @@
1+// SPDX-License-Identifier: MIT
2+
3+package guest
4+
5+import (
6+ "context"
7+ "net"
8+ "path/filepath"
9+ "reflect"
10+ "strings"
11+ "syscall"
12+ "testing"
13+ "time"
14+
15+ "bnema.dev/ermite-runner/internal/execstream"
16+ "bnema.dev/ermite-runner/internal/execstream/execpb"
17+)
18+
19+func TestParseCmdline(t *testing.T) {
20+ got := ParseCmdline("console=ttyS0 ro ermite.connect=vsock:1024 x=\"a b\" ermite.gw=10.0.0.1 ermite.gw=10.0.0.9 -- init\n")
21+ want := map[string]string{"console": "ttyS0", "ro": "", "ermite.connect": "vsock:1024", "x": "a b", "ermite.gw": "10.0.0.9", "--": "", "init": ""}
22+ if !reflect.DeepEqual(got, want) {
23+ t.Fatalf("got %v", got)
24+ }
25+ if len(ParseCmdline("")) != 0 {
26+ t.Fatal("empty command line")
27+ }
28+}
29+
30+func TestParseTarget(t *testing.T) {
31+ for in, want := range map[string]Target{
32+ "unix:/run/ermite/exec.sock": {"unix", "/run/ermite/exec.sock"},
33+ "vsock:1024": {"vsock", "1024"},
34+ } {
35+ got, err := ParseTarget(in)
36+ if err != nil || got != want || got.String() != in {
37+ t.Errorf("%q: %v %v", in, got, err)
38+ }
39+ }
40+ for _, in := range []string{"", "unix", "unix:", "vsock:0", "vsock:abc", "tcp:1.2.3.4:5", "vsock:99999999999"} {
41+ if _, err := ParseTarget(in); err == nil {
42+ t.Errorf("%q accepted", in)
43+ }
44+ }
45+}
46+
47+// host plays the host process on a unix socket and runs one session.
48+func host(t *testing.T, script string, ctxFn func(context.Context) context.Context) (out, errOut string, exit *execpb.Exit, hello *execpb.Hello, code int, execErr error) {
49+ t.Helper()
50+ sock := filepath.Join(t.TempDir(), "s")
51+ ln, err := net.Listen("unix", sock)
52+ if err != nil {
53+ t.Skip(err)
54+ }
55+ defer func() { _ = ln.Close() }()
56+ type res struct {
57+ code int
58+ err error
59+ }
60+ done := make(chan res, 1)
61+ go func() {
62+ c, err := Exec(context.Background(), ExecConfig{Target: Target{"unix", sock}, Sandbox: "container", Version: "test", Shell: "/bin/sh", RunTimeout: 5 * time.Second})
63+ done <- res{c, err}
64+ }()
65+ conn, err := ln.Accept()
66+ if err != nil {
67+ t.Fatal(err)
68+ }
69+ defer func() { _ = conn.Close() }()
70+ ctx := context.Background()
71+ if ctxFn != nil {
72+ ctx = ctxFn(ctx)
73+ }
74+ var so, se strings.Builder
75+ hello, exit, err = execstream.Drive(ctx, conn, &execpb.Run{Script: script, Env: map[string]string{"STUB_VAR": "from-host"}}, func(o *execpb.Output) {
76+ if o.Stream == execpb.StreamKind_STREAM_KIND_STDERR {
77+ se.Write(o.Data)
78+ } else {
79+ so.Write(o.Data)
80+ }
81+ })
82+ if err != nil {
83+ t.Fatal(err)
84+ }
85+ r := <-done
86+ return so.String(), se.String(), exit, hello, r.code, r.err
87+}
88+
89+func TestExecStreamsOutputAndExitCode(t *testing.T) {
90+ out, errOut, exit, hello, code, err := host(t, `echo out-line; echo err-line >&2; echo "$STUB_VAR"; exit 3`, nil)
91+ if err != nil {
92+ t.Fatal(err)
93+ }
94+ if out != "out-line\nfrom-host\n" || errOut != "err-line\n" {
95+ t.Fatalf("out=%q err=%q", out, errOut)
96+ }
97+ if exit.Code != 3 || code != 3 || exit.Error != "" {
98+ t.Fatalf("exit=%v code=%d", exit, code)
99+ }
100+ if hello.Sandbox != "container" || hello.Version != "test" || hello.Hostname == "" {
101+ t.Fatalf("hello=%v", hello)
102+ }
103+}
104+
105+func TestExecDefaultScriptRunsDiagnostics(t *testing.T) {
106+ // The default script is the CI-1 stub: uname, id, os-release, docker info.
107+ // docker may be absent here: only the layout of the output is checked.
108+ out, errOut, exit, _, _, _ := host(t, "", nil)
109+ if !strings.Contains(out, "Linux") || !strings.Contains(out, "uid=") {
110+ t.Fatalf("uname/id missing: out=%q err=%q", out, errOut)
111+ }
112+ if exit == nil {
113+ t.Fatal("no exit frame")
114+ }
115+ if !strings.Contains(DefaultScript, "docker info") || !strings.Contains(DefaultScript, "os-release") {
116+ t.Fatal("default script must run what the plan says")
117+ }
118+}
119+
120+func TestExecLargeOutputIsChunkedAndComplete(t *testing.T) {
121+ out, _, exit, _, _, _ := host(t, `head -c 300000 /dev/zero | tr '\0' 'x'`, nil)
122+ if len(out) != 300000 || exit.Code != 0 {
123+ t.Fatalf("len=%d exit=%v", len(out), exit)
124+ }
125+}
126+
127+func TestExecCancelSendsSIGINT(t *testing.T) {
128+ start := time.Now()
129+ out, _, exit, _, _, _ := host(t, `trap 'echo got-int; exit 130' INT; echo ready; while :; do sleep 0.05; done`, func(ctx context.Context) context.Context {
130+ c, cancel := context.WithCancel(ctx)
131+ time.AfterFunc(500*time.Millisecond, cancel)
132+ return c
133+ })
134+ if !strings.Contains(out, "got-int") || exit.Code != 130 {
135+ t.Fatalf("out=%q exit=%v", out, exit)
136+ }
137+ if time.Since(start) > 5*time.Second {
138+ t.Fatal("cancellation too slow")
139+ }
140+}
141+
142+func TestExecKilledBySignalReports128PlusSignal(t *testing.T) {
143+ _, _, exit, _, _, _ := host(t, `kill -TERM $$`, nil)
144+ if exit.Code != 128+int32(syscall.SIGTERM) {
145+ t.Fatalf("exit=%v", exit)
146+ }
147+}
148+
149+func TestExecMissingShell(t *testing.T) {
150+ sock := filepath.Join(t.TempDir(), "s")
151+ ln, err := net.Listen("unix", sock)
152+ if err != nil {
153+ t.Skip(err)
154+ }
155+ defer func() { _ = ln.Close() }()
156+ go func() {
157+ c, _ := ln.Accept()
158+ defer func() { _ = c.Close() }()
159+ _, exit, _ := execstream.Drive(context.Background(), c, &execpb.Run{Script: "true"}, nil)
160+ if exit == nil || exit.Code != 127 || exit.Error == "" {
161+ t.Errorf("want 127 with an error message, got %v", exit)
162+ }
163+ }()
164+ code, _ := Exec(context.Background(), ExecConfig{Target: Target{"unix", sock}, Shell: "/nonexistent/sh", RunTimeout: 3 * time.Second})
165+ if code != 127 {
166+ t.Fatalf("code=%d", code)
167+ }
168+}
169+
170+func TestExecHostGoneKillsTheScript(t *testing.T) {
171+ sock := filepath.Join(t.TempDir(), "s")
172+ ln, err := net.Listen("unix", sock)
173+ if err != nil {
174+ t.Skip(err)
175+ }
176+ defer func() { _ = ln.Close() }()
177+ done := make(chan int, 1)
178+ go func() {
179+ c, _ := Exec(context.Background(), ExecConfig{Target: Target{"unix", sock}, Shell: "/bin/sh", RunTimeout: 3 * time.Second})
180+ done <- c
181+ }()
182+ conn, _ := ln.Accept()
183+ g := execstream.NewConn(conn)
184+ _, _ = g.Recv() // hello
185+ _ = g.Send(&execpb.Frame{Body: &execpb.Frame_Run{Run: &execpb.Run{Script: "sleep 60"}}})
186+ time.Sleep(200 * time.Millisecond)
187+ _ = conn.Close() // the host died
188+ select {
189+ case code := <-done:
190+ if code == 0 {
191+ t.Fatalf("code=%d", code)
192+ }
193+ case <-time.After(10 * time.Second):
194+ t.Fatal("the executor kept running after the host went away")
195+ }
196+}
197+
198+func TestExecConnectTimeout(t *testing.T) {
199+ start := time.Now()
200+ code, err := Exec(context.Background(), ExecConfig{Target: Target{"unix", filepath.Join(t.TempDir(), "nobody")}, ConnectTimeout: 300 * time.Millisecond})
201+ if code == 0 || err == nil || time.Since(start) > 3*time.Second {
202+ t.Fatalf("code=%d err=%v after %v", code, err, time.Since(start))
203+ }
204+}
205+
206+func TestExecRunTimeout(t *testing.T) {
207+ sock := filepath.Join(t.TempDir(), "s")
208+ ln, err := net.Listen("unix", sock)
209+ if err != nil {
210+ t.Skip(err)
211+ }
212+ defer func() { _ = ln.Close() }()
213+ go func() { c, _ := ln.Accept(); time.Sleep(2 * time.Second); _ = c.Close() }()
214+ code, err := Exec(context.Background(), ExecConfig{Target: Target{"unix", sock}, RunTimeout: 200 * time.Millisecond})
215+ if code == 0 || err == nil || !strings.Contains(err.Error(), "run frame") {
216+ t.Fatalf("code=%d err=%v", code, err)
217+ }
218+}
219+
220+func TestInitInContainerSkipsVMSetup(t *testing.T) {
221+ // In a container (not a VM's PID 1) Init must not mount anything: with a
222+ // command line that says "vm" but a pid that is not 1 it behaves as a
223+ // container, runs the payload and returns the executor's code.
224+ sock := filepath.Join(t.TempDir(), "s")
225+ ln, err := net.Listen("unix", sock)
226+ if err != nil {
227+ t.Skip(err)
228+ }
229+ defer func() { _ = ln.Close() }()
230+ go func() {
231+ c, err := ln.Accept()
232+ if err != nil {
233+ return
234+ }
235+ defer func() { _ = c.Close() }()
236+ _, _, _ = execstream.Drive(context.Background(), c, &execpb.Run{Script: "exit 0"}, nil)
237+ }()
238+ code := Init(context.Background(), InitConfig{
239+ Cmdline: "ermite.sandbox=vm ermite.connect=unix:" + sock,
240+ Getpid: func() int { return 4242 },
241+ ExecArgs: []string{"/bin/sh", "-c", "exit 5"},
242+ DockerTimeout: time.Millisecond,
243+ SkipDocker: true,
244+ })
245+ // ExecArgs is a stand-in for the executor: its exit code is Init's.
246+ if code != 5 {
247+ t.Fatalf("code=%d", code)
248+ }
249+}
250+
251+func TestInitWithoutTargetFails(t *testing.T) {
252+ if code := Init(context.Background(), InitConfig{Cmdline: "ermite.sandbox=container", Getpid: func() int { return 77 }, ExecArgs: []string{"/bin/true"}, SkipDocker: true}); code == 0 {
253+ t.Fatal("init without a connect target must fail")
254+ }
255+}
256+
257+func TestWriteResolvConfNeedsGateway(t *testing.T) {
258+ if err := writeResolvConf(""); err == nil {
259+ t.Fatal("empty gateway accepted")
260+ }
261+}
A internal/guest/init.go +251 -0
@@ -0,0 +1,251 @@
1+// SPDX-License-Identifier: MIT
2+
3+package guest
4+
5+import (
6+ "context"
7+ "fmt"
8+ "os"
9+ "strings"
10+ "syscall"
11+ "time"
12+
13+ "github.com/bnema/zerowrap"
14+ "golang.org/x/sys/unix"
15+
16+ "bnema.dev/ermite-runner/internal/sandbox"
17+)
18+
19+// InitConfig configures Init. The zero value reads the real system; tests
20+// replace the fields.
21+type InitConfig struct {
22+ Version string
23+ // Cmdline is the kernel command line; empty reads /proc/cmdline.
24+ Cmdline string
25+ // Getpid reports the process id; Init only acts as a VM's PID 1 when it is 1.
26+ Getpid func() int
27+ // DockerTimeout bounds the wait for the Docker socket (default 60 s).
28+ DockerTimeout time.Duration
29+ // SkipDocker does not start dockerd (tests, images without Docker).
30+ SkipDocker bool
31+ // ExecArgs is the executor command line; default
32+ // [GuestExecDir/GuestExecName exec].
33+ ExecArgs []string
34+}
35+
36+// Init is `ermite-runner init`: the entrypoint of every sandbox.
37+//
38+// In a micro-VM it is the kernel's init (PID 1, root = the read-only exec disk,
39+// /dev/vdb = shared base image, /dev/vdc = per-job scratch disk): it mounts the
40+// pseudo filesystems, sets the clock and the DNS server, builds an overlayfs
41+// root (lower = base, upper = scratch) and pivots into it, then does what a
42+// container does.
43+//
44+// In a container the root and the pseudo filesystems are ready: it skips all of
45+// that and only prepares cgroups. Then, in both: start dockerd with its data
46+// root on the scratch space, wait for its socket, run the executor
47+// (`ermite-runner exec`) as a child, reap orphans, stop dockerd, and finish by
48+// powering the VM off, or exiting with the executor's code in a container.
49+func Init(ctx context.Context, cfg InitConfig) int {
50+ log := zerowrap.FromCtx(ctx)
51+ if cfg.Getpid == nil {
52+ cfg.Getpid = os.Getpid
53+ }
54+ if cfg.DockerTimeout == 0 {
55+ cfg.DockerTimeout = 60 * time.Second
56+ }
57+ if cfg.ExecArgs == nil {
58+ cfg.ExecArgs = []string{sandbox.GuestExecDir + "/" + sandbox.GuestExecName, "exec"}
59+ }
60+ cmdline := cfg.Cmdline
61+ if cmdline == "" {
62+ b, err := os.ReadFile("/proc/cmdline")
63+ if err != nil && cfg.Getpid() == 1 {
64+ // A VM's init starts with nothing mounted: /proc is needed to read
65+ // the kernel command line that says what we are. setupVM reuses it.
66+ if merr := unix.Mount("proc", "/proc", "proc", 0, ""); merr == nil {
67+ b, err = os.ReadFile("/proc/cmdline")
68+ }
69+ }
70+ if err == nil {
71+ cmdline = string(b)
72+ }
73+ }
74+ args := ParseCmdline(cmdline)
75+ inVM := args[sandbox.CmdlineSandbox] == string(sandbox.VM) && cfg.Getpid() == 1
76+
77+ mode := string(sandbox.Container)
78+ if inVM {
79+ mode = string(sandbox.VM)
80+ if err := setupVM(log, args); err != nil {
81+ log.Error().Err(err).Msg("init: vm setup failed")
82+ powerOff()
83+ return 1
84+ }
85+ }
86+ code := runPayload(ctx, cfg, mode, args)
87+ if inVM {
88+ powerOff()
89+ }
90+ return code
91+}
92+
93+// defaultPath is the PATH of a sandbox whose init was started by the kernel
94+// with an empty environment.
95+const defaultPath = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
96+
97+func runPayload(ctx context.Context, cfg InitConfig, mode string, args map[string]string) int {
98+ log := zerowrap.FromCtx(ctx)
99+ if os.Getenv("PATH") == "" {
100+ _ = os.Setenv("PATH", defaultPath)
101+ }
102+ if os.Getenv("HOME") == "" {
103+ _ = os.Setenv("HOME", "/root")
104+ }
105+ if mode == string(sandbox.Container) {
106+ if err := nestCgroups(); err != nil {
107+ log.Warn().Err(err).Msg("init: cgroup setup (docker inside may not start)")
108+ }
109+ }
110+ target := args[sandbox.CmdlineConnect]
111+ if target == "" {
112+ target = os.Getenv("ERMITE_CONNECT")
113+ }
114+ if _, err := ParseTarget(target); err != nil {
115+ log.Error().Err(err).Msg("init: no executor target")
116+ return 1
117+ }
118+
119+ var dockerPid int
120+ if _, err := os.Stat("/usr/bin/dockerd"); err == nil && !cfg.SkipDocker {
121+ _ = os.MkdirAll(sandbox.GuestScratch+"/docker", 0o711)
122+ pid, err := spawn([]string{"/usr/bin/dockerd", "--data-root", sandbox.GuestScratch + "/docker",
123+ "--storage-driver", "overlay2", "--host", "unix:///var/run/docker.sock"}, nil)
124+ if err != nil {
125+ log.Error().Err(err).Msg("init: start dockerd")
126+ } else {
127+ dockerPid = pid
128+ }
129+ }
130+
131+ execPid, err := spawn(cfg.ExecArgs, append(os.Environ(),
132+ "ERMITE_CONNECT="+target, "ERMITE_SANDBOX="+mode, "ERMITE_VERSION="+cfg.Version))
133+ if err != nil {
134+ log.Error().Err(err).Msg("init: start executor")
135+ return 1
136+ }
137+ // Docker is awaited in the background: the executor starts right away and
138+ // the stub script's `docker info` waits for the socket itself.
139+ if dockerPid != 0 {
140+ go waitSocket(ctx, "/var/run/docker.sock", cfg.DockerTimeout)
141+ }
142+ code := reapUntil(execPid)
143+ if dockerPid != 0 {
144+ _ = syscall.Kill(dockerPid, syscall.SIGTERM)
145+ deadline := time.Now().Add(5 * time.Second)
146+ for time.Now().Before(deadline) {
147+ if pid, _ := wait4NoHang(dockerPid); pid == dockerPid {
148+ break
149+ }
150+ time.Sleep(100 * time.Millisecond)
151+ }
152+ _ = syscall.Kill(dockerPid, syscall.SIGKILL)
153+ }
154+ return code
155+}
156+
157+func waitSocket(ctx context.Context, path string, timeout time.Duration) {
158+ log := zerowrap.FromCtx(ctx)
159+ deadline := time.Now().Add(timeout)
160+ for time.Now().Before(deadline) {
161+ if _, err := os.Stat(path); err == nil {
162+ return
163+ }
164+ time.Sleep(200 * time.Millisecond)
165+ }
166+ log.Warn().Str("socket", path).Msg("init: docker socket did not appear")
167+}
168+
169+// spawn starts argv[0] as a child (own process group) without os/exec, so that
170+// reapUntil is the only waiter and orphans can be reaped safely.
171+func spawn(argv, env []string) (int, error) {
172+ if env == nil {
173+ env = os.Environ()
174+ }
175+ return syscall.ForkExec(argv[0], argv, &syscall.ProcAttr{
176+ Env: env,
177+ Files: []uintptr{0, 1, 2},
178+ Sys: &syscall.SysProcAttr{Setpgid: true},
179+ })
180+}
181+
182+// reapUntil waits for every child (orphans included) and returns when pid
183+// exits, with its exit code (128+signal when killed).
184+func reapUntil(pid int) int {
185+ for {
186+ var ws syscall.WaitStatus
187+ got, err := syscall.Wait4(-1, &ws, 0, nil)
188+ if err == syscall.EINTR {
189+ continue
190+ }
191+ if err != nil { // ECHILD: no child left, the executor included
192+ return 1
193+ }
194+ if got == pid {
195+ switch {
196+ case ws.Exited():
197+ return ws.ExitStatus()
198+ case ws.Signaled():
199+ return 128 + int(ws.Signal())
200+ }
201+ return 1
202+ }
203+ }
204+}
205+
206+func wait4NoHang(pid int) (int, error) {
207+ var ws syscall.WaitStatus
208+ return syscall.Wait4(pid, &ws, syscall.WNOHANG, nil)
209+}
210+
211+func powerOff() {
212+ unix.Sync()
213+ // On x86 Firecracker has no ACPI power off: a reboot makes the VMM exit
214+ // (kernel cmdline reboot=k). On arm64 PSCI handles power off.
215+ _ = unix.Reboot(unix.LINUX_REBOOT_CMD_RESTART)
216+ _ = unix.Reboot(unix.LINUX_REBOOT_CMD_POWER_OFF)
217+}
218+
219+// nestCgroups prepares cgroup v2 for a nested dockerd in a container: a cgroup
220+// with member processes cannot delegate controllers to children, so move every
221+// process to an "init" child and enable the controllers (like the upstream
222+// docker-in-docker entrypoint). No-op without cgroup v2.
223+func nestCgroups() error {
224+ const root = "/sys/fs/cgroup"
225+ ctl, err := os.ReadFile(root + "/cgroup.controllers")
226+ if err != nil {
227+ return nil
228+ }
229+ if err := os.MkdirAll(root+"/init", 0o755); err != nil {
230+ return err
231+ }
232+ procs, err := os.ReadFile(root + "/cgroup.procs")
233+ if err != nil {
234+ return err
235+ }
236+ for _, p := range strings.Fields(string(procs)) {
237+ // Processes may exit meanwhile: ignore per-process errors.
238+ _ = os.WriteFile(root+"/init/cgroup.procs", []byte(p), 0o644)
239+ }
240+ var enable []string
241+ for _, c := range strings.Fields(string(ctl)) {
242+ enable = append(enable, "+"+c)
243+ }
244+ if len(enable) == 0 {
245+ return nil
246+ }
247+ if err := os.WriteFile(root+"/cgroup.subtree_control", []byte(strings.Join(enable, " ")), 0o644); err != nil {
248+ return fmt.Errorf("enable controllers: %w", err)
249+ }
250+ return nil
251+}
@@ -0,0 +1,251 @@
1+// SPDX-License-Identifier: MIT
2+
3+package guest
4+
5+import (
6+ "context"
7+ "fmt"
8+ "os"
9+ "strings"
10+ "syscall"
11+ "time"
12+
13+ "github.com/bnema/zerowrap"
14+ "golang.org/x/sys/unix"
15+
16+ "bnema.dev/ermite-runner/internal/sandbox"
17+)
18+
19+// InitConfig configures Init. The zero value reads the real system; tests
20+// replace the fields.
21+type InitConfig struct {
22+ Version string
23+ // Cmdline is the kernel command line; empty reads /proc/cmdline.
24+ Cmdline string
25+ // Getpid reports the process id; Init only acts as a VM's PID 1 when it is 1.
26+ Getpid func() int
27+ // DockerTimeout bounds the wait for the Docker socket (default 60 s).
28+ DockerTimeout time.Duration
29+ // SkipDocker does not start dockerd (tests, images without Docker).
30+ SkipDocker bool
31+ // ExecArgs is the executor command line; default
32+ // [GuestExecDir/GuestExecName exec].
33+ ExecArgs []string
34+}
35+
36+// Init is `ermite-runner init`: the entrypoint of every sandbox.
37+//
38+// In a micro-VM it is the kernel's init (PID 1, root = the read-only exec disk,
39+// /dev/vdb = shared base image, /dev/vdc = per-job scratch disk): it mounts the
40+// pseudo filesystems, sets the clock and the DNS server, builds an overlayfs
41+// root (lower = base, upper = scratch) and pivots into it, then does what a
42+// container does.
43+//
44+// In a container the root and the pseudo filesystems are ready: it skips all of
45+// that and only prepares cgroups. Then, in both: start dockerd with its data
46+// root on the scratch space, wait for its socket, run the executor
47+// (`ermite-runner exec`) as a child, reap orphans, stop dockerd, and finish by
48+// powering the VM off, or exiting with the executor's code in a container.
49+func Init(ctx context.Context, cfg InitConfig) int {
50+ log := zerowrap.FromCtx(ctx)
51+ if cfg.Getpid == nil {
52+ cfg.Getpid = os.Getpid
53+ }
54+ if cfg.DockerTimeout == 0 {
55+ cfg.DockerTimeout = 60 * time.Second
56+ }
57+ if cfg.ExecArgs == nil {
58+ cfg.ExecArgs = []string{sandbox.GuestExecDir + "/" + sandbox.GuestExecName, "exec"}
59+ }
60+ cmdline := cfg.Cmdline
61+ if cmdline == "" {
62+ b, err := os.ReadFile("/proc/cmdline")
63+ if err != nil && cfg.Getpid() == 1 {
64+ // A VM's init starts with nothing mounted: /proc is needed to read
65+ // the kernel command line that says what we are. setupVM reuses it.
66+ if merr := unix.Mount("proc", "/proc", "proc", 0, ""); merr == nil {
67+ b, err = os.ReadFile("/proc/cmdline")
68+ }
69+ }
70+ if err == nil {
71+ cmdline = string(b)
72+ }
73+ }
74+ args := ParseCmdline(cmdline)
75+ inVM := args[sandbox.CmdlineSandbox] == string(sandbox.VM) && cfg.Getpid() == 1
76+
77+ mode := string(sandbox.Container)
78+ if inVM {
79+ mode = string(sandbox.VM)
80+ if err := setupVM(log, args); err != nil {
81+ log.Error().Err(err).Msg("init: vm setup failed")
82+ powerOff()
83+ return 1
84+ }
85+ }
86+ code := runPayload(ctx, cfg, mode, args)
87+ if inVM {
88+ powerOff()
89+ }
90+ return code
91+}
92+
93+// defaultPath is the PATH of a sandbox whose init was started by the kernel
94+// with an empty environment.
95+const defaultPath = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
96+
97+func runPayload(ctx context.Context, cfg InitConfig, mode string, args map[string]string) int {
98+ log := zerowrap.FromCtx(ctx)
99+ if os.Getenv("PATH") == "" {
100+ _ = os.Setenv("PATH", defaultPath)
101+ }
102+ if os.Getenv("HOME") == "" {
103+ _ = os.Setenv("HOME", "/root")
104+ }
105+ if mode == string(sandbox.Container) {
106+ if err := nestCgroups(); err != nil {
107+ log.Warn().Err(err).Msg("init: cgroup setup (docker inside may not start)")
108+ }
109+ }
110+ target := args[sandbox.CmdlineConnect]
111+ if target == "" {
112+ target = os.Getenv("ERMITE_CONNECT")
113+ }
114+ if _, err := ParseTarget(target); err != nil {
115+ log.Error().Err(err).Msg("init: no executor target")
116+ return 1
117+ }
118+
119+ var dockerPid int
120+ if _, err := os.Stat("/usr/bin/dockerd"); err == nil && !cfg.SkipDocker {
121+ _ = os.MkdirAll(sandbox.GuestScratch+"/docker", 0o711)
122+ pid, err := spawn([]string{"/usr/bin/dockerd", "--data-root", sandbox.GuestScratch + "/docker",
123+ "--storage-driver", "overlay2", "--host", "unix:///var/run/docker.sock"}, nil)
124+ if err != nil {
125+ log.Error().Err(err).Msg("init: start dockerd")
126+ } else {
127+ dockerPid = pid
128+ }
129+ }
130+
131+ execPid, err := spawn(cfg.ExecArgs, append(os.Environ(),
132+ "ERMITE_CONNECT="+target, "ERMITE_SANDBOX="+mode, "ERMITE_VERSION="+cfg.Version))
133+ if err != nil {
134+ log.Error().Err(err).Msg("init: start executor")
135+ return 1
136+ }
137+ // Docker is awaited in the background: the executor starts right away and
138+ // the stub script's `docker info` waits for the socket itself.
139+ if dockerPid != 0 {
140+ go waitSocket(ctx, "/var/run/docker.sock", cfg.DockerTimeout)
141+ }
142+ code := reapUntil(execPid)
143+ if dockerPid != 0 {
144+ _ = syscall.Kill(dockerPid, syscall.SIGTERM)
145+ deadline := time.Now().Add(5 * time.Second)
146+ for time.Now().Before(deadline) {
147+ if pid, _ := wait4NoHang(dockerPid); pid == dockerPid {
148+ break
149+ }
150+ time.Sleep(100 * time.Millisecond)
151+ }
152+ _ = syscall.Kill(dockerPid, syscall.SIGKILL)
153+ }
154+ return code
155+}
156+
157+func waitSocket(ctx context.Context, path string, timeout time.Duration) {
158+ log := zerowrap.FromCtx(ctx)
159+ deadline := time.Now().Add(timeout)
160+ for time.Now().Before(deadline) {
161+ if _, err := os.Stat(path); err == nil {
162+ return
163+ }
164+ time.Sleep(200 * time.Millisecond)
165+ }
166+ log.Warn().Str("socket", path).Msg("init: docker socket did not appear")
167+}
168+
169+// spawn starts argv[0] as a child (own process group) without os/exec, so that
170+// reapUntil is the only waiter and orphans can be reaped safely.
171+func spawn(argv, env []string) (int, error) {
172+ if env == nil {
173+ env = os.Environ()
174+ }
175+ return syscall.ForkExec(argv[0], argv, &syscall.ProcAttr{
176+ Env: env,
177+ Files: []uintptr{0, 1, 2},
178+ Sys: &syscall.SysProcAttr{Setpgid: true},
179+ })
180+}
181+
182+// reapUntil waits for every child (orphans included) and returns when pid
183+// exits, with its exit code (128+signal when killed).
184+func reapUntil(pid int) int {
185+ for {
186+ var ws syscall.WaitStatus
187+ got, err := syscall.Wait4(-1, &ws, 0, nil)
188+ if err == syscall.EINTR {
189+ continue
190+ }
191+ if err != nil { // ECHILD: no child left, the executor included
192+ return 1
193+ }
194+ if got == pid {
195+ switch {
196+ case ws.Exited():
197+ return ws.ExitStatus()
198+ case ws.Signaled():
199+ return 128 + int(ws.Signal())
200+ }
201+ return 1
202+ }
203+ }
204+}
205+
206+func wait4NoHang(pid int) (int, error) {
207+ var ws syscall.WaitStatus
208+ return syscall.Wait4(pid, &ws, syscall.WNOHANG, nil)
209+}
210+
211+func powerOff() {
212+ unix.Sync()
213+ // On x86 Firecracker has no ACPI power off: a reboot makes the VMM exit
214+ // (kernel cmdline reboot=k). On arm64 PSCI handles power off.
215+ _ = unix.Reboot(unix.LINUX_REBOOT_CMD_RESTART)
216+ _ = unix.Reboot(unix.LINUX_REBOOT_CMD_POWER_OFF)
217+}
218+
219+// nestCgroups prepares cgroup v2 for a nested dockerd in a container: a cgroup
220+// with member processes cannot delegate controllers to children, so move every
221+// process to an "init" child and enable the controllers (like the upstream
222+// docker-in-docker entrypoint). No-op without cgroup v2.
223+func nestCgroups() error {
224+ const root = "/sys/fs/cgroup"
225+ ctl, err := os.ReadFile(root + "/cgroup.controllers")
226+ if err != nil {
227+ return nil
228+ }
229+ if err := os.MkdirAll(root+"/init", 0o755); err != nil {
230+ return err
231+ }
232+ procs, err := os.ReadFile(root + "/cgroup.procs")
233+ if err != nil {
234+ return err
235+ }
236+ for _, p := range strings.Fields(string(procs)) {
237+ // Processes may exit meanwhile: ignore per-process errors.
238+ _ = os.WriteFile(root+"/init/cgroup.procs", []byte(p), 0o644)
239+ }
240+ var enable []string
241+ for _, c := range strings.Fields(string(ctl)) {
242+ enable = append(enable, "+"+c)
243+ }
244+ if len(enable) == 0 {
245+ return nil
246+ }
247+ if err := os.WriteFile(root+"/cgroup.subtree_control", []byte(strings.Join(enable, " ")), 0o644); err != nil {
248+ return fmt.Errorf("enable controllers: %w", err)
249+ }
250+ return nil
251+}
A internal/guest/vmsetup.go +203 -0
@@ -0,0 +1,203 @@
1+// SPDX-License-Identifier: MIT
2+
3+package guest
4+
5+import (
6+ "fmt"
7+ "os"
8+ "strconv"
9+ "strings"
10+
11+ "github.com/bnema/zerowrap"
12+ "golang.org/x/sys/unix"
13+
14+ "bnema.dev/ermite-runner/internal/sandbox"
15+)
16+
17+// Devices of the micro-VM, in the order the host attaches them (virtio-blk).
18+const (
19+ devBase = "/dev/vdb" // shared read-only base image (ext4)
20+ devScratch = "/dev/vdc" // per-job scratch disk (ext4)
21+)
22+
23+// KernelArg names of the clock and DNS settings (not part of the contract
24+// constants because only init reads them).
25+const (
26+ cmdlineTime = "ermite.time" // host unix time at boot, seconds
27+)
28+
29+// setupVM turns the minimal boot root (the exec disk) into the job's root:
30+// overlayfs of base + scratch, with the exec disk bound read-only at
31+// /opt/ermite, then pivot_root into it and mount the pseudo filesystems.
32+func setupVM(log zerowrap.Logger, args map[string]string) error {
33+ // Pseudo filesystems of the boot root. Their mountpoints exist on the exec
34+ // disk (the root is read-only).
35+ for _, m := range []struct{ src, dst, typ string }{
36+ {"proc", "/proc", "proc"},
37+ {"sysfs", "/sys", "sysfs"},
38+ {"devtmpfs", "/dev", "devtmpfs"},
39+ } {
40+ // proc is mounted by Init (to read the command line) and devtmpfs by the
41+ // kernel (DEVTMPFS_MOUNT): mount only what is missing.
42+ if isMounted(m.dst) {
43+ continue
44+ }
45+ if err := mount(m.src, m.dst, m.typ, 0, ""); err != nil {
46+ return err
47+ }
48+ }
49+ // Firecracker has no RTC on x86: the guest clock starts at the epoch, which
50+ // breaks TLS. The host passes its time on the command line.
51+ if v, ok := args[cmdlineTime]; ok {
52+ if sec, err := strconv.ParseInt(v, 10, 64); err == nil && sec > 1_600_000_000 {
53+ if err := unix.ClockSettime(unix.CLOCK_REALTIME, &unix.Timespec{Sec: sec}); err != nil {
54+ log.Warn().Err(err).Msg("init: set clock")
55+ }
56+ }
57+ }
58+
59+ if err := mount(devBase, "/mnt/base", "ext4", unix.MS_RDONLY, ""); err != nil {
60+ return err
61+ }
62+ if err := mount(devScratch, "/mnt/scratch", "ext4", unix.MS_NOATIME, ""); err != nil {
63+ return err
64+ }
65+ for _, d := range []string{"/mnt/scratch/upper", "/mnt/scratch/work", "/mnt/scratch/docker"} {
66+ if err := os.MkdirAll(d, 0o755); err != nil {
67+ return err
68+ }
69+ }
70+ opts := "lowerdir=/mnt/base,upperdir=/mnt/scratch/upper,workdir=/mnt/scratch/work"
71+ if err := mount("overlay", "/mnt/newroot", "overlay", 0, opts); err != nil {
72+ return err
73+ }
74+ // The new root needs these mountpoints; they live in the (writable) upper.
75+ for _, d := range []string{"/mnt/newroot/proc", "/mnt/newroot/sys", "/mnt/newroot/dev",
76+ "/mnt/newroot/run", "/mnt/newroot/tmp", "/mnt/newroot/scratch", "/mnt/newroot/.oldroot",
77+ "/mnt/newroot" + sandbox.GuestExecDir} {
78+ if err := os.MkdirAll(d, 0o755); err != nil {
79+ return err
80+ }
81+ }
82+ // Scratch and the exec disk stay reachable after the pivot.
83+ if err := mount("/mnt/scratch", "/mnt/newroot"+sandbox.GuestScratch, "", unix.MS_BIND, ""); err != nil {
84+ return err
85+ }
86+ if err := mount("/", "/mnt/newroot"+sandbox.GuestExecDir, "", unix.MS_BIND, ""); err != nil {
87+ return err
88+ }
89+ if err := mount("", "/mnt/newroot"+sandbox.GuestExecDir, "", unix.MS_BIND|unix.MS_REMOUNT|unix.MS_RDONLY, ""); err != nil {
90+ return err
91+ }
92+ for _, d := range []string{"/proc", "/sys", "/dev"} {
93+ if err := mount(d, "/mnt/newroot"+d, "", unix.MS_MOVE, ""); err != nil {
94+ return err
95+ }
96+ }
97+ if err := unix.Chdir("/mnt/newroot"); err != nil {
98+ return err
99+ }
100+ if err := unix.PivotRoot(".", ".oldroot"); err != nil {
101+ return fmt.Errorf("pivot_root: %w", err)
102+ }
103+ if err := unix.Chroot("."); err != nil {
104+ return err
105+ }
106+ if err := unix.Chdir("/"); err != nil {
107+ return err
108+ }
109+ _ = unix.Unmount("/.oldroot", unix.MNT_DETACH)
110+
111+ for _, m := range []struct {
112+ src, dst, typ string
113+ flags uintptr
114+ data string
115+ }{
116+ {"devpts", "/dev/pts", "devpts", 0, "gid=5,mode=620,ptmxmode=666"},
117+ {"tmpfs", "/dev/shm", "tmpfs", unix.MS_NOSUID | unix.MS_NODEV, ""},
118+ {"tmpfs", "/run", "tmpfs", unix.MS_NOSUID | unix.MS_NODEV, "mode=755"},
119+ {"tmpfs", "/tmp", "tmpfs", unix.MS_NOSUID | unix.MS_NODEV, "mode=1777"},
120+ {"cgroup2", "/sys/fs/cgroup", "cgroup2", unix.MS_NOSUID | unix.MS_NODEV | unix.MS_NOEXEC, ""},
121+ } {
122+ if err := os.MkdirAll(m.dst, 0o755); err != nil {
123+ return err
124+ }
125+ if err := mount(m.src, m.dst, m.typ, m.flags, m.data); err != nil {
126+ // devpts and shm failures are not fatal for the stub.
127+ log.Warn().Err(err).Str("target", m.dst).Msg("init: mount")
128+ }
129+ }
130+ _ = os.Chmod("/tmp", 0o1777)
131+
132+ gw := args[sandbox.CmdlineGateway]
133+ if err := writeResolvConf(gw); err != nil {
134+ return err
135+ }
136+ _ = os.WriteFile("/etc/hostname", []byte("ermite-job\n"), 0o644)
137+ _ = os.WriteFile("/etc/hosts", []byte("127.0.0.1 localhost ermite-job\n"), 0o644)
138+ _ = unix.Sethostname([]byte("ermite-job"))
139+ if err := loUp(); err != nil {
140+ log.Warn().Err(err).Msg("init: loopback")
141+ }
142+ // IPv6 is disabled on the kernel command line (ipv6.disable=1); this is a
143+ // second line of defence when the module is built in but not disabled.
144+ _ = os.WriteFile("/proc/sys/net/ipv6/conf/all/disable_ipv6", []byte("1\n"), 0o644)
145+ return nil
146+}
147+
148+// writeResolvConf points the guest at the gateway, where the runner's DNS
149+// forwarder listens. /etc/resolv.conf may be a dangling symlink in the image
150+// (systemd-resolved stub): replace it.
151+func writeResolvConf(gw string) error {
152+ if gw == "" {
153+ return fmt.Errorf("kernel command line has no %s", sandbox.CmdlineGateway)
154+ }
155+ _ = os.Remove("/etc/resolv.conf")
156+ return os.WriteFile("/etc/resolv.conf", []byte("nameserver "+gw+"\noptions timeout:2 attempts:2\n"), 0o644)
157+}
158+
159+func mount(src, dst, typ string, flags uintptr, data string) error {
160+ if err := os.MkdirAll(dst, 0o755); err != nil && !os.IsExist(err) {
161+ // Read-only boot root: the mountpoint must already exist.
162+ if _, serr := os.Stat(dst); serr != nil {
163+ return fmt.Errorf("mountpoint %s: %w", dst, err)
164+ }
165+ }
166+ if err := unix.Mount(src, dst, typ, flags, data); err != nil {
167+ return fmt.Errorf("mount %s on %s (%s): %w", src, dst, strings.TrimSpace(typ), err)
168+ }
169+ return nil
170+}
171+
172+// loUp brings the loopback interface up (the kernel leaves it down).
173+func loUp() error {
174+ fd, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM|unix.SOCK_CLOEXEC, 0)
175+ if err != nil {
176+ return err
177+ }
178+ defer func() { _ = unix.Close(fd) }()
179+ ifr, err := unix.NewIfreq("lo")
180+ if err != nil {
181+ return err
182+ }
183+ if err := unix.IoctlIfreq(fd, unix.SIOCGIFFLAGS, ifr); err != nil {
184+ return err
185+ }
186+ ifr.SetUint16(ifr.Uint16() | unix.IFF_UP)
187+ return unix.IoctlIfreq(fd, unix.SIOCSIFFLAGS, ifr)
188+}
189+
190+// isMounted reports whether path is a mount point listed in /proc/self/mountinfo.
191+func isMounted(path string) bool {
192+ b, err := os.ReadFile("/proc/self/mountinfo")
193+ if err != nil {
194+ return false
195+ }
196+ for _, line := range strings.Split(string(b), "\n") {
197+ f := strings.Fields(line)
198+ if len(f) > 4 && f[4] == path {
199+ return true
200+ }
201+ }
202+ return false
203+}
@@ -0,0 +1,203 @@
1+// SPDX-License-Identifier: MIT
2+
3+package guest
4+
5+import (
6+ "fmt"
7+ "os"
8+ "strconv"
9+ "strings"
10+
11+ "github.com/bnema/zerowrap"
12+ "golang.org/x/sys/unix"
13+
14+ "bnema.dev/ermite-runner/internal/sandbox"
15+)
16+
17+// Devices of the micro-VM, in the order the host attaches them (virtio-blk).
18+const (
19+ devBase = "/dev/vdb" // shared read-only base image (ext4)
20+ devScratch = "/dev/vdc" // per-job scratch disk (ext4)
21+)
22+
23+// KernelArg names of the clock and DNS settings (not part of the contract
24+// constants because only init reads them).
25+const (
26+ cmdlineTime = "ermite.time" // host unix time at boot, seconds
27+)
28+
29+// setupVM turns the minimal boot root (the exec disk) into the job's root:
30+// overlayfs of base + scratch, with the exec disk bound read-only at
31+// /opt/ermite, then pivot_root into it and mount the pseudo filesystems.
32+func setupVM(log zerowrap.Logger, args map[string]string) error {
33+ // Pseudo filesystems of the boot root. Their mountpoints exist on the exec
34+ // disk (the root is read-only).
35+ for _, m := range []struct{ src, dst, typ string }{
36+ {"proc", "/proc", "proc"},
37+ {"sysfs", "/sys", "sysfs"},
38+ {"devtmpfs", "/dev", "devtmpfs"},
39+ } {
40+ // proc is mounted by Init (to read the command line) and devtmpfs by the
41+ // kernel (DEVTMPFS_MOUNT): mount only what is missing.
42+ if isMounted(m.dst) {
43+ continue
44+ }
45+ if err := mount(m.src, m.dst, m.typ, 0, ""); err != nil {
46+ return err
47+ }
48+ }
49+ // Firecracker has no RTC on x86: the guest clock starts at the epoch, which
50+ // breaks TLS. The host passes its time on the command line.
51+ if v, ok := args[cmdlineTime]; ok {
52+ if sec, err := strconv.ParseInt(v, 10, 64); err == nil && sec > 1_600_000_000 {
53+ if err := unix.ClockSettime(unix.CLOCK_REALTIME, &unix.Timespec{Sec: sec}); err != nil {
54+ log.Warn().Err(err).Msg("init: set clock")
55+ }
56+ }
57+ }
58+
59+ if err := mount(devBase, "/mnt/base", "ext4", unix.MS_RDONLY, ""); err != nil {
60+ return err
61+ }
62+ if err := mount(devScratch, "/mnt/scratch", "ext4", unix.MS_NOATIME, ""); err != nil {
63+ return err
64+ }
65+ for _, d := range []string{"/mnt/scratch/upper", "/mnt/scratch/work", "/mnt/scratch/docker"} {
66+ if err := os.MkdirAll(d, 0o755); err != nil {
67+ return err
68+ }
69+ }
70+ opts := "lowerdir=/mnt/base,upperdir=/mnt/scratch/upper,workdir=/mnt/scratch/work"
71+ if err := mount("overlay", "/mnt/newroot", "overlay", 0, opts); err != nil {
72+ return err
73+ }
74+ // The new root needs these mountpoints; they live in the (writable) upper.
75+ for _, d := range []string{"/mnt/newroot/proc", "/mnt/newroot/sys", "/mnt/newroot/dev",
76+ "/mnt/newroot/run", "/mnt/newroot/tmp", "/mnt/newroot/scratch", "/mnt/newroot/.oldroot",
77+ "/mnt/newroot" + sandbox.GuestExecDir} {
78+ if err := os.MkdirAll(d, 0o755); err != nil {
79+ return err
80+ }
81+ }
82+ // Scratch and the exec disk stay reachable after the pivot.
83+ if err := mount("/mnt/scratch", "/mnt/newroot"+sandbox.GuestScratch, "", unix.MS_BIND, ""); err != nil {
84+ return err
85+ }
86+ if err := mount("/", "/mnt/newroot"+sandbox.GuestExecDir, "", unix.MS_BIND, ""); err != nil {
87+ return err
88+ }
89+ if err := mount("", "/mnt/newroot"+sandbox.GuestExecDir, "", unix.MS_BIND|unix.MS_REMOUNT|unix.MS_RDONLY, ""); err != nil {
90+ return err
91+ }
92+ for _, d := range []string{"/proc", "/sys", "/dev"} {
93+ if err := mount(d, "/mnt/newroot"+d, "", unix.MS_MOVE, ""); err != nil {
94+ return err
95+ }
96+ }
97+ if err := unix.Chdir("/mnt/newroot"); err != nil {
98+ return err
99+ }
100+ if err := unix.PivotRoot(".", ".oldroot"); err != nil {
101+ return fmt.Errorf("pivot_root: %w", err)
102+ }
103+ if err := unix.Chroot("."); err != nil {
104+ return err
105+ }
106+ if err := unix.Chdir("/"); err != nil {
107+ return err
108+ }
109+ _ = unix.Unmount("/.oldroot", unix.MNT_DETACH)
110+
111+ for _, m := range []struct {
112+ src, dst, typ string
113+ flags uintptr
114+ data string
115+ }{
116+ {"devpts", "/dev/pts", "devpts", 0, "gid=5,mode=620,ptmxmode=666"},
117+ {"tmpfs", "/dev/shm", "tmpfs", unix.MS_NOSUID | unix.MS_NODEV, ""},
118+ {"tmpfs", "/run", "tmpfs", unix.MS_NOSUID | unix.MS_NODEV, "mode=755"},
119+ {"tmpfs", "/tmp", "tmpfs", unix.MS_NOSUID | unix.MS_NODEV, "mode=1777"},
120+ {"cgroup2", "/sys/fs/cgroup", "cgroup2", unix.MS_NOSUID | unix.MS_NODEV | unix.MS_NOEXEC, ""},
121+ } {
122+ if err := os.MkdirAll(m.dst, 0o755); err != nil {
123+ return err
124+ }
125+ if err := mount(m.src, m.dst, m.typ, m.flags, m.data); err != nil {
126+ // devpts and shm failures are not fatal for the stub.
127+ log.Warn().Err(err).Str("target", m.dst).Msg("init: mount")
128+ }
129+ }
130+ _ = os.Chmod("/tmp", 0o1777)
131+
132+ gw := args[sandbox.CmdlineGateway]
133+ if err := writeResolvConf(gw); err != nil {
134+ return err
135+ }
136+ _ = os.WriteFile("/etc/hostname", []byte("ermite-job\n"), 0o644)
137+ _ = os.WriteFile("/etc/hosts", []byte("127.0.0.1 localhost ermite-job\n"), 0o644)
138+ _ = unix.Sethostname([]byte("ermite-job"))
139+ if err := loUp(); err != nil {
140+ log.Warn().Err(err).Msg("init: loopback")
141+ }
142+ // IPv6 is disabled on the kernel command line (ipv6.disable=1); this is a
143+ // second line of defence when the module is built in but not disabled.
144+ _ = os.WriteFile("/proc/sys/net/ipv6/conf/all/disable_ipv6", []byte("1\n"), 0o644)
145+ return nil
146+}
147+
148+// writeResolvConf points the guest at the gateway, where the runner's DNS
149+// forwarder listens. /etc/resolv.conf may be a dangling symlink in the image
150+// (systemd-resolved stub): replace it.
151+func writeResolvConf(gw string) error {
152+ if gw == "" {
153+ return fmt.Errorf("kernel command line has no %s", sandbox.CmdlineGateway)
154+ }
155+ _ = os.Remove("/etc/resolv.conf")
156+ return os.WriteFile("/etc/resolv.conf", []byte("nameserver "+gw+"\noptions timeout:2 attempts:2\n"), 0o644)
157+}
158+
159+func mount(src, dst, typ string, flags uintptr, data string) error {
160+ if err := os.MkdirAll(dst, 0o755); err != nil && !os.IsExist(err) {
161+ // Read-only boot root: the mountpoint must already exist.
162+ if _, serr := os.Stat(dst); serr != nil {
163+ return fmt.Errorf("mountpoint %s: %w", dst, err)
164+ }
165+ }
166+ if err := unix.Mount(src, dst, typ, flags, data); err != nil {
167+ return fmt.Errorf("mount %s on %s (%s): %w", src, dst, strings.TrimSpace(typ), err)
168+ }
169+ return nil
170+}
171+
172+// loUp brings the loopback interface up (the kernel leaves it down).
173+func loUp() error {
174+ fd, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM|unix.SOCK_CLOEXEC, 0)
175+ if err != nil {
176+ return err
177+ }
178+ defer func() { _ = unix.Close(fd) }()
179+ ifr, err := unix.NewIfreq("lo")
180+ if err != nil {
181+ return err
182+ }
183+ if err := unix.IoctlIfreq(fd, unix.SIOCGIFFLAGS, ifr); err != nil {
184+ return err
185+ }
186+ ifr.SetUint16(ifr.Uint16() | unix.IFF_UP)
187+ return unix.IoctlIfreq(fd, unix.SIOCSIFFLAGS, ifr)
188+}
189+
190+// isMounted reports whether path is a mount point listed in /proc/self/mountinfo.
191+func isMounted(path string) bool {
192+ b, err := os.ReadFile("/proc/self/mountinfo")
193+ if err != nil {
194+ return false
195+ }
196+ for _, line := range strings.Split(string(b), "\n") {
197+ f := strings.Fields(line)
198+ if len(f) > 4 && f[4] == path {
199+ return true
200+ }
201+ }
202+ return false
203+}

Keyboard shortcuts

Single-key shortcuts are on. ctrl k and esc work whatever this says.

ctrl k :
go to a section, branch, tag, file or repository
?
this list
j k
move the row cursor down / up
enter
open the row under the cursor; on Files, a directory opens or closes in place
↑ ↓ ← → home end
in the file tree (focus on a row): previous / next row, close / open a directory or go to its parent / first entry, first / last row. They never open a file
l h
on Files: open a directory or a file / close a directory or go up to its parent, then to the parent page. Elsewhere: into the side panel (on a merge request, its files) / back to the content; there j k move between its controls
g g G
first / last row
/
filter the list; on Files, find a file anywhere in the repository (esc clears, enter opens the best match)
b
switch branch or tag
g s g f g c g b g m g i g r g a
tabs: summary, files, commits, branches & tags, merge requests, issues, releases, actions (when available)
g h
home
w
wide mode: use the whole width of the screen, or not (remembered)
y
copy a permalink pinned to the commit
d u x
diff: split / unified / collapse or expand all
e E
diff: the whole file in view / every file, with the lines the hunks hide
n
merge request: next note in the diff
esc
close a dialog, leave a field

Branches 4

Tags 11

↑↓ moveenter openesc close