detached ea1f07e

test(sandbox): add opt-in VM and container integration tests

commit
ea1f07e7c2607287375591f7921b0256831b1d14
author
brice · · 2026-10-12 15:35 UTC
parent
d928a74
signature
GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email
4 files · +517 -0
Split
UnifiedSplit
A internal/sandbox/integration/container_test.go +43 -0
@@ -0,0 +1,43 @@
1+// SPDX-License-Identifier: MIT
2+
3+package integration
4+
5+import (
6+ "os"
7+ "os/exec"
8+ "strings"
9+ "testing"
10+
11+ "bnema.dev/ermite-runner/internal/sandbox"
12+ "bnema.dev/ermite-runner/internal/sandbox/drivers"
13+)
14+
15+func TestContainerStubExecutor(t *testing.T) {
16+ if os.Getenv("ERMITE_RUNNER_IT_CONTAINER") != "1" {
17+ t.Skip("set ERMITE_RUNNER_IT_CONTAINER=1 (needs a rootful Docker daemon and the runner image; see README)")
18+ }
19+ need(t, "docker")
20+ needNetAdmin(t)
21+ ctx := ctxWithLog(t)
22+ cfg := loadConfig(t, sandbox.Container)
23+ // A rootless daemon is refused with a clear error: nothing to test there.
24+ if _, err := drivers.Factories().Container(ctx, cfg); err != nil {
25+ if strings.Contains(err.Error(), "rootless") {
26+ t.Skipf("the Docker daemon is rootless, as designed the driver refuses it: %v", err)
27+ }
28+ t.Fatal(err)
29+ }
30+ scenario(t, ctx, cfg, sandbox.Container, func(t *testing.T, sb sandbox.Sandbox) {
31+ out, err := exec.Command("docker", "ps", "-q", "--filter", "label="+sandbox.LabelJob).Output()
32+ if err != nil {
33+ t.Fatal(err)
34+ }
35+ ids := strings.Fields(string(out))
36+ if len(ids) == 0 {
37+ t.Fatal("no job container to kill")
38+ }
39+ if b, err := exec.Command("docker", append([]string{"kill"}, ids...)...).CombinedOutput(); err != nil {
40+ t.Fatalf("docker kill: %v: %s", err, b)
41+ }
42+ })
43+}
@@ -0,0 +1,43 @@
1+// SPDX-License-Identifier: MIT
2+
3+package integration
4+
5+import (
6+ "os"
7+ "os/exec"
8+ "strings"
9+ "testing"
10+
11+ "bnema.dev/ermite-runner/internal/sandbox"
12+ "bnema.dev/ermite-runner/internal/sandbox/drivers"
13+)
14+
15+func TestContainerStubExecutor(t *testing.T) {
16+ if os.Getenv("ERMITE_RUNNER_IT_CONTAINER") != "1" {
17+ t.Skip("set ERMITE_RUNNER_IT_CONTAINER=1 (needs a rootful Docker daemon and the runner image; see README)")
18+ }
19+ need(t, "docker")
20+ needNetAdmin(t)
21+ ctx := ctxWithLog(t)
22+ cfg := loadConfig(t, sandbox.Container)
23+ // A rootless daemon is refused with a clear error: nothing to test there.
24+ if _, err := drivers.Factories().Container(ctx, cfg); err != nil {
25+ if strings.Contains(err.Error(), "rootless") {
26+ t.Skipf("the Docker daemon is rootless, as designed the driver refuses it: %v", err)
27+ }
28+ t.Fatal(err)
29+ }
30+ scenario(t, ctx, cfg, sandbox.Container, func(t *testing.T, sb sandbox.Sandbox) {
31+ out, err := exec.Command("docker", "ps", "-q", "--filter", "label="+sandbox.LabelJob).Output()
32+ if err != nil {
33+ t.Fatal(err)
34+ }
35+ ids := strings.Fields(string(out))
36+ if len(ids) == 0 {
37+ t.Fatal("no job container to kill")
38+ }
39+ if b, err := exec.Command("docker", append([]string{"kill"}, ids...)...).CombinedOutput(); err != nil {
40+ t.Fatalf("docker kill: %v: %s", err, b)
41+ }
42+ })
43+}
A internal/sandbox/integration/integration_test.go +330 -0
@@ -0,0 +1,330 @@
1+// SPDX-License-Identifier: MIT
2+
3+// Package integration holds the end-to-end tests of the sandbox drivers with the
4+// real stub executor: output comes back, exit codes are reported, and nothing
5+// survives 10 jobs, including killed ones. They need privileges and host
6+// software, so they are opt-in:
7+//
8+// ERMITE_RUNNER_IT_VM=1 Firecracker + KVM + kernel + base image,
9+// CAP_NET_ADMIN (TAP, nftables, port 53 on the
10+// TAP address)
11+// ERMITE_RUNNER_IT_CONTAINER=1 a ROOTFUL Docker daemon (DOCKER_HOST) and the
12+// runner image, CAP_NET_ADMIN for nftables
13+//
14+// See README.md ("Integration tests") for the exact commands, including how to
15+// run the VM test without root inside a user and network namespace.
16+package integration
17+
18+import (
19+ "context"
20+ "fmt"
21+ "os"
22+ "os/exec"
23+ "path/filepath"
24+ "regexp"
25+ "strings"
26+ "testing"
27+ "time"
28+
29+ "github.com/bnema/zerowrap"
30+
31+ "bnema.dev/ermite-runner/internal/execstream"
32+ "bnema.dev/ermite-runner/internal/execstream/execpb"
33+ "bnema.dev/ermite-runner/internal/netpolicy"
34+ "bnema.dev/ermite-runner/internal/sandbox"
35+ "bnema.dev/ermite-runner/internal/sandbox/drivers"
36+)
37+
38+// stubScript is the CI-1 stub job of the plan.
39+const stubScript = "uname -a; id; cat /etc/os-release; docker info"
40+
41+func ctxWithLog(t *testing.T) context.Context {
42+ return zerowrap.WithCtx(context.Background(), zerowrap.New(zerowrap.Config{Level: "info", Format: "console", Output: testWriter{t}}))
43+}
44+
45+type testWriter struct{ t *testing.T }
46+
47+func (w testWriter) Write(p []byte) (int, error) {
48+ w.t.Log(strings.TrimRight(string(p), "\n"))
49+ return len(p), nil
50+}
51+
52+// buildRunner builds the static runner executable the sandboxes attach.
53+func buildRunner(t *testing.T) string {
54+ t.Helper()
55+ out, err := exec.Command("go", "list", "-m", "-f", "{{.Dir}}").Output()
56+ if err != nil {
57+ t.Fatal(err)
58+ }
59+ exe := filepath.Join(t.TempDir(), "ermite-runner")
60+ cmd := exec.Command("go", "build", "-o", exe, "bnema.dev/ermite-runner")
61+ cmd.Dir = strings.TrimSpace(string(out))
62+ cmd.Env = append(os.Environ(), "CGO_ENABLED=0", "GOWORK=off")
63+ if b, err := cmd.CombinedOutput(); err != nil {
64+ t.Fatalf("build runner: %v\n%s", err, b)
65+ }
66+ return exe
67+}
68+
69+// loadConfig reads the sandbox settings from the environment, like the runner
70+// does, over a private state directory.
71+func loadConfig(t *testing.T, mode sandbox.Mode) sandbox.Config {
72+ t.Helper()
73+ getenv := func(k string) string {
74+ switch k {
75+ case sandbox.EnvIsolation:
76+ return string(mode)
77+ case sandbox.EnvStateDir:
78+ return stateDir(t)
79+ case sandbox.EnvCapacity:
80+ return "2"
81+ case sandbox.EnvJobDisk:
82+ if v := os.Getenv(k); v != "" {
83+ return v
84+ }
85+ return "4G"
86+ case sandbox.EnvJobMemory:
87+ if v := os.Getenv(k); v != "" {
88+ return v
89+ }
90+ return "1024"
91+ }
92+ return os.Getenv(k)
93+ }
94+ cfg, err := sandbox.LoadConfig(getenv)
95+ if err != nil {
96+ t.Fatal(err)
97+ }
98+ cfg.Executable = buildRunner(t)
99+ return cfg
100+}
101+
102+var stateDirs = map[*testing.T]string{}
103+
104+func stateDir(t *testing.T) string {
105+ if d, ok := stateDirs[t]; ok {
106+ return d
107+ }
108+ base := os.Getenv("ERMITE_RUNNER_IT_STATE_DIR")
109+ if base == "" {
110+ base = t.TempDir()
111+ } else {
112+ base = filepath.Join(base, fmt.Sprintf("it-%d", time.Now().UnixNano()))
113+ _ = os.MkdirAll(base, 0o755)
114+ t.Cleanup(func() { _ = os.RemoveAll(base) })
115+ }
116+ stateDirs[t] = base
117+ return base
118+}
119+
120+type result struct {
121+ out, errOut string
122+ hello *execpb.Hello
123+ exit *execpb.Exit
124+}
125+
126+func runJob(ctx context.Context, conn sandbox.Conn, script string) (result, error) {
127+ var so, se strings.Builder
128+ hello, exit, err := execstream.Drive(ctx, conn, &execpb.Run{Script: script}, func(o *execpb.Output) {
129+ if o.Stream == execpb.StreamKind_STREAM_KIND_STDERR {
130+ se.Write(o.Data)
131+ } else {
132+ so.Write(o.Data)
133+ }
134+ })
135+ return result{so.String(), se.String(), hello, exit}, err
136+}
137+
138+// need skips the test unless every command is available.
139+func need(t *testing.T, cmds ...string) {
140+ t.Helper()
141+ for _, c := range cmds {
142+ if _, err := exec.LookPath(c); err != nil {
143+ t.Skipf("%s not found in PATH", c)
144+ }
145+ }
146+}
147+
148+// needNetAdmin skips unless the process may create TAP devices and nftables
149+// tables (root, CAP_NET_ADMIN, or a user+network namespace).
150+func needNetAdmin(t *testing.T) {
151+ t.Helper()
152+ need(t, "ip", "nft")
153+ const probe = "ertapprobe"
154+ if out, err := exec.Command("ip", "tuntap", "add", "dev", probe, "mode", "tap").CombinedOutput(); err != nil {
155+ t.Skipf("cannot create a TAP device (need CAP_NET_ADMIN): %v: %s", err, strings.TrimSpace(string(out)))
156+ }
157+ _ = exec.Command("ip", "link", "del", "dev", probe).Run()
158+ if out, err := exec.Command("nft", "list", "tables").CombinedOutput(); err != nil {
159+ t.Skipf("cannot use nftables (need CAP_NET_ADMIN): %v: %s", err, strings.TrimSpace(string(out)))
160+ }
161+}
162+
163+// audit fails the test when anything of a job survives: network devices, the
164+// nftables table, state files, processes.
165+func audit(t *testing.T, stateDir string) {
166+ t.Helper()
167+ deadline := time.Now().Add(5 * time.Second)
168+ var left []string
169+ for {
170+ left = leftovers(stateDir)
171+ if len(left) == 0 || time.Now().After(deadline) {
172+ break
173+ }
174+ time.Sleep(100 * time.Millisecond)
175+ }
176+ if len(left) != 0 {
177+ t.Fatalf("leftovers after the jobs:\n %s", strings.Join(left, "\n "))
178+ }
179+}
180+
181+var tapRe = regexp.MustCompile(`(?m)^\d+: (ertap\d+|erbr\d+)[:@]`)
182+
183+func leftovers(stateDir string) []string {
184+ var left []string
185+ if out, err := exec.Command("ip", "-o", "link", "show").Output(); err == nil {
186+ for _, m := range tapRe.FindAllStringSubmatch(string(out), -1) {
187+ left = append(left, "network device "+m[1])
188+ }
189+ }
190+ if out, err := exec.Command("nft", "list", "tables").Output(); err == nil && strings.Contains(string(out), netpolicy.Table) {
191+ left = append(left, "nftables table "+netpolicy.Table)
192+ }
193+ for _, d := range []string{"vm", "vms", "jail", "containers"} {
194+ _ = filepath.WalkDir(filepath.Join(stateDir, d), func(p string, e os.DirEntry, err error) error {
195+ if err == nil && p != filepath.Join(stateDir, d) {
196+ rel, _ := filepath.Rel(stateDir, p)
197+ left = append(left, "file "+rel)
198+ }
199+ return nil
200+ })
201+ }
202+ procs, _ := filepath.Glob("/proc/[0-9]*/cmdline")
203+ for _, p := range procs {
204+ b, err := os.ReadFile(p)
205+ if err != nil {
206+ continue
207+ }
208+ args := strings.Split(strings.TrimRight(string(b), "\x00"), "\x00")
209+ if len(args) > 0 && filepath.Base(args[0]) == "firecracker" {
210+ for i, a := range args {
211+ if a == "--id" && i+1 < len(args) && sandbox.ValidID(args[i+1]) {
212+ left = append(left, "process "+p+" "+args[i+1])
213+ }
214+ }
215+ }
216+ }
217+ return left
218+}
219+
220+func checkStubOutput(t *testing.T, r result, wantSandbox string) {
221+ t.Helper()
222+ if r.hello == nil || r.hello.Sandbox != wantSandbox {
223+ t.Errorf("hello: %v", r.hello)
224+ }
225+ for _, want := range []string{"Linux", "uid=", "Ubuntu", "Server Version"} {
226+ if !strings.Contains(r.out, want) {
227+ t.Errorf("output lacks %q:\n%s\nstderr:\n%s", want, r.out, r.errOut)
228+ }
229+ }
230+ if r.exit == nil || r.exit.Code != 0 {
231+ t.Errorf("stub exit: %v\nstderr: %s", r.exit, r.errOut)
232+ }
233+}
234+
235+// jobs runs the common scenario on a driver: 10 jobs, 2 at a time where
236+// capacity allows, including a failing job, a job killed from outside, and a
237+// job abandoned by a "crashed" runner; then audits.
238+func scenario(t *testing.T, ctx context.Context, cfg sandbox.Config, mode sandbox.Mode, killSandbox func(t *testing.T, sb sandbox.Sandbox)) {
239+ t.Helper()
240+ drv, err := sandbox.Select(ctx, cfg, drivers.Factories(), sandbox.Prober{})
241+ if err != nil {
242+ t.Fatal(err)
243+ }
244+ if drv.Mode() != mode {
245+ t.Fatalf("driver mode %s, want %s", drv.Mode(), mode)
246+ }
247+ if err := drv.Reap(ctx); err != nil {
248+ t.Fatalf("initial reap: %v", err)
249+ }
250+
251+ start := func(id string) (sandbox.Sandbox, sandbox.Conn) {
252+ sb, err := drv.New()
253+ if err != nil {
254+ t.Fatal(err)
255+ }
256+ conn, err := sb.Start(ctx, sandbox.JobSpec{JobID: id})
257+ if err != nil {
258+ _ = sb.Stop(ctx)
259+ t.Fatalf("job %s: %v", id, err)
260+ }
261+ return sb, conn
262+ }
263+
264+ // Jobs 1-7: the stub, in pairs (capacity 2) except the last.
265+ for i := 1; i <= 7; i++ {
266+ sb, conn := start(fmt.Sprintf("stub-%d", i))
267+ r, err := runJob(ctx, conn, stubScript)
268+ if err != nil {
269+ t.Fatalf("job %d: %v", i, err)
270+ }
271+ checkStubOutput(t, r, string(mode))
272+ if err := sb.Stop(ctx); err != nil {
273+ t.Fatalf("job %d stop: %v", i, err)
274+ }
275+ }
276+
277+ // Job 8: a failing script: the exit code comes back.
278+ {
279+ sb, conn := start("failing")
280+ r, err := runJob(ctx, conn, "echo before; echo oops >&2; exit 42")
281+ if err != nil {
282+ t.Fatal(err)
283+ }
284+ if r.exit.Code != 42 || !strings.Contains(r.out, "before") || !strings.Contains(r.errOut, "oops") {
285+ t.Errorf("failing job: %+v", r)
286+ }
287+ if err := sb.Stop(ctx); err != nil {
288+ t.Fatal(err)
289+ }
290+ }
291+
292+ // Job 9: killed from outside while running (the VMM or the container dies).
293+ {
294+ sb, conn := start("killed")
295+ done := make(chan error, 1)
296+ go func() { _, err := runJob(ctx, conn, "echo started; sleep 600"); done <- err }()
297+ time.Sleep(2 * time.Second)
298+ killSandbox(t, sb)
299+ select {
300+ case err := <-done:
301+ if err == nil {
302+ t.Error("a killed sandbox must make the session fail")
303+ }
304+ case <-time.After(30 * time.Second):
305+ t.Error("session did not end after the sandbox was killed")
306+ }
307+ if err := sb.Stop(ctx); err != nil {
308+ t.Fatalf("stop after kill: %v", err)
309+ }
310+ }
311+
312+ // Job 10: the runner itself dies mid-job (no Stop, driver closed); the next
313+ // start reaps everything.
314+ {
315+ _, conn := start("abandoned")
316+ go func() { _, _ = runJob(ctx, conn, "sleep 600") }()
317+ time.Sleep(2 * time.Second)
318+ _ = drv.Close() // what a crash does: locks and listeners vanish, the sandbox stays
319+ _ = conn.Close()
320+ drv2, err := sandbox.Select(ctx, cfg, drivers.Factories(), sandbox.Prober{})
321+ if err != nil {
322+ t.Fatal(err)
323+ }
324+ defer func() { _ = drv2.Close() }()
325+ if err := drv2.Reap(ctx); err != nil {
326+ t.Fatalf("reap after crash: %v", err)
327+ }
328+ }
329+ audit(t, cfg.StateDir)
330+}
@@ -0,0 +1,330 @@
1+// SPDX-License-Identifier: MIT
2+
3+// Package integration holds the end-to-end tests of the sandbox drivers with the
4+// real stub executor: output comes back, exit codes are reported, and nothing
5+// survives 10 jobs, including killed ones. They need privileges and host
6+// software, so they are opt-in:
7+//
8+// ERMITE_RUNNER_IT_VM=1 Firecracker + KVM + kernel + base image,
9+// CAP_NET_ADMIN (TAP, nftables, port 53 on the
10+// TAP address)
11+// ERMITE_RUNNER_IT_CONTAINER=1 a ROOTFUL Docker daemon (DOCKER_HOST) and the
12+// runner image, CAP_NET_ADMIN for nftables
13+//
14+// See README.md ("Integration tests") for the exact commands, including how to
15+// run the VM test without root inside a user and network namespace.
16+package integration
17+
18+import (
19+ "context"
20+ "fmt"
21+ "os"
22+ "os/exec"
23+ "path/filepath"
24+ "regexp"
25+ "strings"
26+ "testing"
27+ "time"
28+
29+ "github.com/bnema/zerowrap"
30+
31+ "bnema.dev/ermite-runner/internal/execstream"
32+ "bnema.dev/ermite-runner/internal/execstream/execpb"
33+ "bnema.dev/ermite-runner/internal/netpolicy"
34+ "bnema.dev/ermite-runner/internal/sandbox"
35+ "bnema.dev/ermite-runner/internal/sandbox/drivers"
36+)
37+
38+// stubScript is the CI-1 stub job of the plan.
39+const stubScript = "uname -a; id; cat /etc/os-release; docker info"
40+
41+func ctxWithLog(t *testing.T) context.Context {
42+ return zerowrap.WithCtx(context.Background(), zerowrap.New(zerowrap.Config{Level: "info", Format: "console", Output: testWriter{t}}))
43+}
44+
45+type testWriter struct{ t *testing.T }
46+
47+func (w testWriter) Write(p []byte) (int, error) {
48+ w.t.Log(strings.TrimRight(string(p), "\n"))
49+ return len(p), nil
50+}
51+
52+// buildRunner builds the static runner executable the sandboxes attach.
53+func buildRunner(t *testing.T) string {
54+ t.Helper()
55+ out, err := exec.Command("go", "list", "-m", "-f", "{{.Dir}}").Output()
56+ if err != nil {
57+ t.Fatal(err)
58+ }
59+ exe := filepath.Join(t.TempDir(), "ermite-runner")
60+ cmd := exec.Command("go", "build", "-o", exe, "bnema.dev/ermite-runner")
61+ cmd.Dir = strings.TrimSpace(string(out))
62+ cmd.Env = append(os.Environ(), "CGO_ENABLED=0", "GOWORK=off")
63+ if b, err := cmd.CombinedOutput(); err != nil {
64+ t.Fatalf("build runner: %v\n%s", err, b)
65+ }
66+ return exe
67+}
68+
69+// loadConfig reads the sandbox settings from the environment, like the runner
70+// does, over a private state directory.
71+func loadConfig(t *testing.T, mode sandbox.Mode) sandbox.Config {
72+ t.Helper()
73+ getenv := func(k string) string {
74+ switch k {
75+ case sandbox.EnvIsolation:
76+ return string(mode)
77+ case sandbox.EnvStateDir:
78+ return stateDir(t)
79+ case sandbox.EnvCapacity:
80+ return "2"
81+ case sandbox.EnvJobDisk:
82+ if v := os.Getenv(k); v != "" {
83+ return v
84+ }
85+ return "4G"
86+ case sandbox.EnvJobMemory:
87+ if v := os.Getenv(k); v != "" {
88+ return v
89+ }
90+ return "1024"
91+ }
92+ return os.Getenv(k)
93+ }
94+ cfg, err := sandbox.LoadConfig(getenv)
95+ if err != nil {
96+ t.Fatal(err)
97+ }
98+ cfg.Executable = buildRunner(t)
99+ return cfg
100+}
101+
102+var stateDirs = map[*testing.T]string{}
103+
104+func stateDir(t *testing.T) string {
105+ if d, ok := stateDirs[t]; ok {
106+ return d
107+ }
108+ base := os.Getenv("ERMITE_RUNNER_IT_STATE_DIR")
109+ if base == "" {
110+ base = t.TempDir()
111+ } else {
112+ base = filepath.Join(base, fmt.Sprintf("it-%d", time.Now().UnixNano()))
113+ _ = os.MkdirAll(base, 0o755)
114+ t.Cleanup(func() { _ = os.RemoveAll(base) })
115+ }
116+ stateDirs[t] = base
117+ return base
118+}
119+
120+type result struct {
121+ out, errOut string
122+ hello *execpb.Hello
123+ exit *execpb.Exit
124+}
125+
126+func runJob(ctx context.Context, conn sandbox.Conn, script string) (result, error) {
127+ var so, se strings.Builder
128+ hello, exit, err := execstream.Drive(ctx, conn, &execpb.Run{Script: script}, func(o *execpb.Output) {
129+ if o.Stream == execpb.StreamKind_STREAM_KIND_STDERR {
130+ se.Write(o.Data)
131+ } else {
132+ so.Write(o.Data)
133+ }
134+ })
135+ return result{so.String(), se.String(), hello, exit}, err
136+}
137+
138+// need skips the test unless every command is available.
139+func need(t *testing.T, cmds ...string) {
140+ t.Helper()
141+ for _, c := range cmds {
142+ if _, err := exec.LookPath(c); err != nil {
143+ t.Skipf("%s not found in PATH", c)
144+ }
145+ }
146+}
147+
148+// needNetAdmin skips unless the process may create TAP devices and nftables
149+// tables (root, CAP_NET_ADMIN, or a user+network namespace).
150+func needNetAdmin(t *testing.T) {
151+ t.Helper()
152+ need(t, "ip", "nft")
153+ const probe = "ertapprobe"
154+ if out, err := exec.Command("ip", "tuntap", "add", "dev", probe, "mode", "tap").CombinedOutput(); err != nil {
155+ t.Skipf("cannot create a TAP device (need CAP_NET_ADMIN): %v: %s", err, strings.TrimSpace(string(out)))
156+ }
157+ _ = exec.Command("ip", "link", "del", "dev", probe).Run()
158+ if out, err := exec.Command("nft", "list", "tables").CombinedOutput(); err != nil {
159+ t.Skipf("cannot use nftables (need CAP_NET_ADMIN): %v: %s", err, strings.TrimSpace(string(out)))
160+ }
161+}
162+
163+// audit fails the test when anything of a job survives: network devices, the
164+// nftables table, state files, processes.
165+func audit(t *testing.T, stateDir string) {
166+ t.Helper()
167+ deadline := time.Now().Add(5 * time.Second)
168+ var left []string
169+ for {
170+ left = leftovers(stateDir)
171+ if len(left) == 0 || time.Now().After(deadline) {
172+ break
173+ }
174+ time.Sleep(100 * time.Millisecond)
175+ }
176+ if len(left) != 0 {
177+ t.Fatalf("leftovers after the jobs:\n %s", strings.Join(left, "\n "))
178+ }
179+}
180+
181+var tapRe = regexp.MustCompile(`(?m)^\d+: (ertap\d+|erbr\d+)[:@]`)
182+
183+func leftovers(stateDir string) []string {
184+ var left []string
185+ if out, err := exec.Command("ip", "-o", "link", "show").Output(); err == nil {
186+ for _, m := range tapRe.FindAllStringSubmatch(string(out), -1) {
187+ left = append(left, "network device "+m[1])
188+ }
189+ }
190+ if out, err := exec.Command("nft", "list", "tables").Output(); err == nil && strings.Contains(string(out), netpolicy.Table) {
191+ left = append(left, "nftables table "+netpolicy.Table)
192+ }
193+ for _, d := range []string{"vm", "vms", "jail", "containers"} {
194+ _ = filepath.WalkDir(filepath.Join(stateDir, d), func(p string, e os.DirEntry, err error) error {
195+ if err == nil && p != filepath.Join(stateDir, d) {
196+ rel, _ := filepath.Rel(stateDir, p)
197+ left = append(left, "file "+rel)
198+ }
199+ return nil
200+ })
201+ }
202+ procs, _ := filepath.Glob("/proc/[0-9]*/cmdline")
203+ for _, p := range procs {
204+ b, err := os.ReadFile(p)
205+ if err != nil {
206+ continue
207+ }
208+ args := strings.Split(strings.TrimRight(string(b), "\x00"), "\x00")
209+ if len(args) > 0 && filepath.Base(args[0]) == "firecracker" {
210+ for i, a := range args {
211+ if a == "--id" && i+1 < len(args) && sandbox.ValidID(args[i+1]) {
212+ left = append(left, "process "+p+" "+args[i+1])
213+ }
214+ }
215+ }
216+ }
217+ return left
218+}
219+
220+func checkStubOutput(t *testing.T, r result, wantSandbox string) {
221+ t.Helper()
222+ if r.hello == nil || r.hello.Sandbox != wantSandbox {
223+ t.Errorf("hello: %v", r.hello)
224+ }
225+ for _, want := range []string{"Linux", "uid=", "Ubuntu", "Server Version"} {
226+ if !strings.Contains(r.out, want) {
227+ t.Errorf("output lacks %q:\n%s\nstderr:\n%s", want, r.out, r.errOut)
228+ }
229+ }
230+ if r.exit == nil || r.exit.Code != 0 {
231+ t.Errorf("stub exit: %v\nstderr: %s", r.exit, r.errOut)
232+ }
233+}
234+
235+// jobs runs the common scenario on a driver: 10 jobs, 2 at a time where
236+// capacity allows, including a failing job, a job killed from outside, and a
237+// job abandoned by a "crashed" runner; then audits.
238+func scenario(t *testing.T, ctx context.Context, cfg sandbox.Config, mode sandbox.Mode, killSandbox func(t *testing.T, sb sandbox.Sandbox)) {
239+ t.Helper()
240+ drv, err := sandbox.Select(ctx, cfg, drivers.Factories(), sandbox.Prober{})
241+ if err != nil {
242+ t.Fatal(err)
243+ }
244+ if drv.Mode() != mode {
245+ t.Fatalf("driver mode %s, want %s", drv.Mode(), mode)
246+ }
247+ if err := drv.Reap(ctx); err != nil {
248+ t.Fatalf("initial reap: %v", err)
249+ }
250+
251+ start := func(id string) (sandbox.Sandbox, sandbox.Conn) {
252+ sb, err := drv.New()
253+ if err != nil {
254+ t.Fatal(err)
255+ }
256+ conn, err := sb.Start(ctx, sandbox.JobSpec{JobID: id})
257+ if err != nil {
258+ _ = sb.Stop(ctx)
259+ t.Fatalf("job %s: %v", id, err)
260+ }
261+ return sb, conn
262+ }
263+
264+ // Jobs 1-7: the stub, in pairs (capacity 2) except the last.
265+ for i := 1; i <= 7; i++ {
266+ sb, conn := start(fmt.Sprintf("stub-%d", i))
267+ r, err := runJob(ctx, conn, stubScript)
268+ if err != nil {
269+ t.Fatalf("job %d: %v", i, err)
270+ }
271+ checkStubOutput(t, r, string(mode))
272+ if err := sb.Stop(ctx); err != nil {
273+ t.Fatalf("job %d stop: %v", i, err)
274+ }
275+ }
276+
277+ // Job 8: a failing script: the exit code comes back.
278+ {
279+ sb, conn := start("failing")
280+ r, err := runJob(ctx, conn, "echo before; echo oops >&2; exit 42")
281+ if err != nil {
282+ t.Fatal(err)
283+ }
284+ if r.exit.Code != 42 || !strings.Contains(r.out, "before") || !strings.Contains(r.errOut, "oops") {
285+ t.Errorf("failing job: %+v", r)
286+ }
287+ if err := sb.Stop(ctx); err != nil {
288+ t.Fatal(err)
289+ }
290+ }
291+
292+ // Job 9: killed from outside while running (the VMM or the container dies).
293+ {
294+ sb, conn := start("killed")
295+ done := make(chan error, 1)
296+ go func() { _, err := runJob(ctx, conn, "echo started; sleep 600"); done <- err }()
297+ time.Sleep(2 * time.Second)
298+ killSandbox(t, sb)
299+ select {
300+ case err := <-done:
301+ if err == nil {
302+ t.Error("a killed sandbox must make the session fail")
303+ }
304+ case <-time.After(30 * time.Second):
305+ t.Error("session did not end after the sandbox was killed")
306+ }
307+ if err := sb.Stop(ctx); err != nil {
308+ t.Fatalf("stop after kill: %v", err)
309+ }
310+ }
311+
312+ // Job 10: the runner itself dies mid-job (no Stop, driver closed); the next
313+ // start reaps everything.
314+ {
315+ _, conn := start("abandoned")
316+ go func() { _, _ = runJob(ctx, conn, "sleep 600") }()
317+ time.Sleep(2 * time.Second)
318+ _ = drv.Close() // what a crash does: locks and listeners vanish, the sandbox stays
319+ _ = conn.Close()
320+ drv2, err := sandbox.Select(ctx, cfg, drivers.Factories(), sandbox.Prober{})
321+ if err != nil {
322+ t.Fatal(err)
323+ }
324+ defer func() { _ = drv2.Close() }()
325+ if err := drv2.Reap(ctx); err != nil {
326+ t.Fatalf("reap after crash: %v", err)
327+ }
328+ }
329+ audit(t, cfg.StateDir)
330+}
A internal/sandbox/integration/vm_test.go +108 -0
@@ -0,0 +1,108 @@
1+// SPDX-License-Identifier: MIT
2+
3+package integration
4+
5+import (
6+ "context"
7+ "os"
8+ "os/exec"
9+ "strings"
10+ "syscall"
11+ "testing"
12+
13+ "bnema.dev/ermite-runner/internal/sandbox"
14+ "bnema.dev/ermite-runner/internal/sandbox/drivers"
15+)
16+
17+func TestVMStubExecutor(t *testing.T) {
18+ if os.Getenv("ERMITE_RUNNER_IT_VM") != "1" {
19+ t.Skip("set ERMITE_RUNNER_IT_VM=1 (needs KVM, firecracker, a kernel and a base image; see README)")
20+ }
21+ need(t, "mkfs.ext4")
22+ needNetAdmin(t)
23+ ctx := ctxWithLog(t)
24+ cfg := loadConfig(t, sandbox.VM)
25+ if why := sandbox.VMUnavailable(cfg, sandbox.Prober{}); len(why) > 0 {
26+ t.Fatalf("vm isolation is not available here: %s", strings.Join(why, "; "))
27+ }
28+
29+ scenario(t, ctx, cfg, sandbox.VM, func(t *testing.T, sb sandbox.Sandbox) {
30+ // Kill the VMM process of the only running VM.
31+ out, err := exec.Command("pgrep", "-f", "firecracker.* --id er").Output()
32+ if err != nil {
33+ t.Fatalf("no firecracker to kill: %v", err)
34+ }
35+ for _, f := range strings.Fields(string(out)) {
36+ pid := 0
37+ for _, c := range f {
38+ pid = pid*10 + int(c-'0')
39+ }
40+ _ = syscall.Kill(pid, syscall.SIGKILL)
41+ }
42+ })
43+}
44+
45+// The guest must reach DNS on its gateway and nothing else on the host.
46+func TestVMNetworkPolicy(t *testing.T) {
47+ if os.Getenv("ERMITE_RUNNER_IT_VM") != "1" {
48+ t.Skip("set ERMITE_RUNNER_IT_VM=1")
49+ }
50+ need(t, "mkfs.ext4")
51+ needNetAdmin(t)
52+ ctx := ctxWithLog(t)
53+ cfg := loadConfig(t, sandbox.VM)
54+ drv, err := sandbox.Select(ctx, cfg, drivers.Factories(), sandbox.Prober{})
55+ if err != nil {
56+ t.Fatal(err)
57+ }
58+ defer func() { _ = drv.Close() }()
59+ _ = drv.Reap(ctx)
60+ sb, err := drv.New()
61+ if err != nil {
62+ t.Fatal(err)
63+ }
64+ defer func() { _ = sb.Stop(context.Background()) }()
65+ conn, err := sb.Start(ctx, sandbox.JobSpec{JobID: "netpolicy"})
66+ if err != nil {
67+ t.Fatal(err)
68+ }
69+ // 10.200.0.1 is the gateway of slot 0.
70+ script := `
71+probe() { timeout 4 bash -c "echo > /dev/tcp/$1/$2" >/dev/null 2>&1; echo "$1:$2 rc=$?"; }
72+probe 10.200.0.1 22 # host SSH-like port: must be DROPPED (timeout, rc=124), not refused
73+probe 10.200.0.1 8080
74+probe 169.254.169.254 80 # cloud metadata
75+probe 100.64.0.1 80 # CGNAT / VPN overlay
76+probe 192.168.0.1 80 # RFC 1918
77+probe 172.16.0.1 80
78+probe 10.0.0.1 80
79+python3 - <<'PY'
80+import socket
81+q=bytes.fromhex("abcd01000001000000000000076578616d706c6503636f6d0000010001")
82+s=socket.socket(socket.AF_INET,socket.SOCK_DGRAM); s.settimeout(8); s.sendto(q,("10.200.0.1",53))
83+try:
84+ r=s.recv(512); print("dns answered", len(r))
85+except Exception as e: print("dns silent", e)
86+PY
87+ip -6 addr show scope global | grep -c inet6 || true
88+`
89+ r, err := runJob(ctx, conn, script)
90+ if err != nil {
91+ t.Fatal(err)
92+ }
93+ t.Log("\n" + r.out)
94+ for _, line := range strings.Split(r.out, "\n") {
95+ if !strings.Contains(line, " rc=") {
96+ continue
97+ }
98+ if strings.HasSuffix(line, "rc=0") {
99+ t.Errorf("a guest reached a forbidden destination: %s", line)
100+ }
101+ }
102+ if !strings.Contains(r.out, "10.200.0.1:22 rc=124") {
103+ t.Errorf("host port must be dropped (timeout), got:\n%s", r.out)
104+ }
105+ if !strings.Contains(r.out, "dns answered") {
106+ t.Errorf("the DNS forwarder on the gateway must answer:\n%s", r.out)
107+ }
108+}
@@ -0,0 +1,108 @@
1+// SPDX-License-Identifier: MIT
2+
3+package integration
4+
5+import (
6+ "context"
7+ "os"
8+ "os/exec"
9+ "strings"
10+ "syscall"
11+ "testing"
12+
13+ "bnema.dev/ermite-runner/internal/sandbox"
14+ "bnema.dev/ermite-runner/internal/sandbox/drivers"
15+)
16+
17+func TestVMStubExecutor(t *testing.T) {
18+ if os.Getenv("ERMITE_RUNNER_IT_VM") != "1" {
19+ t.Skip("set ERMITE_RUNNER_IT_VM=1 (needs KVM, firecracker, a kernel and a base image; see README)")
20+ }
21+ need(t, "mkfs.ext4")
22+ needNetAdmin(t)
23+ ctx := ctxWithLog(t)
24+ cfg := loadConfig(t, sandbox.VM)
25+ if why := sandbox.VMUnavailable(cfg, sandbox.Prober{}); len(why) > 0 {
26+ t.Fatalf("vm isolation is not available here: %s", strings.Join(why, "; "))
27+ }
28+
29+ scenario(t, ctx, cfg, sandbox.VM, func(t *testing.T, sb sandbox.Sandbox) {
30+ // Kill the VMM process of the only running VM.
31+ out, err := exec.Command("pgrep", "-f", "firecracker.* --id er").Output()
32+ if err != nil {
33+ t.Fatalf("no firecracker to kill: %v", err)
34+ }
35+ for _, f := range strings.Fields(string(out)) {
36+ pid := 0
37+ for _, c := range f {
38+ pid = pid*10 + int(c-'0')
39+ }
40+ _ = syscall.Kill(pid, syscall.SIGKILL)
41+ }
42+ })
43+}
44+
45+// The guest must reach DNS on its gateway and nothing else on the host.
46+func TestVMNetworkPolicy(t *testing.T) {
47+ if os.Getenv("ERMITE_RUNNER_IT_VM") != "1" {
48+ t.Skip("set ERMITE_RUNNER_IT_VM=1")
49+ }
50+ need(t, "mkfs.ext4")
51+ needNetAdmin(t)
52+ ctx := ctxWithLog(t)
53+ cfg := loadConfig(t, sandbox.VM)
54+ drv, err := sandbox.Select(ctx, cfg, drivers.Factories(), sandbox.Prober{})
55+ if err != nil {
56+ t.Fatal(err)
57+ }
58+ defer func() { _ = drv.Close() }()
59+ _ = drv.Reap(ctx)
60+ sb, err := drv.New()
61+ if err != nil {
62+ t.Fatal(err)
63+ }
64+ defer func() { _ = sb.Stop(context.Background()) }()
65+ conn, err := sb.Start(ctx, sandbox.JobSpec{JobID: "netpolicy"})
66+ if err != nil {
67+ t.Fatal(err)
68+ }
69+ // 10.200.0.1 is the gateway of slot 0.
70+ script := `
71+probe() { timeout 4 bash -c "echo > /dev/tcp/$1/$2" >/dev/null 2>&1; echo "$1:$2 rc=$?"; }
72+probe 10.200.0.1 22 # host SSH-like port: must be DROPPED (timeout, rc=124), not refused
73+probe 10.200.0.1 8080
74+probe 169.254.169.254 80 # cloud metadata
75+probe 100.64.0.1 80 # CGNAT / VPN overlay
76+probe 192.168.0.1 80 # RFC 1918
77+probe 172.16.0.1 80
78+probe 10.0.0.1 80
79+python3 - <<'PY'
80+import socket
81+q=bytes.fromhex("abcd01000001000000000000076578616d706c6503636f6d0000010001")
82+s=socket.socket(socket.AF_INET,socket.SOCK_DGRAM); s.settimeout(8); s.sendto(q,("10.200.0.1",53))
83+try:
84+ r=s.recv(512); print("dns answered", len(r))
85+except Exception as e: print("dns silent", e)
86+PY
87+ip -6 addr show scope global | grep -c inet6 || true
88+`
89+ r, err := runJob(ctx, conn, script)
90+ if err != nil {
91+ t.Fatal(err)
92+ }
93+ t.Log("\n" + r.out)
94+ for _, line := range strings.Split(r.out, "\n") {
95+ if !strings.Contains(line, " rc=") {
96+ continue
97+ }
98+ if strings.HasSuffix(line, "rc=0") {
99+ t.Errorf("a guest reached a forbidden destination: %s", line)
100+ }
101+ }
102+ if !strings.Contains(r.out, "10.200.0.1:22 rc=124") {
103+ t.Errorf("host port must be dropped (timeout), got:\n%s", r.out)
104+ }
105+ if !strings.Contains(r.out, "dns answered") {
106+ t.Errorf("the DNS forwarder on the gateway must answer:\n%s", r.out)
107+ }
108+}
A scripts/it-vm-userns.sh +36 -0
@@ -0,0 +1,36 @@
1+#!/usr/bin/env bash
2+# SPDX-License-Identifier: MIT
3+#
4+# Run the VM integration tests WITHOUT root: inside a user + network namespace
5+# the process is "root" with CAP_NET_ADMIN over a private network stack, which is
6+# enough for TAP devices, nftables and the DNS forwarder on the TAP address.
7+# Only /dev/kvm access (group kvm) and the Firecracker binary are needed. The
8+# jailer needs real root, so it is switched off here (see README: this tests the
9+# whole driver except the jailer's chroot, uid drop and cgroups).
10+#
11+# scripts/it-vm-userns.sh [go test args]
12+#
13+# Environment: FIRECRACKER (default: firecracker in PATH), IMAGE_DIR (directory
14+# with vmlinux and base.ext4, default tmp/image). Outbound internet from the
15+# private namespace is not available: the tests do not need it.
16+set -euo pipefail
17+root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
18+cd "$root"
19+fc=${FIRECRACKER:-$(command -v firecracker || true)}
20+[ -x "${fc:-}" ] || { echo "firecracker not found: set FIRECRACKER" >&2; exit 1; }
21+img=${IMAGE_DIR:-$root/tmp/image}
22+exec unshare --user --map-root-user --net bash -c '
23+ ip link set lo up
24+ # A default route, so that forwarded packets reach the firewall instead of
25+ # being answered "network unreachable" by the routing table: the drops the
26+ # network test sees are then the nftables policy.
27+ ip link add dummy0 type dummy
28+ ip addr add 192.0.2.1/24 dev dummy0
29+ ip link set dummy0 up
30+ ip route add default via 192.0.2.254 dev dummy0
31+ export GOWORK=off ERMITE_RUNNER_IT_VM=1 \
32+ ERMITE_RUNNER_ISOLATION=vm ERMITE_RUNNER_IMAGE="$1" \
33+ ERMITE_RUNNER_FIRECRACKER="$2" ERMITE_RUNNER_JAILER_MODE=off
34+ shift 2
35+ exec go test -count=1 -v -timeout 20m ./internal/sandbox/integration "$@"
36+' _ "$img" "$fc" "$@"
@@ -0,0 +1,36 @@
1+#!/usr/bin/env bash
2+# SPDX-License-Identifier: MIT
3+#
4+# Run the VM integration tests WITHOUT root: inside a user + network namespace
5+# the process is "root" with CAP_NET_ADMIN over a private network stack, which is
6+# enough for TAP devices, nftables and the DNS forwarder on the TAP address.
7+# Only /dev/kvm access (group kvm) and the Firecracker binary are needed. The
8+# jailer needs real root, so it is switched off here (see README: this tests the
9+# whole driver except the jailer's chroot, uid drop and cgroups).
10+#
11+# scripts/it-vm-userns.sh [go test args]
12+#
13+# Environment: FIRECRACKER (default: firecracker in PATH), IMAGE_DIR (directory
14+# with vmlinux and base.ext4, default tmp/image). Outbound internet from the
15+# private namespace is not available: the tests do not need it.
16+set -euo pipefail
17+root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
18+cd "$root"
19+fc=${FIRECRACKER:-$(command -v firecracker || true)}
20+[ -x "${fc:-}" ] || { echo "firecracker not found: set FIRECRACKER" >&2; exit 1; }
21+img=${IMAGE_DIR:-$root/tmp/image}
22+exec unshare --user --map-root-user --net bash -c '
23+ ip link set lo up
24+ # A default route, so that forwarded packets reach the firewall instead of
25+ # being answered "network unreachable" by the routing table: the drops the
26+ # network test sees are then the nftables policy.
27+ ip link add dummy0 type dummy
28+ ip addr add 192.0.2.1/24 dev dummy0
29+ ip link set dummy0 up
30+ ip route add default via 192.0.2.254 dev dummy0
31+ export GOWORK=off ERMITE_RUNNER_IT_VM=1 \
32+ ERMITE_RUNNER_ISOLATION=vm ERMITE_RUNNER_IMAGE="$1" \
33+ ERMITE_RUNNER_FIRECRACKER="$2" ERMITE_RUNNER_JAILER_MODE=off
34+ shift 2
35+ exec go test -count=1 -v -timeout 20m ./internal/sandbox/integration "$@"
36+' _ "$img" "$fc" "$@"

Keyboard shortcuts

Single-key shortcuts are on. ctrl k and esc work whatever this says.

ctrl k :
go to a section, branch, tag, file or repository
?
this list
j k
move the row cursor down / up
enter
open the row under the cursor; on Files, a directory opens or closes in place
↑ ↓ ← → home end
in the file tree (focus on a row): previous / next row, close / open a directory or go to its parent / first entry, first / last row. They never open a file
l h
on Files: open a directory or a file / close a directory or go up to its parent, then to the parent page. Elsewhere: into the side panel (on a merge request, its files) / back to the content; there j k move between its controls
g g G
first / last row
/
filter the list; on Files, find a file anywhere in the repository (esc clears, enter opens the best match)
b
switch branch or tag
g s g f g c g b g m g i g r g a
tabs: summary, files, commits, branches & tags, merge requests, issues, releases, actions (when available)
g h
home
w
wide mode: use the whole width of the screen, or not (remembered)
y
copy a permalink pinned to the commit
d u x
diff: split / unified / collapse or expand all
e E
diff: the whole file in view / every file, with the lines the hunks hide
n
merge request: next note in the diff
esc
close a dialog, leave a field

Branches 4

Tags 11

↑↓ moveenter openesc close