1// SPDX-License-Identifier: MIT
2
3package container
4
5import (
6 "context"
7 "io"
8 "os"
9 "os/exec"
10 "path/filepath"
11 "strings"
12 "testing"
13 "time"
14
15 "bnema.dev/ermite-runner/internal/cmdrun"
16 "bnema.dev/ermite-runner/internal/execstream"
17 "bnema.dev/ermite-runner/internal/execstream/execpb"
18 "bnema.dev/ermite-runner/internal/sandbox"
19 "bnema.dev/ermite-runner/internal/sandbox/disk"
20 "bnema.dev/ermite-runner/internal/sandbox/sandboxtest"
21)
22
23// TestDockerAPIWiring runs the real Docker Engine calls (network with a named
24// bridge, labelled container, bind mounts, wait, removal, reaping by
25// label) against whatever daemon DOCKER_HOST points at, including a rootless
26// one. It does not test isolation (a rootless daemon cannot run the nested
27// dockerd): the integration test in internal/sandbox/integration does.
28//
29// ERMITE_RUNNER_IT_DOCKER_API=1 ERMITE_RUNNER_TEST_IMAGE=<image> go test -run DockerAPIWiring ./internal/sandbox/container
30func TestDockerAPIWiring(t *testing.T) {
31 if os.Getenv("ERMITE_RUNNER_IT_DOCKER_API") != "1" {
32 t.Skip("set ERMITE_RUNNER_IT_DOCKER_API=1 (needs a reachable Docker daemon)")
33 }
34 image := os.Getenv("ERMITE_RUNNER_TEST_IMAGE")
35 if image == "" {
36 t.Fatal("set ERMITE_RUNNER_TEST_IMAGE to an image with /bin/sh (e.g. the runner image)")
37 }
38 if _, err := exec.LookPath("docker"); err != nil {
39 t.Skip("docker CLI needed for the verification")
40 }
41 // A static test executable that plays the executor: the entrypoint of the
42 // container is "<exe> init", and init runs "<exe> exec"... replaced here by a
43 // tiny shell executor that speaks the stream through a Go helper binary.
44 dir := t.TempDir()
45 helper := filepath.Join(dir, "ermite-runner")
46 build := exec.Command("go", "build", "-o", helper, "bnema.dev/ermite-runner")
47 build.Env = append(os.Environ(), "CGO_ENABLED=0", "GOWORK=off")
48 build.Dir = repoRoot(t)
49 if out, err := build.CombinedOutput(); err != nil {
50 t.Fatalf("build runner: %v\n%s", err, out)
51 }
52
53 plumb := sandboxtest.NewPlumbing()
54 cfg := sandbox.Config{
55 Isolation: sandbox.Container, StateDir: filepath.Join(dir, "state"), Capacity: 1, Image: image,
56 JobCPUs: 1, JobMemoryMiB: 512, JobDiskMiB: 1024, UIDBase: 900000,
57 Network: mustPrefix("10.213.0.0/16"), Executable: helper, DNSPort: 0,
58 LockFile: filepath.Join(dir, "host.lock"), JobPids: 4096,
59 }
60 ctx := context.Background()
61 // Loop mounts need root: replace them by plain directories (mkfs still runs).
62 run := noMountRunner{}
63 empty := filepath.Join(dir, "mountinfo")
64 _ = os.WriteFile(empty, nil, 0o644)
65 d, err := New(ctx, cfg, Options{Net: plumb, UnsafeAllowRootlessForTests: true, ConnectTimeout: 60 * time.Second, Run: run,
66 Mounter: &disk.Mounter{Run: run, Mountinfo: empty}})
67 if err != nil {
68 t.Fatal(err)
69 }
70 defer func() { _ = d.Close() }()
71 if err := d.Reap(ctx); err != nil {
72 t.Fatal(err)
73 }
74
75 for i := range 2 {
76 sb, err := d.New()
77 if err != nil {
78 t.Fatal(err)
79 }
80 conn, err := sb.Start(ctx, sandbox.JobSpec{JobID: "api-wiring"})
81 if err != nil {
82 t.Fatalf("job %d: %v", i, err)
83 }
84 var out strings.Builder
85 hello, exit, err := execstream.Drive(conn, &execpb.Run{Script: "echo wired; id -u; exit 4"}, nil, func(o *execpb.Output) { out.Write(o.Data) })
86 if err != nil {
87 t.Fatal(err)
88 }
89 if hello.Sandbox != "container" || exit.Code != 4 || !strings.HasPrefix(out.String(), "wired\n") {
90 t.Fatalf("hello=%v exit=%v out=%q", hello, exit, out.String())
91 }
92 // While running, Docker knows our objects by label.
93 if i == 0 {
94 if ids := dockerLines(t, "ps", "-aq", "--filter", "label="+sandbox.LabelJob); len(ids) != 1 {
95 t.Fatalf("expected one labelled container, got %v", ids)
96 }
97 if n := dockerLines(t, "network", "ls", "-q", "--filter", "label="+sandbox.LabelJob); len(n) != 1 {
98 t.Fatalf("expected one labelled network, got %v", n)
99 }
100 }
101 if err := sb.Stop(ctx); err != nil {
102 t.Fatal(err)
103 }
104 }
105 for _, args := range [][]string{
106 {"ps", "-aq", "--filter", "label=" + sandbox.LabelJob},
107 {"network", "ls", "-q", "--filter", "label=" + sandbox.LabelJob},
108 } {
109 if l := dockerLines(t, args...); len(l) != 0 {
110 t.Fatalf("docker %v left %v", args, l)
111 }
112 }
113 if plumb.Live() != 0 {
114 t.Fatal("firewall attachments leaked")
115 }
116}
117
118// noMountRunner runs real commands except mount and umount, which succeed
119// without doing anything.
120type noMountRunner struct{}
121
122func (noMountRunner) Run(ctx context.Context, in io.Reader, name string, args ...string) ([]byte, error) {
123 if name == "mount" || name == "umount" {
124 return nil, nil
125 }
126 return cmdrun.Exec{}.Run(ctx, in, name, args...)
127}
128
129func dockerLines(t *testing.T, args ...string) []string {
130 t.Helper()
131 out, err := exec.Command("docker", args...).Output()
132 if err != nil {
133 t.Fatalf("docker %v: %v", args, err)
134 }
135 return strings.Fields(string(out))
136}
137
138func repoRoot(t *testing.T) string {
139 t.Helper()
140 out, err := exec.Command("go", "list", "-m", "-f", "{{.Dir}}").Output()
141 if err != nil {
142 t.Fatal(err)
143 }
144 return strings.TrimSpace(string(out))
145}