1// SPDX-License-Identifier: MIT
2
3package container
4
5import (
6 "context"
7 "net"
8 "net/http"
9 "os"
10 "path/filepath"
11 "strings"
12 "testing"
13
14 "bnema.dev/ermite-runner/internal/sandbox"
15 "bnema.dev/ermite-runner/internal/sandbox/sandboxtest"
16)
17
18func cacheSock(d *Driver, sb sandbox.Sandbox) string {
19 return filepath.Join(d.sockDir(sb.ID()), sandbox.CacheSocket)
20}
21
22// Each container gets its own cache socket in its private run directory (the
23// one directory bind-mounted into it), served for its own job only: a token of
24// the other container's job gets a 403 on it, and nothing listens on a network.
25// Stopping a container removes its channel and leaves the other one alone.
26func TestCacheSocketPerContainer(t *testing.T) {
27 e := newCEnv(t)
28 svc := sandboxtest.NewCache(t)
29 e.cfg.Cache = svc
30 d := e.driver(t)
31 start := func(job string) sandbox.Sandbox {
32 t.Helper()
33 sb, err := d.New()
34 if err != nil {
35 t.Fatal(err)
36 }
37 if _, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: job}); err != nil {
38 t.Fatal(err)
39 }
40 return sb
41 }
42 a, b := start("job-a"), start("job-b")
43 tokA, tokB := sandboxtest.CacheToken(svc, "job-a"), sandboxtest.CacheToken(svc, "job-b")
44
45 if cacheSock(d, a) == cacheSock(d, b) {
46 t.Fatal("two containers share a cache socket")
47 }
48 // The socket is in the directory that is mounted as /run/ermite, next to
49 // the executor socket, and in no other container's.
50 e.eng.mu.Lock()
51 mounts := map[string]bool{}
52 for _, c := range e.eng.containers {
53 for _, bm := range c.Binds {
54 if bm.Target == sandbox.ContainerRunDir {
55 mounts[bm.Source] = true
56 }
57 }
58 }
59 e.eng.mu.Unlock()
60 for _, sb := range []sandbox.Sandbox{a, b} {
61 if !mounts[filepath.Dir(cacheSock(d, sb))] {
62 t.Fatalf("%s: the cache socket is not in a directory mounted in a container", sb.ID())
63 }
64 if st, err := os.Stat(cacheSock(d, sb)); err != nil || st.Mode().Perm() != 0o666 {
65 t.Fatalf("%s: cache socket %v %v", sb.ID(), st, err)
66 }
67 }
68
69 for _, tc := range []struct {
70 sb sandbox.Sandbox
71 tok string
72 want int
73 }{
74 {a, tokA, http.StatusNoContent},
75 {b, tokB, http.StatusNoContent},
76 {a, tokB, http.StatusForbidden},
77 {b, tokA, http.StatusForbidden},
78 } {
79 got, err := sandboxtest.CacheLookup(cacheSock(d, tc.sb), tc.tok)
80 if err != nil || got != tc.want {
81 t.Errorf("%s: status %d err %v, want %d", tc.sb.ID(), got, err, tc.want)
82 }
83 }
84
85 if err := a.Stop(context.Background()); err != nil {
86 t.Fatal(err)
87 }
88 if c, err := net.Dial("unix", cacheSock(d, a)); err == nil {
89 _ = c.Close()
90 t.Fatal("A's cache socket answers after Stop")
91 }
92 if got, err := sandboxtest.CacheLookup(cacheSock(d, b), tokB); err != nil || got != http.StatusNoContent {
93 t.Fatalf("B after A's Stop: %d %v", got, err)
94 }
95 if err := b.Stop(context.Background()); err != nil {
96 t.Fatal(err)
97 }
98 if left := e.leftovers(); len(left) != 0 {
99 t.Fatalf("leftovers: %v", left)
100 }
101}
102
103// A failed start does not leave the channel behind.
104func TestCacheSocketClosedWhenStartFails(t *testing.T) {
105 e := newCEnv(t)
106 svc := sandboxtest.NewCache(t)
107 e.cfg.Cache = svc
108 e.eng.failStart = os.ErrInvalid
109 d := e.driver(t)
110 sb, _ := d.New()
111 if _, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"}); err == nil {
112 t.Fatal("expected a start error")
113 }
114 if err := sb.Stop(context.Background()); err != nil {
115 t.Fatal(err)
116 }
117 // The same sandbox id could be served again: the service forgot it.
118 ln, err := net.Listen("unix", filepath.Join(t.TempDir(), "s"))
119 if err != nil {
120 t.Fatal(err)
121 }
122 if err := svc.Serve(sb.ID(), ln, "j"); err != nil {
123 t.Fatalf("the channel of the failed start is still registered: %v", err)
124 }
125}
126
127// The sun_path limit covers the cache socket, which is one byte longer than the
128// executor socket: a directory that fits the executor socket exactly is refused
129// when the cache is on, and still accepted when it is not.
130func TestSockPathLimitCoversTheCacheSocket(t *testing.T) {
131 pad := func(n int) string { return "/" + strings.Repeat("d", n-1) }
132 // dir + "/exec.sock" is exactly maxSockPath bytes.
133 exact := pad(maxSockPath - len("/"+sandbox.ContainerSocket))
134 if len(filepath.Join(exact, sandbox.ContainerSocket)) != maxSockPath {
135 t.Fatalf("setup: %d", len(filepath.Join(exact, sandbox.ContainerSocket)))
136 }
137 if err := checkSockPaths(exact, false); err != nil {
138 t.Fatalf("%d-byte executor socket without cache: %v", maxSockPath, err)
139 }
140 err := checkSockPaths(exact, true)
141 if err == nil || !strings.Contains(err.Error(), "cache socket path") || !strings.Contains(err.Error(), sandbox.EnvStateDir) {
142 t.Fatalf("%d-byte executor socket with cache: %v", maxSockPath, err)
143 }
144 // One byte shorter: both fit.
145 if err := checkSockPaths(exact[:len(exact)-1], true); err != nil {
146 t.Fatalf("both sockets fit: %v", err)
147 }
148 // One byte longer: the executor socket is refused first.
149 if err := checkSockPaths(exact+"d", true); err == nil || !strings.Contains(err.Error(), "executor socket path") {
150 t.Fatalf("executor socket over the limit: %v", err)
151 }
152}
153
154// At Start, a state directory that fits the executor socket but not the cache
155// socket fails with the actionable error and leaves nothing behind.
156func TestStartRefusesACacheSocketPathOverTheLimit(t *testing.T) {
157 e := newCEnv(t)
158 e.cfg.Cache = sandboxtest.NewCache(t)
159 pid, err := sandbox.NewID()
160 if err != nil {
161 t.Fatal(err)
162 }
163 // Grow the state directory so that exec.sock is exactly at the limit (ids
164 // have a fixed length).
165 probe := &Driver{cfg: e.cfg}
166 grow := maxSockPath - len(filepath.Join(probe.sockDir(pid), sandbox.ContainerSocket))
167 if grow < 2 {
168 t.Skipf("state directory already %d bytes over", -grow)
169 }
170 e.cfg.StateDir = e.cfg.StateDir + "/" + strings.Repeat("d", grow-1)
171 d := e.driver(t)
172 sb, err := d.New()
173 if err != nil {
174 t.Fatal(err)
175 }
176 if got := len(filepath.Join(d.sockDir(sb.ID()), sandbox.ContainerSocket)); got != maxSockPath {
177 t.Fatalf("setup: executor socket path is %d bytes, want %d", got, maxSockPath)
178 }
179 _, err = sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
180 if err == nil || !strings.Contains(err.Error(), "cache socket path") || !strings.Contains(err.Error(), "limit of a Unix socket") {
181 t.Fatalf("got %v", err)
182 }
183 if left := e.leftovers(); len(left) != 0 {
184 t.Fatalf("leftovers: %v", left)
185 }
186}
187
188// Without a cache service there is no cache socket.
189func TestNoCacheSocketWithoutService(t *testing.T) {
190 e := newCEnv(t)
191 d := e.driver(t)
192 sb, _ := d.New()
193 if _, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"}); err != nil {
194 t.Fatal(err)
195 }
196 if _, err := os.Stat(cacheSock(d, sb)); err == nil {
197 t.Fatal("a cache socket exists with no cache service")
198 }
199 _ = sb.Stop(context.Background())
200}