1// SPDX-License-Identifier: MIT
2
3package container
4
5import (
6 "bytes"
7 "context"
8 "errors"
9 "fmt"
10 "net"
11 "net/netip"
12 "os"
13 "path/filepath"
14 "slices"
15 "strings"
16 "sync"
17 "testing"
18 "time"
19
20 "github.com/bnema/zerowrap"
21 "github.com/moby/moby/api/types/container"
22
23 "bnema.dev/ermite-runner/internal/cmdrun"
24 "bnema.dev/ermite-runner/internal/execstream"
25 "bnema.dev/ermite-runner/internal/execstream/execpb"
26 "bnema.dev/ermite-runner/internal/sandbox"
27 "bnema.dev/ermite-runner/internal/sandbox/disk"
28 "bnema.dev/ermite-runner/internal/sandbox/sandboxtest"
29)
30
31// fakeEngine plays the Docker daemon. When a container "starts", it plays the
32// executor inside it by connecting to the bind-mounted socket.
33type fakeEngine struct {
34 mu sync.Mutex
35 info daemonInfo
36 infoErr error
37 imageErr error
38 networks map[string]networkSpec
39 containers map[string]containerSpec
40 failStart error
41 noConnect bool
42 exitOnRun bool
43 removeErr error
44 waiters map[string]chan exitResult
45 seq int
46 pulled []string
47}
48
49func newFake() *fakeEngine {
50 return &fakeEngine{
51 info: daemonInfo{OSType: "linux", Runtimes: []string{"runc"}},
52 networks: map[string]networkSpec{},
53 containers: map[string]containerSpec{},
54 waiters: map[string]chan exitResult{},
55 }
56}
57
58func (f *fakeEngine) Info(context.Context) (daemonInfo, error) { return f.info, f.infoErr }
59func (f *fakeEngine) EnsureImage(_ context.Context, ref string) error {
60 f.pulled = append(f.pulled, ref)
61 return f.imageErr
62}
63func (f *fakeEngine) CreateNetwork(_ context.Context, s networkSpec) (string, error) {
64 f.mu.Lock()
65 defer f.mu.Unlock()
66 f.seq++
67 id := fmt.Sprintf("net%d", f.seq)
68 f.networks[id] = s
69 return id, nil
70}
71func (f *fakeEngine) RemoveNetwork(_ context.Context, id string) error {
72 f.mu.Lock()
73 defer f.mu.Unlock()
74 delete(f.networks, id)
75 return nil
76}
77func (f *fakeEngine) ListNetworks(context.Context, string) ([]string, error) {
78 f.mu.Lock()
79 defer f.mu.Unlock()
80 var out []string
81 for id, n := range f.networks {
82 if _, ok := n.Labels[sandbox.LabelJob]; ok {
83 out = append(out, id)
84 }
85 }
86 return out, nil
87}
88func (f *fakeEngine) CreateContainer(_ context.Context, s containerSpec) (string, error) {
89 f.mu.Lock()
90 defer f.mu.Unlock()
91 f.seq++
92 id := fmt.Sprintf("ctr%d", f.seq)
93 f.containers[id] = s
94 f.waiters[id] = make(chan exitResult, 1)
95 return id, nil
96}
97func (f *fakeEngine) StartContainer(_ context.Context, id string) error {
98 f.mu.Lock()
99 spec := f.containers[id]
100 fail, noConnect, exitOnRun := f.failStart, f.noConnect, f.exitOnRun
101 w := f.waiters[id]
102 f.mu.Unlock()
103 if fail != nil {
104 return fail
105 }
106 if exitOnRun {
107 w <- exitResult{Code: 125}
108 return nil
109 }
110 if noConnect {
111 return nil
112 }
113 // The host side of the bind mount is the source of the run-dir mount.
114 var sock string
115 for _, b := range spec.Binds {
116 if b.Target == sandbox.ContainerRunDir {
117 sock = filepath.Join(b.Source, sandbox.ContainerSocket)
118 }
119 }
120 go func() {
121 c, err := net.Dial("unix", sock)
122 if err != nil {
123 return
124 }
125 defer func() { _ = c.Close() }()
126 g := execstream.NewConn(c)
127 _ = g.Send(&execpb.Frame{Body: &execpb.Frame_Hello{Hello: &execpb.Hello{Sandbox: "container"}}})
128 if _, err := g.Recv(); err != nil {
129 return
130 }
131 _ = g.Send(&execpb.Frame{Body: &execpb.Frame_Output{Output: &execpb.Output{Data: []byte("hello from fake container\n")}}})
132 _ = g.Send(&execpb.Frame{Body: &execpb.Frame_Exit{Exit: &execpb.Exit{Code: 7}}})
133 _, _ = g.Recv()
134 }()
135 return nil
136}
137func (f *fakeEngine) WaitExit(_ context.Context, id string) <-chan exitResult {
138 f.mu.Lock()
139 defer f.mu.Unlock()
140 return f.waiters[id]
141}
142func (f *fakeEngine) RemoveContainer(_ context.Context, id string) error {
143 f.mu.Lock()
144 defer f.mu.Unlock()
145 if f.removeErr != nil {
146 return f.removeErr
147 }
148 delete(f.containers, id)
149 return nil
150}
151func (f *fakeEngine) ListContainers(context.Context, string) ([]string, error) {
152 f.mu.Lock()
153 defer f.mu.Unlock()
154 var out []string
155 for id := range f.containers {
156 out = append(out, id)
157 }
158 return out, nil
159}
160func (f *fakeEngine) LogsTail(context.Context, string, int) (string, error) {
161 return "fake container log line", nil
162}
163func (f *fakeEngine) Close() error { return nil }
164
165func (f *fakeEngine) objects() []string {
166 f.mu.Lock()
167 defer f.mu.Unlock()
168 var out []string
169 for id := range f.containers {
170 out = append(out, "container "+id)
171 }
172 for id := range f.networks {
173 out = append(out, "network "+id)
174 }
175 return out
176}
177
178// fakeMounts plays mount/umount: it records mounts and writes a mountinfo file
179// that disk.Mounter reads.
180type fakeMounts struct {
181 mu sync.Mutex
182 mounted map[string]bool
183 file string
184 failUm error
185}
186
187func (m *fakeMounts) sync() {
188 var b strings.Builder
189 for d := range m.mounted {
190 fmt.Fprintf(&b, "40 28 7:0 / %s rw - ext4 /dev/loop0 rw\n", d)
191 }
192 _ = os.WriteFile(m.file, []byte(b.String()), 0o644)
193}
194
195func (m *fakeMounts) handle(c cmdrun.Call) ([]byte, error) {
196 m.mu.Lock()
197 defer m.mu.Unlock()
198 switch c.Name {
199 case "mkfs.ext4":
200 if len(c.Args) == 1 && c.Args[0] == "-V" {
201 return []byte("mke2fs 1.47.4 (6-Mar-2025)\n"), nil
202 }
203 case "mount":
204 m.mounted[c.Args[len(c.Args)-1]] = true
205 m.sync()
206 case "umount":
207 if m.failUm != nil {
208 return nil, m.failUm
209 }
210 delete(m.mounted, c.Args[0])
211 m.sync()
212 }
213 return nil, nil
214}
215
216func (m *fakeMounts) count() int { m.mu.Lock(); defer m.mu.Unlock(); return len(m.mounted) }
217
218type cenv struct {
219 cfg sandbox.Config
220 eng *fakeEngine
221 plumb *sandboxtest.Plumbing
222 mounts *fakeMounts
223 run *cmdrun.Fake
224}
225
226func newCEnv(t *testing.T) *cenv {
227 t.Helper()
228 // A short directory: the executor socket path is limited to 107 bytes and
229 // t.TempDir embeds the (long) test name.
230 dir, err := os.MkdirTemp("", "erc")
231 if err != nil {
232 t.Fatal(err)
233 }
234 t.Cleanup(func() { _ = os.RemoveAll(dir) })
235 exe := filepath.Join(dir, "runner")
236 sandboxtest.WriteStaticELF(t, exe)
237 fm := &fakeMounts{mounted: map[string]bool{}, file: filepath.Join(dir, "mountinfo")}
238 fm.sync()
239 return &cenv{
240 eng: newFake(),
241 plumb: sandboxtest.NewPlumbing(),
242 mounts: fm,
243 run: &cmdrun.Fake{Handle: fm.handle},
244 cfg: sandbox.Config{
245 Isolation: sandbox.Container, StateDir: filepath.Join(dir, "state"), Capacity: 2,
246 JobCPUs: 2, JobMemoryMiB: 1024, JobDiskMiB: 1024, UIDBase: 900000,
247 Network: netip.MustParsePrefix("10.200.0.0/16"), Executable: exe, DNSPort: 0,
248 LockFile: filepath.Join(dir, "host.lock"), JobPids: 32768,
249 },
250 }
251}
252
253func (e *cenv) opts() Options {
254 return Options{engine: e.eng, Net: e.plumb, ConnectTimeout: 3 * time.Second, Run: e.run,
255 Mounter: &disk.Mounter{Run: e.run, Mountinfo: e.mounts.file}}
256}
257
258func (e *cenv) driver(t *testing.T) *Driver {
259 t.Helper()
260 d, err := New(context.Background(), e.cfg, e.opts())
261 if err != nil {
262 t.Fatal(err)
263 }
264 t.Cleanup(func() { _ = d.Close() })
265 return d
266}
267
268func withTimeout(o Options, d time.Duration) Options { o.ConnectTimeout = d; return o }
269
270func (e *cenv) leftovers() []string {
271 out := e.eng.objects()
272 if n := e.mounts.count(); n != 0 {
273 out = append(out, fmt.Sprintf("%d scratch mounts", n))
274 }
275 if n := e.plumb.Live(); n != 0 {
276 out = append(out, fmt.Sprintf("%d attached guests", n))
277 }
278 ents, _ := os.ReadDir(filepath.Join(e.cfg.StateDir, "containers"))
279 for _, d := range ents {
280 out = append(out, "dir "+d.Name())
281 }
282 return out
283}
284
285func stub(t *testing.T, conn sandbox.Conn) (string, int) {
286 t.Helper()
287 var out strings.Builder
288 _, exit, err := execstream.Drive(conn, &execpb.Run{}, nil, func(o *execpb.Output) { out.Write(o.Data) })
289 if err != nil {
290 t.Fatal(err)
291 }
292 return out.String(), int(exit.Code)
293}
294
295func TestRootlessDockerIsRefused(t *testing.T) {
296 e := newCEnv(t)
297 e.eng.info.Rootless = true
298 _, err := New(context.Background(), e.cfg, e.opts())
299 if err == nil || !strings.Contains(err.Error(), "rootless") || !strings.Contains(err.Error(), "rootful") {
300 t.Fatalf("want a clear rootless error, got %v", err)
301 }
302}
303
304func TestRuntimeSelection(t *testing.T) {
305 e := newCEnv(t)
306 d := e.driver(t)
307 if !d.Privileged() || d.runtime != "" {
308 t.Fatal("without sysbox the driver must run privileged")
309 }
310 e2 := newCEnv(t)
311 e2.eng.info.Runtimes = []string{"runc", "sysbox-runc"}
312 d2 := e2.driver(t)
313 if d2.Privileged() || d2.runtime != SysboxRuntime {
314 t.Fatal("sysbox listed: must use it, unprivileged")
315 }
316}
317
318func TestNewErrors(t *testing.T) {
319 e := newCEnv(t)
320 e.eng.infoErr = errors.New("connection refused")
321 if _, err := New(context.Background(), e.cfg, e.opts()); err == nil || !strings.Contains(err.Error(), "DOCKER_HOST") {
322 t.Fatalf("got %v", err)
323 }
324 e = newCEnv(t)
325 e.eng.imageErr = errors.New("no such image")
326 if _, err := New(context.Background(), e.cfg, e.opts()); err == nil {
327 t.Fatal("missing image accepted")
328 }
329 e = newCEnv(t)
330 e.eng.info.OSType = "windows"
331 if _, err := New(context.Background(), e.cfg, e.opts()); err == nil {
332 t.Fatal("windows daemon accepted")
333 }
334 e = newCEnv(t)
335 e.cfg.Executable = "/bin/sh"
336 if _, err := New(context.Background(), e.cfg, e.opts()); err == nil {
337 t.Skip("/bin/sh is static")
338 }
339}
340
341func TestStartRunStop(t *testing.T) {
342 e := newCEnv(t)
343 d := e.driver(t)
344 sb, err := d.New()
345 if err != nil {
346 t.Fatal(err)
347 }
348 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "job-7", CPUs: 3, MemoryMiB: 2048})
349 if err != nil {
350 t.Fatal(err)
351 }
352 out, code := stub(t, conn)
353 if out != "hello from fake container\n" || code != 7 {
354 t.Fatalf("out=%q code=%d", out, code)
355 }
356
357 // What was asked of Docker.
358 e.eng.mu.Lock()
359 var spec containerSpec
360 for _, s := range e.eng.containers {
361 spec = s
362 }
363 var netSpec networkSpec
364 for _, n := range e.eng.networks {
365 netSpec = n
366 }
367 e.eng.mu.Unlock()
368 if !spec.Privileged || spec.Runtime != "" {
369 t.Errorf("privileged fallback expected: %+v", spec)
370 }
371 if spec.Labels[sandbox.LabelJob] != "job-7" || netSpec.Labels[sandbox.LabelJob] != "job-7" {
372 t.Errorf("every object must carry the job label: %v / %v", spec.Labels, netSpec.Labels)
373 }
374 if spec.MemoryMiB != 2048 || spec.CPUs != 3 {
375 t.Errorf("resources: %+v", spec)
376 }
377 if !slices.Equal(spec.Entrypoint, []string{"/opt/ermite/ermite-runner", "init"}) {
378 t.Errorf("entrypoint: %v", spec.Entrypoint)
379 }
380 var roExe, runDir bool
381 for _, b := range spec.Binds {
382 if b.Target == "/opt/ermite/ermite-runner" && b.ReadOnly && b.Source == e.cfg.Executable {
383 roExe = true
384 }
385 if b.Target == sandbox.ContainerRunDir && !b.ReadOnly {
386 runDir = true
387 }
388 }
389 if !roExe || !runDir {
390 t.Errorf("binds: %+v", spec.Binds)
391 }
392 var scratch, work, tmp bool
393 for _, b := range spec.Binds {
394 switch b.Target {
395 case sandbox.GuestScratch:
396 scratch = strings.HasSuffix(b.Source, "/scratch") && !b.ReadOnly
397 case "/home/runner/work":
398 work = strings.HasSuffix(b.Source, "/scratch/work")
399 case "/tmp":
400 tmp = strings.HasSuffix(b.Source, "/scratch/tmp")
401 }
402 }
403 if !scratch || !work || !tmp {
404 t.Errorf("scratch, workspace and /tmp must come from the loop-mounted disk: %+v", spec.Binds)
405 }
406 if spec.DNS.String() != "10.200.0.1" || spec.IPv4.String() != "10.200.0.2" || netSpec.Bridge != "erbr0" || netSpec.Subnet.String() != "10.200.0.0/30" {
407 t.Errorf("network: %+v %+v", spec, netSpec)
408 }
409 if !slices.Contains(spec.Env, "ERMITE_CONNECT=unix:/run/ermite/exec.sock") {
410 t.Errorf("env: %v", spec.Env)
411 }
412 if e.plumb.Live() != 1 {
413 t.Fatal("firewall policy not attached")
414 }
415
416 if err := sb.Stop(context.Background()); err != nil {
417 t.Fatal(err)
418 }
419 if left := e.leftovers(); len(left) != 0 {
420 t.Fatalf("leftovers: %v", left)
421 }
422 if err := sb.Stop(context.Background()); err != nil {
423 t.Fatalf("Stop must be idempotent: %v", err)
424 }
425 if d.slots.InUse() != 0 {
426 t.Fatal("slot not released")
427 }
428}
429
430func TestSysboxJobIsNotPrivileged(t *testing.T) {
431 e := newCEnv(t)
432 e.eng.info.Runtimes = []string{"sysbox-runc", "runc"}
433 d := e.driver(t)
434 sb, _ := d.New()
435 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
436 if err != nil {
437 t.Fatal(err)
438 }
439 stub(t, conn)
440 e.eng.mu.Lock()
441 for _, s := range e.eng.containers {
442 if s.Privileged || s.Runtime != "sysbox-runc" {
443 t.Errorf("sysbox job: %+v", s)
444 }
445 }
446 e.eng.mu.Unlock()
447 _ = sb.Stop(context.Background())
448}
449
450func TestStartFailuresCleanEverything(t *testing.T) {
451 cases := map[string]func(*cenv){
452 "container cannot start": func(e *cenv) { e.eng.failStart = errors.New("oci runtime error") },
453 "container exits early": func(e *cenv) { e.eng.exitOnRun = true },
454 "executor never connects": func(e *cenv) {
455 e.eng.noConnect = true
456 },
457 "firewall attach fails": func(e *cenv) { e.plumb.FailNext = errors.New("nft broke") },
458 }
459 for name, setup := range cases {
460 t.Run(name, func(t *testing.T) {
461 e := newCEnv(t)
462 setup(e)
463 d, err := New(context.Background(), e.cfg, withTimeout(e.opts(), 300*time.Millisecond))
464 if err != nil {
465 t.Fatal(err)
466 }
467 defer func() { _ = d.Close() }()
468 sb, _ := d.New()
469 _, err = sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
470 if err == nil {
471 t.Fatal("Start succeeded")
472 }
473 if left := e.leftovers(); len(left) != 0 {
474 t.Fatalf("leftovers: %v (err %v)", left, err)
475 }
476 if d.slots.InUse() != 0 {
477 t.Fatal("slot leaked")
478 }
479 })
480 }
481}
482
483func TestFailedStartIncludesContainerLogs(t *testing.T) {
484 e := newCEnv(t)
485 e.eng.exitOnRun = true
486 d := e.driver(t)
487 sb, _ := d.New()
488 _, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
489 if err == nil || !strings.Contains(err.Error(), "fake container log line") {
490 t.Fatalf("got %v", err)
491 }
492}
493
494func TestCleanupFailureKeepsSlotAndIsRetriedByReap(t *testing.T) {
495 e := newCEnv(t)
496 d := e.driver(t)
497 sb, _ := d.New()
498 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
499 if err != nil {
500 t.Fatal(err)
501 }
502 stub(t, conn)
503 e.eng.removeErr = errors.New("daemon busy")
504 if err := sb.Stop(context.Background()); err == nil {
505 t.Fatal("Stop must report the failed removal")
506 }
507 if d.slots.InUse() != 1 {
508 t.Fatal("the slot must stay taken while its bridge and subnet still exist")
509 }
510 e.eng.removeErr = nil
511 if err := d.Reap(context.Background()); err != nil {
512 t.Fatal(err)
513 }
514 if left := e.leftovers(); len(left) != 0 {
515 t.Fatalf("Reap left %v", left)
516 }
517}
518
519func TestReapRemovesLabelledObjectsOnly(t *testing.T) {
520 e := newCEnv(t)
521 d := e.driver(t)
522 ctx := context.Background()
523 // Leftovers of a crashed run, and an unrelated user's network.
524 _, _ = e.eng.CreateContainer(ctx, containerSpec{Name: "old", Labels: map[string]string{sandbox.LabelJob: "x"}})
525 _, _ = e.eng.CreateNetwork(ctx, networkSpec{Name: "old", Labels: map[string]string{sandbox.LabelJob: "x"}})
526 _, _ = e.eng.CreateNetwork(ctx, networkSpec{Name: "mine", Labels: map[string]string{"app": "other"}})
527 _ = os.MkdirAll(filepath.Join(e.cfg.StateDir, "containers", "er00112233"), 0o700)
528 _ = os.MkdirAll(filepath.Join(e.cfg.StateDir, "containers", "keep-me"), 0o700)
529
530 if err := d.Reap(ctx); err != nil {
531 t.Fatal(err)
532 }
533 left := e.leftovers()
534 slices.Sort(left)
535 if len(left) != 2 || !strings.HasPrefix(left[0], "dir keep-me") || !strings.HasPrefix(left[1], "network") {
536 t.Fatalf("only the unrelated network and directory may remain: %v", left)
537 }
538 if _, err := os.Stat(filepath.Join(e.cfg.StateDir, "containers", "keep-me")); err != nil {
539 t.Fatal("Reap removed a directory that is not ours")
540 }
541 if e.plumb.Resets != 1 {
542 t.Fatal("nftables table not reset")
543 }
544}
545
546func TestCapacityAndSlotIsolation(t *testing.T) {
547 e := newCEnv(t)
548 d := e.driver(t)
549 a, _ := d.New()
550 b, _ := d.New()
551 if _, err := d.New(); !errors.Is(err, sandbox.ErrCapacity) {
552 t.Fatalf("got %v", err)
553 }
554 for _, sb := range []sandbox.Sandbox{a, b} {
555 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
556 if err != nil {
557 t.Fatal(err)
558 }
559 stub(t, conn)
560 }
561 ifaces := map[string]bool{}
562 for iface := range e.plumb.Attached {
563 ifaces[iface] = true
564 }
565 if !ifaces["erbr0"] || !ifaces["erbr1"] {
566 t.Fatalf("each job needs its own bridge: %v", ifaces)
567 }
568 _ = a.Stop(context.Background())
569 _ = b.Stop(context.Background())
570 if left := e.leftovers(); len(left) != 0 {
571 t.Fatalf("leftovers: %v", left)
572 }
573}
574
575func TestStopWithCancelledContext(t *testing.T) {
576 e := newCEnv(t)
577 d := e.driver(t)
578 sb, _ := d.New()
579 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
580 if err != nil {
581 t.Fatal(err)
582 }
583 stub(t, conn)
584 ctx, cancel := context.WithCancel(context.Background())
585 cancel()
586 if err := sb.Stop(ctx); err != nil {
587 t.Fatal(err)
588 }
589 if left := e.leftovers(); len(left) != 0 {
590 t.Fatalf("leftovers: %v", left)
591 }
592}
593
594func TestDoubleStartRefused(t *testing.T) {
595 e := newCEnv(t)
596 d := e.driver(t)
597 sb, _ := d.New()
598 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
599 if err != nil {
600 t.Fatal(err)
601 }
602 stub(t, conn)
603 if _, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"}); err == nil {
604 t.Fatal("second Start accepted")
605 }
606 _ = sb.Stop(context.Background())
607}
608
609func mustPrefix(s string) netip.Prefix { return netip.MustParsePrefix(s) }
610
611func logger(buf *bytes.Buffer) context.Context {
612 return zerowrap.WithCtx(context.Background(), zerowrap.New(zerowrap.Config{Level: "debug", Format: "json", Output: buf}))
613}
614
615// S4: the privileged fallback is allowed by default, warned about at the start
616// of EVERY job, reported as the effective isolation, and refused on demand.
617func TestPrivilegedWarnedAtEveryJobAndReportedAsIsolation(t *testing.T) {
618 e := newCEnv(t)
619 var buf bytes.Buffer
620 ctx := logger(&buf)
621 d, err := New(ctx, e.cfg, e.opts())
622 if err != nil {
623 t.Fatal(err)
624 }
625 defer func() { _ = d.Close() }()
626 if d.Isolation() != sandbox.IsolationContainerPrivileged || d.Mode() != sandbox.Container {
627 t.Fatalf("isolation %q", d.Isolation())
628 }
629 const msg = "privileged container: host-root equivalent, network policy best effort"
630 if n := strings.Count(buf.String(), msg); n != 1 {
631 t.Fatalf("one warning at driver start, got %d", n)
632 }
633 for i := range 3 {
634 sb, _ := d.New()
635 conn, err := sb.Start(ctx, sandbox.JobSpec{JobID: fmt.Sprint("job", i)})
636 if err != nil {
637 t.Fatal(err)
638 }
639 stub(t, conn)
640 _ = sb.Stop(ctx)
641 }
642 if n := strings.Count(buf.String(), msg); n != 4 {
643 t.Fatalf("the warning must be logged at every job start (1 + 3), got %d:\n%s", n, buf.String())
644 }
645 if !strings.Contains(buf.String(), `"job":"job2"`) {
646 t.Fatal("the warning must name the job")
647 }
648}
649
650func TestSysboxIsReportedAndSilent(t *testing.T) {
651 e := newCEnv(t)
652 e.eng.info.Runtimes = []string{"runc", "sysbox-runc"}
653 var buf bytes.Buffer
654 ctx := logger(&buf)
655 d, err := New(ctx, e.cfg, e.opts())
656 if err != nil {
657 t.Fatal(err)
658 }
659 defer func() { _ = d.Close() }()
660 if d.Isolation() != sandbox.IsolationContainerSysbox {
661 t.Fatalf("isolation %q", d.Isolation())
662 }
663 sb, _ := d.New()
664 conn, _ := sb.Start(ctx, sandbox.JobSpec{JobID: "j"})
665 stub(t, conn)
666 _ = sb.Stop(ctx)
667 if strings.Contains(buf.String(), "host-root equivalent") {
668 t.Fatal("no privileged warning with sysbox")
669 }
670}
671
672func TestDenyPrivileged(t *testing.T) {
673 e := newCEnv(t)
674 e.cfg.DenyPrivileged = true
675 _, err := New(context.Background(), e.cfg, e.opts())
676 if err == nil || !strings.Contains(err.Error(), "ERMITE_RUNNER_DENY_PRIVILEGED") || !strings.Contains(err.Error(), "sysbox") {
677 t.Fatalf("got %v", err)
678 }
679 // The host lock must be free again after a refused start.
680 if l, err := sandbox.AcquireHostLock(e.cfg.LockFile); err != nil {
681 t.Fatalf("a failed New must release the host lock: %v", err)
682 } else {
683 _ = l.Release()
684 }
685 // With sysbox available the setting does nothing.
686 e2 := newCEnv(t)
687 e2.cfg.DenyPrivileged = true
688 e2.eng.info.Runtimes = []string{"sysbox-runc"}
689 d, err := New(context.Background(), e2.cfg, e2.opts())
690 if err != nil {
691 t.Fatalf("deny-privileged with sysbox must work: %v", err)
692 }
693 _ = d.Close()
694}
695
696// S5: pids limit from the config and no swap.
697func TestHostConfigPidsAndSwap(t *testing.T) {
698 hc := hostConfigFor(containerSpec{Network: "n", MemoryMiB: 2048, CPUs: 3, Pids: 4096, DNS: netip.MustParseAddr("10.200.0.1")})
699 if hc.Memory != 2048<<20 || hc.MemorySwap != hc.Memory {
700 t.Errorf("memory %d swap %d: swap must equal memory (no swap)", hc.Memory, hc.MemorySwap)
701 }
702 if hc.PidsLimit == nil || *hc.PidsLimit != 4096 {
703 t.Errorf("pids limit %v", hc.PidsLimit)
704 }
705 if hc.NanoCPUs != 3_000_000_000 {
706 t.Errorf("cpus %d", hc.NanoCPUs)
707 }
708 if hc.Sysctls["net.ipv6.conf.all.disable_ipv6"] != "1" {
709 t.Error("IPv6 must be disabled in the container")
710 }
711 if hostConfigFor(containerSpec{Network: "n"}).PidsLimit != nil {
712 t.Error("no pids limit configured: leave it unset")
713 }
714}
715
716func TestJobPidsFlowsFromConfig(t *testing.T) {
717 e := newCEnv(t)
718 e.cfg.JobPids = 777
719 d := e.driver(t)
720 sb, _ := d.New()
721 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
722 if err != nil {
723 t.Fatal(err)
724 }
725 stub(t, conn)
726 defer func() { _ = sb.Stop(context.Background()) }()
727 e.eng.mu.Lock()
728 defer e.eng.mu.Unlock()
729 for _, s := range e.eng.containers {
730 if s.Pids != 777 {
731 t.Fatalf("spec pids %d", s.Pids)
732 }
733 }
734}
735
736// C4: the disk is enforced by a loop-mounted ext4 file of ERMITE_RUNNER_JOB_DISK,
737// mounted while the job runs and unmounted (before the files are removed) after.
738func TestScratchDiskIsLoopMountedAndSized(t *testing.T) {
739 e := newCEnv(t)
740 d := e.driver(t)
741 sb, _ := d.New()
742 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j", DiskMiB: 2048})
743 if err != nil {
744 t.Fatal(err)
745 }
746 stub(t, conn)
747 if e.mounts.count() != 1 {
748 t.Fatalf("mounts: %d", e.mounts.count())
749 }
750 var mkfs, mount string
751 for _, l := range e.run.Lines() {
752 if strings.HasPrefix(l, "mkfs.ext4") && strings.Contains(l, "scratch.ext4") {
753 mkfs = l
754 }
755 if strings.HasPrefix(l, "mount ") {
756 mount = l
757 }
758 }
759 if !strings.Contains(mkfs, "ermite-scratch") || !strings.Contains(mount, "loop") || !strings.Contains(mount, "scratch.ext4") {
760 t.Fatalf("mkfs=%q mount=%q", mkfs, mount)
761 }
762 // The scratch file has the configured size (sparse) and is not visible to the job.
763 matches, _ := filepath.Glob(filepath.Join(e.cfg.StateDir, "containers", sb.ID(), "scratch.ext4"))
764 if len(matches) != 1 {
765 t.Fatal("no scratch file")
766 }
767 if st, _ := os.Stat(matches[0]); st.Size() != 2048<<20 {
768 t.Fatalf("size %d", st.Size())
769 }
770 e.eng.mu.Lock()
771 for _, s := range e.eng.containers {
772 for _, b := range s.Binds {
773 if strings.HasSuffix(b.Source, "scratch.ext4") || b.Source == filepath.Join(e.cfg.StateDir, "containers", sb.ID()) {
774 t.Errorf("the job must not see the disk file or the whole run directory: %+v", b)
775 }
776 }
777 }
778 e.eng.mu.Unlock()
779 if err := sb.Stop(context.Background()); err != nil {
780 t.Fatal(err)
781 }
782 // Order: umount comes before the directory removal (nothing below a mount is deleted).
783 lines := e.run.Lines()
784 if lines[len(lines)-1] != "umount "+filepath.Join(e.cfg.StateDir, "containers", sb.ID(), "scratch") {
785 t.Fatalf("last command: %s", lines[len(lines)-1])
786 }
787 if left := e.leftovers(); len(left) != 0 {
788 t.Fatalf("leftovers: %v", left)
789 }
790}
791
792// C5: when the container cannot be removed it may still run: keep the firewall
793// policy, the bridge, the mount and the files; report ErrLeaked; keep the slot;
794// let Reap finish.
795func TestStopKeepsPolicyWhenTheContainerMayStillRun(t *testing.T) {
796 e := newCEnv(t)
797 d := e.driver(t)
798 sb, _ := d.New()
799 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
800 if err != nil {
801 t.Fatal(err)
802 }
803 stub(t, conn)
804 e.eng.removeErr = errors.New("daemon busy")
805 err = sb.Stop(context.Background())
806 if !errors.Is(err, sandbox.ErrLeaked) {
807 t.Fatalf("want ErrLeaked, got %v", err)
808 }
809 if e.plumb.Live() != 1 {
810 t.Fatal("the firewall policy must stay while the container may run")
811 }
812 e.eng.mu.Lock()
813 nets := len(e.eng.networks)
814 e.eng.mu.Unlock()
815 if nets != 1 || e.mounts.count() != 1 {
816 t.Fatalf("the bridge and the scratch mount must stay: nets=%d mounts=%d", nets, e.mounts.count())
817 }
818 if _, err := os.Stat(filepath.Join(e.cfg.StateDir, "containers", sb.ID(), "scratch.ext4")); err != nil {
819 t.Fatal("the scratch file must stay")
820 }
821 if err := sb.Stop(context.Background()); !errors.Is(err, sandbox.ErrLeaked) {
822 t.Fatalf("second Stop: %v", err)
823 }
824 // C8: the slot is taken, a new sandbox gets another slot, Reap is busy while it runs.
825 other, err := d.New()
826 if err != nil {
827 t.Fatal(err)
828 }
829 if err := d.Reap(context.Background()); !errors.Is(err, sandbox.ErrBusy) {
830 t.Fatalf("Reap while a sandbox exists: %v", err)
831 }
832 _ = other.Stop(context.Background())
833 e.eng.removeErr = nil
834 if err := d.Reap(context.Background()); err != nil {
835 t.Fatal(err)
836 }
837 if left := e.leftovers(); len(left) != 0 {
838 t.Fatalf("leftovers after Reap: %v", left)
839 }
840 if d.slots.InUse() != 0 {
841 t.Fatal("Reap must free the leaked slot")
842 }
843}
844
845// A scratch disk that cannot be unmounted keeps its files: nothing is deleted
846// below a live mount.
847func TestStopNeverDeletesBelowALiveMount(t *testing.T) {
848 e := newCEnv(t)
849 d := e.driver(t)
850 sb, _ := d.New()
851 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
852 if err != nil {
853 t.Fatal(err)
854 }
855 stub(t, conn)
856 e.mounts.failUm = errors.New("target is busy")
857 if err := sb.Stop(context.Background()); !errors.Is(err, sandbox.ErrLeaked) {
858 t.Fatalf("got %v", err)
859 }
860 if _, err := os.Stat(filepath.Join(e.cfg.StateDir, "containers", sb.ID(), "scratch.ext4")); err != nil {
861 t.Fatal("files below a mount that is still there were deleted")
862 }
863 e.mounts.failUm = nil
864 if err := d.Reap(context.Background()); err != nil {
865 t.Fatal(err)
866 }
867 if e.mounts.count() != 0 {
868 t.Fatal("Reap must unmount leftover scratch disks")
869 }
870 if left := e.leftovers(); len(left) != 0 {
871 t.Fatalf("leftovers: %v", left)
872 }
873}
874
875// C7: Stop aborts a boot in progress instead of waiting for the timeout.
876func TestStopAbortsABootInProgress(t *testing.T) {
877 e := newCEnv(t)
878 e.eng.noConnect = true
879 o := e.opts()
880 o.ConnectTimeout = time.Minute
881 d, err := New(context.Background(), e.cfg, o)
882 if err != nil {
883 t.Fatal(err)
884 }
885 defer func() { _ = d.Close() }()
886 sb, _ := d.New()
887 started := make(chan error, 1)
888 go func() { _, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"}); started <- err }()
889 time.Sleep(500 * time.Millisecond)
890 t0 := time.Now()
891 if err := sb.Stop(context.Background()); err != nil {
892 t.Fatal(err)
893 }
894 select {
895 case err := <-started:
896 if !errors.Is(err, sandbox.ErrAborted) {
897 t.Fatalf("Start must report the abort, got %v", err)
898 }
899 case <-time.After(10 * time.Second):
900 t.Fatal("Start was not aborted")
901 }
902 if time.Since(t0) > 5*time.Second {
903 t.Fatalf("Stop took %v", time.Since(t0))
904 }
905 if left := e.leftovers(); len(left) != 0 {
906 t.Fatalf("leftovers: %v", left)
907 }
908}
909
910// C16: ID names the Docker objects.
911func TestSandboxIDNamesDockerObjects(t *testing.T) {
912 e := newCEnv(t)
913 d := e.driver(t)
914 sb, _ := d.New()
915 if !sandbox.ValidID(sb.ID()) {
916 t.Fatal(sb.ID())
917 }
918 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
919 if err != nil {
920 t.Fatal(err)
921 }
922 stub(t, conn)
923 defer func() { _ = sb.Stop(context.Background()) }()
924 e.eng.mu.Lock()
925 defer e.eng.mu.Unlock()
926 for _, s := range e.eng.containers {
927 if s.Name != "ermite-"+sb.ID() {
928 t.Fatalf("container name %q", s.Name)
929 }
930 }
931}
932
933// S3: a second runner on the same host is refused whatever its state directory.
934func TestHostLockAcrossStateDirs(t *testing.T) {
935 e := newCEnv(t)
936 d := e.driver(t)
937 _ = d
938 e2 := newCEnv(t)
939 e2.cfg.LockFile = e.cfg.LockFile // same host, another state directory
940 _, err := New(context.Background(), e2.cfg, e2.opts())
941 if err == nil || !strings.Contains(err.Error(), "another ermite-runner is running on this host") {
942 t.Fatalf("got %v", err)
943 }
944}
945
946// C10: a cancelled wait is reported as cancelled, never as a clean exit.
947func TestAwaitExit(t *testing.T) {
948 res := make(chan container.WaitResponse, 1)
949 errc := make(chan error, 1)
950 res <- container.WaitResponse{StatusCode: 3}
951 if r := <-awaitExit(context.Background(), res, errc); r.Code != 3 || r.Err != nil {
952 t.Fatalf("%+v", r)
953 }
954 errc <- errors.New("daemon gone")
955 if r := <-awaitExit(context.Background(), make(chan container.WaitResponse), errc); r.Code != -1 || r.Err == nil {
956 t.Fatalf("%+v", r)
957 }
958 // Both ready and the context already cancelled: the cancellation wins.
959 ctx, cancel := context.WithCancel(context.Background())
960 cancel()
961 for range 50 {
962 res2 := make(chan container.WaitResponse, 1)
963 res2 <- container.WaitResponse{StatusCode: 0}
964 if r := <-awaitExit(ctx, res2, make(chan error)); r.Err == nil || !errors.Is(r.Err, context.Canceled) || r.Code != -1 {
965 t.Fatalf("a cancelled wait must not look like a clean exit: %+v", r)
966 }
967 }
968}
969
970func TestFreeSpaceWarningInContainerMode(t *testing.T) {
971 e := newCEnv(t)
972 var buf bytes.Buffer
973 o := e.opts()
974 o.FreeSpace = func(string) (int64, error) { return 1, nil }
975 d, err := New(logger(&buf), e.cfg, o)
976 if err != nil {
977 t.Fatal(err)
978 }
979 defer func() { _ = d.Close() }()
980 if !strings.Contains(buf.String(), "less free space") {
981 t.Fatalf("no warning: %s", buf.String())
982 }
983}
984
985func TestMkfsTooOldRefusedAtStart(t *testing.T) {
986 e := newCEnv(t)
987 run := &cmdrun.Fake{Handle: func(c cmdrun.Call) ([]byte, error) {
988 if c.Name == "mkfs.ext4" {
989 return []byte("mke2fs 1.42.9 (4-Feb-2014)\n"), nil
990 }
991 return nil, nil
992 }}
993 o := e.opts()
994 o.Run = run
995 if _, err := New(context.Background(), e.cfg, o); err == nil || !strings.Contains(err.Error(), "1.43") {
996 t.Fatalf("got %v", err)
997 }
998}
999
1000func TestSocketPathTooLongIsExplained(t *testing.T) {
1001 e := newCEnv(t)
1002 e.cfg.StateDir = filepath.Join(e.cfg.StateDir, strings.Repeat("d", 100))
1003 d := e.driver(t)
1004 sb, _ := d.New()
1005 _, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
1006 if err == nil || !strings.Contains(err.Error(), "108-byte limit") && !strings.Contains(err.Error(), "limit of a Unix socket") {
1007 t.Fatalf("got %v", err)
1008 }
1009 if left := e.leftovers(); len(left) != 0 {
1010 t.Fatalf("leftovers: %v", left)
1011 }
1012}
1013
1014// GO-002: only a podman daemon gets a DNS listener that may precede the gateway
1015// address; Docker keeps a normal bind.
1016func TestOnlyPodmanGetsAFreeBindDNSListener(t *testing.T) {
1017 if newNet(nil, 0, false, "").DNS.FreeBind {
1018 t.Fatal("Docker must keep a normal bind")
1019 }
1020 if !newNet(nil, 0, true, "").DNS.FreeBind {
1021 t.Fatal("podman needs the listener before the bridge address")
1022 }
1023}
1024
1025// Guest forge access: the exception reaches the policy of every guest, and the
1026// warning naming the origin is logged at the driver start and at every job.
1027func TestGuestForgeIsAttachedAndWarnedAtEveryJob(t *testing.T) {
1028 e := newCEnv(t)
1029 var err error
1030 if e.cfg.GuestForge, err = sandbox.ParseGuestForge("http://ermite.bnema.dev:8088", true); err != nil {
1031 t.Fatal(err)
1032 }
1033 var buf bytes.Buffer
1034 ctx := logger(&buf)
1035 d, err := New(ctx, e.cfg, e.opts())
1036 if err != nil {
1037 t.Fatal(err)
1038 }
1039 defer func() { _ = d.Close() }()
1040 msg := "guest forge access: sandboxes may reach http://ermite.bnema.dev:8088 on the host"
1041 if n := strings.Count(buf.String(), msg); n != 1 {
1042 t.Fatalf("one warning at driver start, got %d", n)
1043 }
1044 for i := range 2 {
1045 sb, _ := d.New()
1046 conn, err := sb.Start(ctx, sandbox.JobSpec{JobID: fmt.Sprint("job", i)})
1047 if err != nil {
1048 t.Fatal(err)
1049 }
1050 for _, g := range e.plumb.Attached {
1051 if g.ForgePort != 8088 {
1052 t.Errorf("guest attached with ForgePort %d", g.ForgePort)
1053 }
1054 }
1055 stub(t, conn)
1056 _ = sb.Stop(ctx)
1057 }
1058 if n := strings.Count(buf.String(), msg); n != 3 {
1059 t.Fatalf("the warning must be logged at every job start (1 + 2), got %d", n)
1060 }
1061}
1062
1063func TestNoGuestForgeByDefault(t *testing.T) {
1064 e := newCEnv(t)
1065 var buf bytes.Buffer
1066 ctx := logger(&buf)
1067 d := e.driver(t)
1068 sb, _ := d.New()
1069 conn, err := sb.Start(ctx, sandbox.JobSpec{JobID: "j"})
1070 if err != nil {
1071 t.Fatal(err)
1072 }
1073 for _, g := range e.plumb.Attached {
1074 if g.ForgePort != 0 {
1075 t.Errorf("ForgePort %d without the setting", g.ForgePort)
1076 }
1077 }
1078 stub(t, conn)
1079 _ = sb.Stop(ctx)
1080 if strings.Contains(buf.String(), "guest forge") {
1081 t.Fatal("a guest forge warning without the setting")
1082 }
1083 if n := newNet(nil, 0, false, "ermite.example.org").DNS.Override; n != "ermite.example.org" {
1084 t.Fatalf("override %q", n)
1085 }
1086}
1087
1088// The host's external tools are bind-mounted read-only at
1089// /home/runner/externals/<name> (GitHub's layout), whichever the isolation.
1090func TestExternalsAreBoundReadOnly(t *testing.T) {
1091 e := newCEnv(t)
1092 src := &sandboxtest.Externals{}
1093 n24 := sandboxtest.MakeExternal(t, "node24", "a", "v24")
1094 n20 := sandboxtest.MakeExternal(t, "node20", "b", "v20")
1095 gone := sandbox.External{Name: "node24_alpine", ID: "c", Dir: "/state/externals/.unavailable/x", Unavailable: true}
1096 src.Set(n24, n20, gone)
1097 e.cfg.Externals = src
1098 d := e.driver(t)
1099 run := func() containerSpec {
1100 t.Helper()
1101 sb, _ := d.New()
1102 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
1103 if err != nil {
1104 t.Fatal(err)
1105 }
1106 stub(t, conn)
1107 e.eng.mu.Lock()
1108 var spec containerSpec
1109 for _, s := range e.eng.containers {
1110 spec = s
1111 }
1112 e.eng.mu.Unlock()
1113 _ = sb.Stop(context.Background())
1114 return spec
1115 }
1116 spec := run()
1117 got := map[string]bindMount{}
1118 for _, b := range spec.Binds {
1119 got[b.Target] = b
1120 }
1121 for _, w := range []sandbox.External{n24, n20, gone} {
1122 b, ok := got["/home/runner/externals/"+w.Name]
1123 if !ok || !b.ReadOnly || b.Source != w.Dir {
1124 t.Errorf("%s: %+v (binds %+v)", w.Name, b, spec.Binds)
1125 }
1126 }
1127 // The executable and the other binds are untouched.
1128 if b := got["/opt/ermite/ermite-runner"]; !b.ReadOnly || b.Source != e.cfg.Executable {
1129 t.Errorf("executable bind: %+v", b)
1130 }
1131 if len(spec.Binds) != 5+3 {
1132 t.Errorf("binds: %+v", spec.Binds)
1133 }
1134
1135 // A new Node.js is picked up by the next job, without a driver restart.
1136 src.Set(sandboxtest.MakeExternal(t, "node24", "d", "v24.22"))
1137 spec = run()
1138 n := 0
1139 for _, b := range spec.Binds {
1140 if strings.HasPrefix(b.Target, "/home/runner/externals/") {
1141 n++
1142 }
1143 }
1144 if n != 1 {
1145 t.Errorf("externals binds after the change: %+v", spec.Binds)
1146 }
1147}
1148
1149func TestNoExternalsNoBinds(t *testing.T) {
1150 e := newCEnv(t)
1151 d := e.driver(t)
1152 sb, _ := d.New()
1153 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
1154 if err != nil {
1155 t.Fatal(err)
1156 }
1157 stub(t, conn)
1158 defer func() { _ = sb.Stop(context.Background()) }()
1159 e.eng.mu.Lock()
1160 defer e.eng.mu.Unlock()
1161 for _, s := range e.eng.containers {
1162 for _, b := range s.Binds {
1163 if strings.Contains(b.Target, "externals") {
1164 t.Errorf("unexpected bind %+v", b)
1165 }
1166 }
1167 }
1168}