1// SPDX-License-Identifier: MIT
2
3package container
4
5import (
6 "bytes"
7 "context"
8 "errors"
9 "fmt"
10 "net"
11 "net/netip"
12 "os"
13 "path/filepath"
14 "slices"
15 "strings"
16 "sync"
17 "testing"
18 "time"
19
20 "github.com/bnema/zerowrap"
21 "github.com/moby/moby/api/types/container"
22
23 "bnema.dev/ermite-runner/internal/cmdrun"
24 "bnema.dev/ermite-runner/internal/execstream"
25 "bnema.dev/ermite-runner/internal/execstream/execpb"
26 "bnema.dev/ermite-runner/internal/sandbox"
27 "bnema.dev/ermite-runner/internal/sandbox/disk"
28 "bnema.dev/ermite-runner/internal/sandbox/sandboxtest"
29)
30
31// fakeEngine plays the Docker daemon. When a container "starts", it plays the
32// executor inside it by connecting to the bind-mounted socket.
33type fakeEngine struct {
34 mu sync.Mutex
35 info daemonInfo
36 infoErr error
37 imageErr error
38 networks map[string]networkSpec
39 containers map[string]containerSpec
40 failStart error
41 noConnect bool
42 exitOnRun bool
43 removeErr error
44 waiters map[string]chan exitResult
45 seq int
46 pulled []string
47}
48
49func newFake() *fakeEngine {
50 return &fakeEngine{
51 info: daemonInfo{OSType: "linux", Runtimes: []string{"runc"}},
52 networks: map[string]networkSpec{},
53 containers: map[string]containerSpec{},
54 waiters: map[string]chan exitResult{},
55 }
56}
57
58func (f *fakeEngine) Info(context.Context) (daemonInfo, error) { return f.info, f.infoErr }
59func (f *fakeEngine) EnsureImage(_ context.Context, ref string) error {
60 f.pulled = append(f.pulled, ref)
61 return f.imageErr
62}
63func (f *fakeEngine) CreateNetwork(_ context.Context, s networkSpec) (string, error) {
64 f.mu.Lock()
65 defer f.mu.Unlock()
66 f.seq++
67 id := fmt.Sprintf("net%d", f.seq)
68 f.networks[id] = s
69 return id, nil
70}
71func (f *fakeEngine) RemoveNetwork(_ context.Context, id string) error {
72 f.mu.Lock()
73 defer f.mu.Unlock()
74 delete(f.networks, id)
75 return nil
76}
77func (f *fakeEngine) ListNetworks(context.Context, string) ([]string, error) {
78 f.mu.Lock()
79 defer f.mu.Unlock()
80 var out []string
81 for id, n := range f.networks {
82 if _, ok := n.Labels[sandbox.LabelJob]; ok {
83 out = append(out, id)
84 }
85 }
86 return out, nil
87}
88func (f *fakeEngine) CreateContainer(_ context.Context, s containerSpec) (string, error) {
89 f.mu.Lock()
90 defer f.mu.Unlock()
91 f.seq++
92 id := fmt.Sprintf("ctr%d", f.seq)
93 f.containers[id] = s
94 f.waiters[id] = make(chan exitResult, 1)
95 return id, nil
96}
97func (f *fakeEngine) StartContainer(_ context.Context, id string) error {
98 f.mu.Lock()
99 spec := f.containers[id]
100 fail, noConnect, exitOnRun := f.failStart, f.noConnect, f.exitOnRun
101 w := f.waiters[id]
102 f.mu.Unlock()
103 if fail != nil {
104 return fail
105 }
106 if exitOnRun {
107 w <- exitResult{Code: 125}
108 return nil
109 }
110 if noConnect {
111 return nil
112 }
113 // The host side of the bind mount is the source of the run-dir mount.
114 var sock string
115 for _, b := range spec.Binds {
116 if b.Target == sandbox.ContainerRunDir {
117 sock = filepath.Join(b.Source, sandbox.ContainerSocket)
118 }
119 }
120 go func() {
121 c, err := net.Dial("unix", sock)
122 if err != nil {
123 return
124 }
125 defer func() { _ = c.Close() }()
126 g := execstream.NewConn(c)
127 _ = g.Send(&execpb.Frame{Body: &execpb.Frame_Hello{Hello: &execpb.Hello{Sandbox: "container"}}})
128 if _, err := g.Recv(); err != nil {
129 return
130 }
131 _ = g.Send(&execpb.Frame{Body: &execpb.Frame_Output{Output: &execpb.Output{Data: []byte("hello from fake container\n")}}})
132 _ = g.Send(&execpb.Frame{Body: &execpb.Frame_Exit{Exit: &execpb.Exit{Code: 7}}})
133 _, _ = g.Recv()
134 }()
135 return nil
136}
137func (f *fakeEngine) WaitExit(_ context.Context, id string) <-chan exitResult {
138 f.mu.Lock()
139 defer f.mu.Unlock()
140 return f.waiters[id]
141}
142func (f *fakeEngine) RemoveContainer(_ context.Context, id string) error {
143 f.mu.Lock()
144 defer f.mu.Unlock()
145 if f.removeErr != nil {
146 return f.removeErr
147 }
148 delete(f.containers, id)
149 return nil
150}
151func (f *fakeEngine) ListContainers(context.Context, string) ([]string, error) {
152 f.mu.Lock()
153 defer f.mu.Unlock()
154 var out []string
155 for id := range f.containers {
156 out = append(out, id)
157 }
158 return out, nil
159}
160func (f *fakeEngine) LogsTail(context.Context, string, int) (string, error) {
161 return "fake container log line", nil
162}
163func (f *fakeEngine) Close() error { return nil }
164
165func (f *fakeEngine) objects() []string {
166 f.mu.Lock()
167 defer f.mu.Unlock()
168 var out []string
169 for id := range f.containers {
170 out = append(out, "container "+id)
171 }
172 for id := range f.networks {
173 out = append(out, "network "+id)
174 }
175 return out
176}
177
178// fakeMounts plays mount/umount: it records mounts and writes a mountinfo file
179// that disk.Mounter reads.
180type fakeMounts struct {
181 mu sync.Mutex
182 mounted map[string]bool
183 file string
184 failUm error
185}
186
187func (m *fakeMounts) sync() {
188 var b strings.Builder
189 for d := range m.mounted {
190 fmt.Fprintf(&b, "40 28 7:0 / %s rw - ext4 /dev/loop0 rw\n", d)
191 }
192 _ = os.WriteFile(m.file, []byte(b.String()), 0o644)
193}
194
195func (m *fakeMounts) handle(c cmdrun.Call) ([]byte, error) {
196 m.mu.Lock()
197 defer m.mu.Unlock()
198 switch c.Name {
199 case "mkfs.ext4":
200 if len(c.Args) == 1 && c.Args[0] == "-V" {
201 return []byte("mke2fs 1.47.4 (6-Mar-2025)\n"), nil
202 }
203 case "mount":
204 m.mounted[c.Args[len(c.Args)-1]] = true
205 m.sync()
206 case "umount":
207 if m.failUm != nil {
208 return nil, m.failUm
209 }
210 delete(m.mounted, c.Args[0])
211 m.sync()
212 }
213 return nil, nil
214}
215
216func (m *fakeMounts) count() int { m.mu.Lock(); defer m.mu.Unlock(); return len(m.mounted) }
217
218type cenv struct {
219 cfg sandbox.Config
220 eng *fakeEngine
221 plumb *sandboxtest.Plumbing
222 mounts *fakeMounts
223 run *cmdrun.Fake
224}
225
226func newCEnv(t *testing.T) *cenv {
227 t.Helper()
228 // A short directory: the executor socket path is limited to 107 bytes and
229 // t.TempDir embeds the (long) test name.
230 dir, err := os.MkdirTemp("", "erc")
231 if err != nil {
232 t.Fatal(err)
233 }
234 t.Cleanup(func() { _ = os.RemoveAll(dir) })
235 exe := filepath.Join(dir, "runner")
236 sandboxtest.WriteStaticELF(t, exe)
237 fm := &fakeMounts{mounted: map[string]bool{}, file: filepath.Join(dir, "mountinfo")}
238 fm.sync()
239 return &cenv{
240 eng: newFake(),
241 plumb: sandboxtest.NewPlumbing(),
242 mounts: fm,
243 run: &cmdrun.Fake{Handle: fm.handle},
244 cfg: sandbox.Config{
245 Isolation: sandbox.Container, StateDir: filepath.Join(dir, "state"), Capacity: 2,
246 JobCPUs: 2, JobMemoryMiB: 1024, JobDiskMiB: 1024, UIDBase: 900000,
247 Network: netip.MustParsePrefix("10.200.0.0/16"), Executable: exe, DNSPort: 0,
248 LockFile: filepath.Join(dir, "host.lock"), JobPids: 32768,
249 },
250 }
251}
252
253func (e *cenv) opts() Options {
254 return Options{engine: e.eng, Net: e.plumb, ConnectTimeout: 3 * time.Second, Run: e.run,
255 Mounter: &disk.Mounter{Run: e.run, Mountinfo: e.mounts.file}}
256}
257
258func (e *cenv) driver(t *testing.T) *Driver {
259 t.Helper()
260 d, err := New(context.Background(), e.cfg, e.opts())
261 if err != nil {
262 t.Fatal(err)
263 }
264 t.Cleanup(func() { _ = d.Close() })
265 return d
266}
267
268func withTimeout(o Options, d time.Duration) Options { o.ConnectTimeout = d; return o }
269
270func (e *cenv) leftovers() []string {
271 out := e.eng.objects()
272 if n := e.mounts.count(); n != 0 {
273 out = append(out, fmt.Sprintf("%d scratch mounts", n))
274 }
275 if n := e.plumb.Live(); n != 0 {
276 out = append(out, fmt.Sprintf("%d attached guests", n))
277 }
278 ents, _ := os.ReadDir(filepath.Join(e.cfg.StateDir, "containers"))
279 for _, d := range ents {
280 out = append(out, "dir "+d.Name())
281 }
282 return out
283}
284
285func stub(t *testing.T, conn sandbox.Conn) (string, int) {
286 t.Helper()
287 var out strings.Builder
288 _, exit, err := execstream.Drive(conn, &execpb.Run{}, nil, func(o *execpb.Output) { out.Write(o.Data) })
289 if err != nil {
290 t.Fatal(err)
291 }
292 return out.String(), int(exit.Code)
293}
294
295func TestRootlessDockerIsRefused(t *testing.T) {
296 e := newCEnv(t)
297 e.eng.info.Rootless = true
298 _, err := New(context.Background(), e.cfg, e.opts())
299 if err == nil || !strings.Contains(err.Error(), "rootless") || !strings.Contains(err.Error(), "rootful") {
300 t.Fatalf("want a clear rootless error, got %v", err)
301 }
302}
303
304func TestRuntimeSelection(t *testing.T) {
305 e := newCEnv(t)
306 d := e.driver(t)
307 if !d.Privileged() || d.runtime != "" {
308 t.Fatal("without sysbox the driver must run privileged")
309 }
310 e2 := newCEnv(t)
311 e2.eng.info.Runtimes = []string{"runc", "sysbox-runc"}
312 d2 := e2.driver(t)
313 if d2.Privileged() || d2.runtime != SysboxRuntime {
314 t.Fatal("sysbox listed: must use it, unprivileged")
315 }
316}
317
318func TestNewErrors(t *testing.T) {
319 e := newCEnv(t)
320 e.eng.infoErr = errors.New("connection refused")
321 if _, err := New(context.Background(), e.cfg, e.opts()); err == nil || !strings.Contains(err.Error(), "DOCKER_HOST") {
322 t.Fatalf("got %v", err)
323 }
324 e = newCEnv(t)
325 e.eng.imageErr = errors.New("no such image")
326 if _, err := New(context.Background(), e.cfg, e.opts()); err == nil {
327 t.Fatal("missing image accepted")
328 }
329 e = newCEnv(t)
330 e.eng.info.OSType = "windows"
331 if _, err := New(context.Background(), e.cfg, e.opts()); err == nil {
332 t.Fatal("windows daemon accepted")
333 }
334 e = newCEnv(t)
335 e.cfg.Executable = "/bin/sh"
336 if _, err := New(context.Background(), e.cfg, e.opts()); err == nil {
337 t.Skip("/bin/sh is static")
338 }
339}
340
341func TestStartRunStop(t *testing.T) {
342 e := newCEnv(t)
343 d := e.driver(t)
344 sb, err := d.New()
345 if err != nil {
346 t.Fatal(err)
347 }
348 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "job-7", CPUs: 3, MemoryMiB: 2048})
349 if err != nil {
350 t.Fatal(err)
351 }
352 timed := sb.(sandbox.Timed)
353 timings := timed.StartTimings()
354 for _, key := range []string{"socket_ms", "cache_ms", "scratch_ms", "net_ms", "container_ms", "connect_ms"} {
355 if value, ok := timings[key]; !ok || value < 0 {
356 t.Errorf("StartTimings[%q] = %d, present=%v", key, value, ok)
357 }
358 }
359 timings["socket_ms"] = -1
360 if timed.StartTimings()["socket_ms"] < 0 {
361 t.Error("StartTimings returned shared storage")
362 }
363 out, code := stub(t, conn)
364 if out != "hello from fake container\n" || code != 7 {
365 t.Fatalf("out=%q code=%d", out, code)
366 }
367
368 // What was asked of Docker.
369 e.eng.mu.Lock()
370 var spec containerSpec
371 for _, s := range e.eng.containers {
372 spec = s
373 }
374 var netSpec networkSpec
375 for _, n := range e.eng.networks {
376 netSpec = n
377 }
378 e.eng.mu.Unlock()
379 if !spec.Privileged || spec.Runtime != "" {
380 t.Errorf("privileged fallback expected: %+v", spec)
381 }
382 if spec.Labels[sandbox.LabelJob] != "job-7" || netSpec.Labels[sandbox.LabelJob] != "job-7" {
383 t.Errorf("every object must carry the job label: %v / %v", spec.Labels, netSpec.Labels)
384 }
385 if spec.MemoryMiB != 2048 || spec.CPUs != 3 {
386 t.Errorf("resources: %+v", spec)
387 }
388 if !slices.Equal(spec.Entrypoint, []string{"/opt/ermite/ermite-runner", "init"}) {
389 t.Errorf("entrypoint: %v", spec.Entrypoint)
390 }
391 var roExe, runDir bool
392 for _, b := range spec.Binds {
393 if b.Target == "/opt/ermite/ermite-runner" && b.ReadOnly && b.Source == e.cfg.Executable {
394 roExe = true
395 }
396 if b.Target == sandbox.ContainerRunDir && !b.ReadOnly {
397 runDir = true
398 }
399 }
400 if !roExe || !runDir {
401 t.Errorf("binds: %+v", spec.Binds)
402 }
403 var scratch, work, tmp bool
404 for _, b := range spec.Binds {
405 switch b.Target {
406 case sandbox.GuestScratch:
407 scratch = strings.HasSuffix(b.Source, "/scratch") && !b.ReadOnly
408 case "/home/runner/work":
409 work = strings.HasSuffix(b.Source, "/scratch/work")
410 case "/tmp":
411 tmp = strings.HasSuffix(b.Source, "/scratch/tmp")
412 }
413 }
414 if !scratch || !work || !tmp {
415 t.Errorf("scratch, workspace and /tmp must come from the loop-mounted disk: %+v", spec.Binds)
416 }
417 if spec.DNS.String() != "10.200.0.1" || spec.IPv4.String() != "10.200.0.2" || netSpec.Bridge != "erbr0" || netSpec.Subnet.String() != "10.200.0.0/30" {
418 t.Errorf("network: %+v %+v", spec, netSpec)
419 }
420 if !slices.Contains(spec.Env, "ERMITE_CONNECT=unix:/run/ermite/exec.sock") {
421 t.Errorf("env: %v", spec.Env)
422 }
423 if e.plumb.Live() != 1 {
424 t.Fatal("firewall policy not attached")
425 }
426
427 if err := sb.Stop(context.Background()); err != nil {
428 t.Fatal(err)
429 }
430 if left := e.leftovers(); len(left) != 0 {
431 t.Fatalf("leftovers: %v", left)
432 }
433 if err := sb.Stop(context.Background()); err != nil {
434 t.Fatalf("Stop must be idempotent: %v", err)
435 }
436 if d.slots.InUse() != 0 {
437 t.Fatal("slot not released")
438 }
439}
440
441func TestSysboxJobIsNotPrivileged(t *testing.T) {
442 e := newCEnv(t)
443 e.eng.info.Runtimes = []string{"sysbox-runc", "runc"}
444 d := e.driver(t)
445 sb, _ := d.New()
446 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
447 if err != nil {
448 t.Fatal(err)
449 }
450 stub(t, conn)
451 e.eng.mu.Lock()
452 for _, s := range e.eng.containers {
453 if s.Privileged || s.Runtime != "sysbox-runc" {
454 t.Errorf("sysbox job: %+v", s)
455 }
456 }
457 e.eng.mu.Unlock()
458 _ = sb.Stop(context.Background())
459}
460
461func TestStartFailuresCleanEverything(t *testing.T) {
462 cases := map[string]func(*cenv){
463 "container cannot start": func(e *cenv) { e.eng.failStart = errors.New("oci runtime error") },
464 "container exits early": func(e *cenv) { e.eng.exitOnRun = true },
465 "executor never connects": func(e *cenv) {
466 e.eng.noConnect = true
467 },
468 "firewall attach fails": func(e *cenv) { e.plumb.FailNext = errors.New("nft broke") },
469 }
470 for name, setup := range cases {
471 t.Run(name, func(t *testing.T) {
472 e := newCEnv(t)
473 setup(e)
474 d, err := New(context.Background(), e.cfg, withTimeout(e.opts(), 300*time.Millisecond))
475 if err != nil {
476 t.Fatal(err)
477 }
478 defer func() { _ = d.Close() }()
479 sb, _ := d.New()
480 _, err = sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
481 if err == nil {
482 t.Fatal("Start succeeded")
483 }
484 if left := e.leftovers(); len(left) != 0 {
485 t.Fatalf("leftovers: %v (err %v)", left, err)
486 }
487 if d.slots.InUse() != 0 {
488 t.Fatal("slot leaked")
489 }
490 })
491 }
492}
493
494func TestFailedStartIncludesContainerLogs(t *testing.T) {
495 e := newCEnv(t)
496 e.eng.exitOnRun = true
497 d := e.driver(t)
498 sb, _ := d.New()
499 _, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
500 if err == nil || !strings.Contains(err.Error(), "fake container log line") {
501 t.Fatalf("got %v", err)
502 }
503}
504
505func TestCleanupFailureKeepsSlotAndIsRetriedByReap(t *testing.T) {
506 e := newCEnv(t)
507 d := e.driver(t)
508 sb, _ := d.New()
509 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
510 if err != nil {
511 t.Fatal(err)
512 }
513 stub(t, conn)
514 e.eng.removeErr = errors.New("daemon busy")
515 if err := sb.Stop(context.Background()); err == nil {
516 t.Fatal("Stop must report the failed removal")
517 }
518 if d.slots.InUse() != 1 {
519 t.Fatal("the slot must stay taken while its bridge and subnet still exist")
520 }
521 e.eng.removeErr = nil
522 if err := d.Reap(context.Background()); err != nil {
523 t.Fatal(err)
524 }
525 if left := e.leftovers(); len(left) != 0 {
526 t.Fatalf("Reap left %v", left)
527 }
528}
529
530func TestReapRemovesLabelledObjectsOnly(t *testing.T) {
531 e := newCEnv(t)
532 d := e.driver(t)
533 ctx := context.Background()
534 // Leftovers of a crashed run, and an unrelated user's network.
535 _, _ = e.eng.CreateContainer(ctx, containerSpec{Name: "old", Labels: map[string]string{sandbox.LabelJob: "x"}})
536 _, _ = e.eng.CreateNetwork(ctx, networkSpec{Name: "old", Labels: map[string]string{sandbox.LabelJob: "x"}})
537 _, _ = e.eng.CreateNetwork(ctx, networkSpec{Name: "mine", Labels: map[string]string{"app": "other"}})
538 _ = os.MkdirAll(filepath.Join(e.cfg.StateDir, "containers", "er00112233"), 0o700)
539 _ = os.MkdirAll(filepath.Join(e.cfg.StateDir, "containers", "keep-me"), 0o700)
540
541 if err := d.Reap(ctx); err != nil {
542 t.Fatal(err)
543 }
544 left := e.leftovers()
545 slices.Sort(left)
546 if len(left) != 2 || !strings.HasPrefix(left[0], "dir keep-me") || !strings.HasPrefix(left[1], "network") {
547 t.Fatalf("only the unrelated network and directory may remain: %v", left)
548 }
549 if _, err := os.Stat(filepath.Join(e.cfg.StateDir, "containers", "keep-me")); err != nil {
550 t.Fatal("Reap removed a directory that is not ours")
551 }
552 if e.plumb.Resets != 1 {
553 t.Fatal("nftables table not reset")
554 }
555}
556
557func TestCapacityAndSlotIsolation(t *testing.T) {
558 e := newCEnv(t)
559 d := e.driver(t)
560 a, _ := d.New()
561 b, _ := d.New()
562 if _, err := d.New(); !errors.Is(err, sandbox.ErrCapacity) {
563 t.Fatalf("got %v", err)
564 }
565 for _, sb := range []sandbox.Sandbox{a, b} {
566 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
567 if err != nil {
568 t.Fatal(err)
569 }
570 stub(t, conn)
571 }
572 ifaces := map[string]bool{}
573 for iface := range e.plumb.Attached {
574 ifaces[iface] = true
575 }
576 if !ifaces["erbr0"] || !ifaces["erbr1"] {
577 t.Fatalf("each job needs its own bridge: %v", ifaces)
578 }
579 _ = a.Stop(context.Background())
580 _ = b.Stop(context.Background())
581 if left := e.leftovers(); len(left) != 0 {
582 t.Fatalf("leftovers: %v", left)
583 }
584}
585
586func TestStopWithCancelledContext(t *testing.T) {
587 e := newCEnv(t)
588 d := e.driver(t)
589 sb, _ := d.New()
590 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
591 if err != nil {
592 t.Fatal(err)
593 }
594 stub(t, conn)
595 ctx, cancel := context.WithCancel(context.Background())
596 cancel()
597 if err := sb.Stop(ctx); err != nil {
598 t.Fatal(err)
599 }
600 if left := e.leftovers(); len(left) != 0 {
601 t.Fatalf("leftovers: %v", left)
602 }
603}
604
605func TestDoubleStartRefused(t *testing.T) {
606 e := newCEnv(t)
607 d := e.driver(t)
608 sb, _ := d.New()
609 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
610 if err != nil {
611 t.Fatal(err)
612 }
613 stub(t, conn)
614 if _, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"}); err == nil {
615 t.Fatal("second Start accepted")
616 }
617 _ = sb.Stop(context.Background())
618}
619
620func mustPrefix(s string) netip.Prefix { return netip.MustParsePrefix(s) }
621
622func logger(buf *bytes.Buffer) context.Context {
623 return zerowrap.WithCtx(context.Background(), zerowrap.New(zerowrap.Config{Level: "debug", Format: "json", Output: buf}))
624}
625
626// S4: the privileged fallback is allowed by default, warned about at the start
627// of EVERY job, reported as the effective isolation, and refused on demand.
628func TestPrivilegedWarnedAtEveryJobAndReportedAsIsolation(t *testing.T) {
629 e := newCEnv(t)
630 var buf bytes.Buffer
631 ctx := logger(&buf)
632 d, err := New(ctx, e.cfg, e.opts())
633 if err != nil {
634 t.Fatal(err)
635 }
636 defer func() { _ = d.Close() }()
637 if d.Isolation() != sandbox.IsolationContainerPrivileged || d.Mode() != sandbox.Container {
638 t.Fatalf("isolation %q", d.Isolation())
639 }
640 const msg = "privileged container: host-root equivalent, network policy best effort"
641 if n := strings.Count(buf.String(), msg); n != 1 {
642 t.Fatalf("one warning at driver start, got %d", n)
643 }
644 for i := range 3 {
645 sb, _ := d.New()
646 conn, err := sb.Start(ctx, sandbox.JobSpec{JobID: fmt.Sprint("job", i)})
647 if err != nil {
648 t.Fatal(err)
649 }
650 stub(t, conn)
651 _ = sb.Stop(ctx)
652 }
653 if n := strings.Count(buf.String(), msg); n != 4 {
654 t.Fatalf("the warning must be logged at every job start (1 + 3), got %d:\n%s", n, buf.String())
655 }
656 if !strings.Contains(buf.String(), `"job":"job2"`) {
657 t.Fatal("the warning must name the job")
658 }
659}
660
661func TestSysboxIsReportedAndSilent(t *testing.T) {
662 e := newCEnv(t)
663 e.eng.info.Runtimes = []string{"runc", "sysbox-runc"}
664 var buf bytes.Buffer
665 ctx := logger(&buf)
666 d, err := New(ctx, e.cfg, e.opts())
667 if err != nil {
668 t.Fatal(err)
669 }
670 defer func() { _ = d.Close() }()
671 if d.Isolation() != sandbox.IsolationContainerSysbox {
672 t.Fatalf("isolation %q", d.Isolation())
673 }
674 sb, _ := d.New()
675 conn, _ := sb.Start(ctx, sandbox.JobSpec{JobID: "j"})
676 stub(t, conn)
677 _ = sb.Stop(ctx)
678 if strings.Contains(buf.String(), "host-root equivalent") {
679 t.Fatal("no privileged warning with sysbox")
680 }
681}
682
683func TestDenyPrivileged(t *testing.T) {
684 e := newCEnv(t)
685 e.cfg.DenyPrivileged = true
686 _, err := New(context.Background(), e.cfg, e.opts())
687 if err == nil || !strings.Contains(err.Error(), "ERMITE_RUNNER_DENY_PRIVILEGED") || !strings.Contains(err.Error(), "sysbox") {
688 t.Fatalf("got %v", err)
689 }
690 // The host lock must be free again after a refused start.
691 if l, err := sandbox.AcquireHostLock(e.cfg.LockFile); err != nil {
692 t.Fatalf("a failed New must release the host lock: %v", err)
693 } else {
694 _ = l.Release()
695 }
696 // With sysbox available the setting does nothing.
697 e2 := newCEnv(t)
698 e2.cfg.DenyPrivileged = true
699 e2.eng.info.Runtimes = []string{"sysbox-runc"}
700 d, err := New(context.Background(), e2.cfg, e2.opts())
701 if err != nil {
702 t.Fatalf("deny-privileged with sysbox must work: %v", err)
703 }
704 _ = d.Close()
705}
706
707// S5: pids limit from the config and no swap.
708func TestHostConfigPidsAndSwap(t *testing.T) {
709 hc := hostConfigFor(containerSpec{Network: "n", MemoryMiB: 2048, CPUs: 3, Pids: 4096, DNS: netip.MustParseAddr("10.200.0.1")})
710 if hc.Memory != 2048<<20 || hc.MemorySwap != hc.Memory {
711 t.Errorf("memory %d swap %d: swap must equal memory (no swap)", hc.Memory, hc.MemorySwap)
712 }
713 if hc.PidsLimit == nil || *hc.PidsLimit != 4096 {
714 t.Errorf("pids limit %v", hc.PidsLimit)
715 }
716 if hc.NanoCPUs != 3_000_000_000 {
717 t.Errorf("cpus %d", hc.NanoCPUs)
718 }
719 if hc.Sysctls["net.ipv6.conf.all.disable_ipv6"] != "1" {
720 t.Error("IPv6 must be disabled in the container")
721 }
722 if hostConfigFor(containerSpec{Network: "n"}).PidsLimit != nil {
723 t.Error("no pids limit configured: leave it unset")
724 }
725}
726
727func TestJobPidsFlowsFromConfig(t *testing.T) {
728 e := newCEnv(t)
729 e.cfg.JobPids = 777
730 d := e.driver(t)
731 sb, _ := d.New()
732 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
733 if err != nil {
734 t.Fatal(err)
735 }
736 stub(t, conn)
737 defer func() { _ = sb.Stop(context.Background()) }()
738 e.eng.mu.Lock()
739 defer e.eng.mu.Unlock()
740 for _, s := range e.eng.containers {
741 if s.Pids != 777 {
742 t.Fatalf("spec pids %d", s.Pids)
743 }
744 }
745}
746
747// C4: the disk is enforced by a loop-mounted ext4 file of ERMITE_RUNNER_JOB_DISK,
748// mounted while the job runs and unmounted (before the files are removed) after.
749func TestScratchDiskIsLoopMountedAndSized(t *testing.T) {
750 e := newCEnv(t)
751 d := e.driver(t)
752 sb, _ := d.New()
753 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j", DiskMiB: 2048})
754 if err != nil {
755 t.Fatal(err)
756 }
757 stub(t, conn)
758 if e.mounts.count() != 1 {
759 t.Fatalf("mounts: %d", e.mounts.count())
760 }
761 var mkfs, mount string
762 for _, l := range e.run.Lines() {
763 if strings.HasPrefix(l, "mkfs.ext4") && strings.Contains(l, "scratch.ext4") {
764 mkfs = l
765 }
766 if strings.HasPrefix(l, "mount ") {
767 mount = l
768 }
769 }
770 if !strings.Contains(mkfs, "ermite-scratch") || !strings.Contains(mount, "loop") || !strings.Contains(mount, "scratch.ext4") {
771 t.Fatalf("mkfs=%q mount=%q", mkfs, mount)
772 }
773 // The scratch file has the configured size (sparse) and is not visible to the job.
774 matches, _ := filepath.Glob(filepath.Join(e.cfg.StateDir, "containers", sb.ID(), "scratch.ext4"))
775 if len(matches) != 1 {
776 t.Fatal("no scratch file")
777 }
778 if st, _ := os.Stat(matches[0]); st.Size() != 2048<<20 {
779 t.Fatalf("size %d", st.Size())
780 }
781 e.eng.mu.Lock()
782 for _, s := range e.eng.containers {
783 for _, b := range s.Binds {
784 if strings.HasSuffix(b.Source, "scratch.ext4") || b.Source == filepath.Join(e.cfg.StateDir, "containers", sb.ID()) {
785 t.Errorf("the job must not see the disk file or the whole run directory: %+v", b)
786 }
787 }
788 }
789 e.eng.mu.Unlock()
790 if err := sb.Stop(context.Background()); err != nil {
791 t.Fatal(err)
792 }
793 // Order: umount comes before the directory removal (nothing below a mount is deleted).
794 lines := e.run.Lines()
795 if lines[len(lines)-1] != "umount "+filepath.Join(e.cfg.StateDir, "containers", sb.ID(), "scratch") {
796 t.Fatalf("last command: %s", lines[len(lines)-1])
797 }
798 if left := e.leftovers(); len(left) != 0 {
799 t.Fatalf("leftovers: %v", left)
800 }
801}
802
803// C5: when the container cannot be removed it may still run: keep the firewall
804// policy, the bridge, the mount and the files; report ErrLeaked; keep the slot;
805// let Reap finish.
806func TestStopKeepsPolicyWhenTheContainerMayStillRun(t *testing.T) {
807 e := newCEnv(t)
808 d := e.driver(t)
809 sb, _ := d.New()
810 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
811 if err != nil {
812 t.Fatal(err)
813 }
814 stub(t, conn)
815 e.eng.removeErr = errors.New("daemon busy")
816 err = sb.Stop(context.Background())
817 if !errors.Is(err, sandbox.ErrLeaked) {
818 t.Fatalf("want ErrLeaked, got %v", err)
819 }
820 if e.plumb.Live() != 1 {
821 t.Fatal("the firewall policy must stay while the container may run")
822 }
823 e.eng.mu.Lock()
824 nets := len(e.eng.networks)
825 e.eng.mu.Unlock()
826 if nets != 1 || e.mounts.count() != 1 {
827 t.Fatalf("the bridge and the scratch mount must stay: nets=%d mounts=%d", nets, e.mounts.count())
828 }
829 if _, err := os.Stat(filepath.Join(e.cfg.StateDir, "containers", sb.ID(), "scratch.ext4")); err != nil {
830 t.Fatal("the scratch file must stay")
831 }
832 if err := sb.Stop(context.Background()); !errors.Is(err, sandbox.ErrLeaked) {
833 t.Fatalf("second Stop: %v", err)
834 }
835 // C8: the slot is taken, a new sandbox gets another slot, Reap is busy while it runs.
836 other, err := d.New()
837 if err != nil {
838 t.Fatal(err)
839 }
840 if err := d.Reap(context.Background()); !errors.Is(err, sandbox.ErrBusy) {
841 t.Fatalf("Reap while a sandbox exists: %v", err)
842 }
843 _ = other.Stop(context.Background())
844 e.eng.removeErr = nil
845 if err := d.Reap(context.Background()); err != nil {
846 t.Fatal(err)
847 }
848 if left := e.leftovers(); len(left) != 0 {
849 t.Fatalf("leftovers after Reap: %v", left)
850 }
851 if d.slots.InUse() != 0 {
852 t.Fatal("Reap must free the leaked slot")
853 }
854}
855
856// A scratch disk that cannot be unmounted keeps its files: nothing is deleted
857// below a live mount.
858func TestStopNeverDeletesBelowALiveMount(t *testing.T) {
859 e := newCEnv(t)
860 d := e.driver(t)
861 sb, _ := d.New()
862 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
863 if err != nil {
864 t.Fatal(err)
865 }
866 stub(t, conn)
867 e.mounts.failUm = errors.New("target is busy")
868 if err := sb.Stop(context.Background()); !errors.Is(err, sandbox.ErrLeaked) {
869 t.Fatalf("got %v", err)
870 }
871 if _, err := os.Stat(filepath.Join(e.cfg.StateDir, "containers", sb.ID(), "scratch.ext4")); err != nil {
872 t.Fatal("files below a mount that is still there were deleted")
873 }
874 e.mounts.failUm = nil
875 if err := d.Reap(context.Background()); err != nil {
876 t.Fatal(err)
877 }
878 if e.mounts.count() != 0 {
879 t.Fatal("Reap must unmount leftover scratch disks")
880 }
881 if left := e.leftovers(); len(left) != 0 {
882 t.Fatalf("leftovers: %v", left)
883 }
884}
885
886// C7: Stop aborts a boot in progress instead of waiting for the timeout.
887func TestStopAbortsABootInProgress(t *testing.T) {
888 e := newCEnv(t)
889 e.eng.noConnect = true
890 o := e.opts()
891 o.ConnectTimeout = time.Minute
892 d, err := New(context.Background(), e.cfg, o)
893 if err != nil {
894 t.Fatal(err)
895 }
896 defer func() { _ = d.Close() }()
897 sb, _ := d.New()
898 started := make(chan error, 1)
899 go func() { _, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"}); started <- err }()
900 time.Sleep(500 * time.Millisecond)
901 t0 := time.Now()
902 if err := sb.Stop(context.Background()); err != nil {
903 t.Fatal(err)
904 }
905 select {
906 case err := <-started:
907 if !errors.Is(err, sandbox.ErrAborted) {
908 t.Fatalf("Start must report the abort, got %v", err)
909 }
910 case <-time.After(10 * time.Second):
911 t.Fatal("Start was not aborted")
912 }
913 if time.Since(t0) > 5*time.Second {
914 t.Fatalf("Stop took %v", time.Since(t0))
915 }
916 if left := e.leftovers(); len(left) != 0 {
917 t.Fatalf("leftovers: %v", left)
918 }
919}
920
921// C16: ID names the Docker objects.
922func TestSandboxIDNamesDockerObjects(t *testing.T) {
923 e := newCEnv(t)
924 d := e.driver(t)
925 sb, _ := d.New()
926 if !sandbox.ValidID(sb.ID()) {
927 t.Fatal(sb.ID())
928 }
929 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
930 if err != nil {
931 t.Fatal(err)
932 }
933 stub(t, conn)
934 defer func() { _ = sb.Stop(context.Background()) }()
935 e.eng.mu.Lock()
936 defer e.eng.mu.Unlock()
937 for _, s := range e.eng.containers {
938 if s.Name != "ermite-"+sb.ID() {
939 t.Fatalf("container name %q", s.Name)
940 }
941 }
942}
943
944// S3: a second runner on the same host is refused whatever its state directory.
945func TestHostLockAcrossStateDirs(t *testing.T) {
946 e := newCEnv(t)
947 d := e.driver(t)
948 _ = d
949 e2 := newCEnv(t)
950 e2.cfg.LockFile = e.cfg.LockFile // same host, another state directory
951 _, err := New(context.Background(), e2.cfg, e2.opts())
952 if err == nil || !strings.Contains(err.Error(), "another ermite-runner is running on this host") {
953 t.Fatalf("got %v", err)
954 }
955}
956
957// C10: a cancelled wait is reported as cancelled, never as a clean exit.
958func TestAwaitExit(t *testing.T) {
959 res := make(chan container.WaitResponse, 1)
960 errc := make(chan error, 1)
961 res <- container.WaitResponse{StatusCode: 3}
962 if r := <-awaitExit(context.Background(), res, errc); r.Code != 3 || r.Err != nil {
963 t.Fatalf("%+v", r)
964 }
965 errc <- errors.New("daemon gone")
966 if r := <-awaitExit(context.Background(), make(chan container.WaitResponse), errc); r.Code != -1 || r.Err == nil {
967 t.Fatalf("%+v", r)
968 }
969 // Both ready and the context already cancelled: the cancellation wins.
970 ctx, cancel := context.WithCancel(context.Background())
971 cancel()
972 for range 50 {
973 res2 := make(chan container.WaitResponse, 1)
974 res2 <- container.WaitResponse{StatusCode: 0}
975 if r := <-awaitExit(ctx, res2, make(chan error)); r.Err == nil || !errors.Is(r.Err, context.Canceled) || r.Code != -1 {
976 t.Fatalf("a cancelled wait must not look like a clean exit: %+v", r)
977 }
978 }
979}
980
981func TestFreeSpaceWarningInContainerMode(t *testing.T) {
982 e := newCEnv(t)
983 var buf bytes.Buffer
984 o := e.opts()
985 o.FreeSpace = func(string) (int64, error) { return 1, nil }
986 d, err := New(logger(&buf), e.cfg, o)
987 if err != nil {
988 t.Fatal(err)
989 }
990 defer func() { _ = d.Close() }()
991 if !strings.Contains(buf.String(), "less free space") {
992 t.Fatalf("no warning: %s", buf.String())
993 }
994}
995
996func TestMkfsTooOldRefusedAtStart(t *testing.T) {
997 e := newCEnv(t)
998 run := &cmdrun.Fake{Handle: func(c cmdrun.Call) ([]byte, error) {
999 if c.Name == "mkfs.ext4" {
1000 return []byte("mke2fs 1.42.9 (4-Feb-2014)\n"), nil
1001 }
1002 return nil, nil
1003 }}
1004 o := e.opts()
1005 o.Run = run
1006 if _, err := New(context.Background(), e.cfg, o); err == nil || !strings.Contains(err.Error(), "1.43") {
1007 t.Fatalf("got %v", err)
1008 }
1009}
1010
1011func TestSocketPathTooLongIsExplained(t *testing.T) {
1012 e := newCEnv(t)
1013 e.cfg.StateDir = filepath.Join(e.cfg.StateDir, strings.Repeat("d", 100))
1014 d := e.driver(t)
1015 sb, _ := d.New()
1016 _, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
1017 if err == nil || !strings.Contains(err.Error(), "108-byte limit") && !strings.Contains(err.Error(), "limit of a Unix socket") {
1018 t.Fatalf("got %v", err)
1019 }
1020 if left := e.leftovers(); len(left) != 0 {
1021 t.Fatalf("leftovers: %v", left)
1022 }
1023}
1024
1025// GO-002: only a podman daemon gets a DNS listener that may precede the gateway
1026// address; Docker keeps a normal bind.
1027func TestOnlyPodmanGetsAFreeBindDNSListener(t *testing.T) {
1028 if newNet(nil, 0, false, "", "").DNS.FreeBind {
1029 t.Fatal("Docker must keep a normal bind")
1030 }
1031 if !newNet(nil, 0, true, "", "").DNS.FreeBind {
1032 t.Fatal("podman needs the listener before the bridge address")
1033 }
1034 if newNet(nil, 0, false, "forge", "forge:80").Relay.FreeBind {
1035 t.Fatal("Docker must keep a normal bind for the relay")
1036 }
1037 if !newNet(nil, 0, true, "forge", "forge:80").Relay.FreeBind {
1038 t.Fatal("podman needs the relay listener before the bridge address")
1039 }
1040}
1041
1042// Guest forge access: the exception reaches the policy of every guest, and the
1043// warning naming the origin is logged at the driver start and at every job.
1044func TestGuestForgeIsAttachedAndWarnedAtEveryJob(t *testing.T) {
1045 e := newCEnv(t)
1046 var err error
1047 if e.cfg.GuestForge, err = sandbox.ParseGuestForge("http://ermite.bnema.dev:8088", true); err != nil {
1048 t.Fatal(err)
1049 }
1050 var buf bytes.Buffer
1051 ctx := logger(&buf)
1052 d, err := New(ctx, e.cfg, e.opts())
1053 if err != nil {
1054 t.Fatal(err)
1055 }
1056 defer func() { _ = d.Close() }()
1057 msg := "guest forge access: sandboxes may reach http://ermite.bnema.dev:8088 on the host"
1058 if n := strings.Count(buf.String(), msg); n != 1 {
1059 t.Fatalf("one warning at driver start, got %d", n)
1060 }
1061 for i := range 2 {
1062 sb, _ := d.New()
1063 conn, err := sb.Start(ctx, sandbox.JobSpec{JobID: fmt.Sprint("job", i)})
1064 if err != nil {
1065 t.Fatal(err)
1066 }
1067 for _, g := range e.plumb.Attached {
1068 if g.ForgePort != 8088 {
1069 t.Errorf("guest attached with ForgePort %d", g.ForgePort)
1070 }
1071 }
1072 stub(t, conn)
1073 _ = sb.Stop(ctx)
1074 }
1075 if n := strings.Count(buf.String(), msg); n != 3 {
1076 t.Fatalf("the warning must be logged at every job start (1 + 2), got %d", n)
1077 }
1078}
1079
1080func TestNoGuestForgeByDefault(t *testing.T) {
1081 e := newCEnv(t)
1082 var buf bytes.Buffer
1083 ctx := logger(&buf)
1084 d := e.driver(t)
1085 sb, _ := d.New()
1086 conn, err := sb.Start(ctx, sandbox.JobSpec{JobID: "j"})
1087 if err != nil {
1088 t.Fatal(err)
1089 }
1090 for _, g := range e.plumb.Attached {
1091 if g.ForgePort != 0 {
1092 t.Errorf("ForgePort %d without the setting", g.ForgePort)
1093 }
1094 }
1095 stub(t, conn)
1096 _ = sb.Stop(ctx)
1097 if strings.Contains(buf.String(), "guest forge") {
1098 t.Fatal("a guest forge warning without the setting")
1099 }
1100 if n := newNet(nil, 0, false, "ermite.example.org", "").DNS.Override; n != "ermite.example.org" {
1101 t.Fatalf("override %q", n)
1102 }
1103}
1104
1105// The host's external tools are bind-mounted read-only at
1106// /home/runner/externals/<name> (GitHub's layout), whichever the isolation.
1107func TestExternalsAreBoundReadOnly(t *testing.T) {
1108 e := newCEnv(t)
1109 src := &sandboxtest.Externals{}
1110 n24 := sandboxtest.MakeExternal(t, "node24", "a", "v24")
1111 n20 := sandboxtest.MakeExternal(t, "node20", "b", "v20")
1112 gone := sandbox.External{Name: "node24_alpine", ID: "c", Dir: "/state/externals/.unavailable/x", Unavailable: true}
1113 src.Set(n24, n20, gone)
1114 e.cfg.Externals = src
1115 d := e.driver(t)
1116 run := func() containerSpec {
1117 t.Helper()
1118 sb, _ := d.New()
1119 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
1120 if err != nil {
1121 t.Fatal(err)
1122 }
1123 stub(t, conn)
1124 e.eng.mu.Lock()
1125 var spec containerSpec
1126 for _, s := range e.eng.containers {
1127 spec = s
1128 }
1129 e.eng.mu.Unlock()
1130 _ = sb.Stop(context.Background())
1131 return spec
1132 }
1133 spec := run()
1134 got := map[string]bindMount{}
1135 for _, b := range spec.Binds {
1136 got[b.Target] = b
1137 }
1138 for _, w := range []sandbox.External{n24, n20, gone} {
1139 b, ok := got["/home/runner/externals/"+w.Name]
1140 if !ok || !b.ReadOnly || b.Source != w.Dir {
1141 t.Errorf("%s: %+v (binds %+v)", w.Name, b, spec.Binds)
1142 }
1143 }
1144 // The executable and the other binds are untouched.
1145 if b := got["/opt/ermite/ermite-runner"]; !b.ReadOnly || b.Source != e.cfg.Executable {
1146 t.Errorf("executable bind: %+v", b)
1147 }
1148 if len(spec.Binds) != 5+3 {
1149 t.Errorf("binds: %+v", spec.Binds)
1150 }
1151
1152 // A new Node.js is picked up by the next job, without a driver restart.
1153 src.Set(sandboxtest.MakeExternal(t, "node24", "d", "v24.22"))
1154 spec = run()
1155 n := 0
1156 for _, b := range spec.Binds {
1157 if strings.HasPrefix(b.Target, "/home/runner/externals/") {
1158 n++
1159 }
1160 }
1161 if n != 1 {
1162 t.Errorf("externals binds after the change: %+v", spec.Binds)
1163 }
1164}
1165
1166func TestNoExternalsNoBinds(t *testing.T) {
1167 e := newCEnv(t)
1168 d := e.driver(t)
1169 sb, _ := d.New()
1170 conn, err := sb.Start(context.Background(), sandbox.JobSpec{JobID: "j"})
1171 if err != nil {
1172 t.Fatal(err)
1173 }
1174 stub(t, conn)
1175 defer func() { _ = sb.Stop(context.Background()) }()
1176 e.eng.mu.Lock()
1177 defer e.eng.mu.Unlock()
1178 for _, s := range e.eng.containers {
1179 for _, b := range s.Binds {
1180 if strings.Contains(b.Target, "externals") {
1181 t.Errorf("unexpected bind %+v", b)
1182 }
1183 }
1184 }
1185}