detached 819edf3

fix(guest): wait for the Docker API before starting the executor

Init waits for GET /_ping on the Docker socket (at most 60 s) before starting the executor, reaping every child meanwhile. A shutdown during the wait stops dockerd (TERM, KILL after 5 s, reaped) and the job is reported cancelled; a dockerd that never answers or exits only logs a warning.
commit
819edf3a50ab9812b7a1c9a087683cceb893a494
author
brice · · 2026-10-14 09:59 UTC
committer
bnema · · 2026-10-14 09:59 UTC
parent
298aaf7
signature
SSH UnverifiedSHA256:VOCA5lYARHcbKDZ7GhTV45VmcjWnJ6ZQIBazj17LivA The key is unknown
2 files · +508 -24
Unified
UnifiedSplit
M internal/guest/guest_test.go +266 -0
@@ -4,8 +4,10 @@ package guest
44
55 import (
66 "context"
7+ "errors"
78 "fmt"
89 "net"
10+ "net/http"
911 "os"
1012 "path/filepath"
1113 "reflect"
@@ -17,6 +19,8 @@ import (
1719 "testing"
1820 "time"
1921
22+ "github.com/bnema/zerowrap"
23+
2024 "bnema.dev/ermite-runner/internal/execstream"
2125 "bnema.dev/ermite-runner/internal/execstream/execpb"
2226 )
@@ -409,3 +413,265 @@ func TestInitForwardsSIGTERMToTheExecutor(t *testing.T) {
409413 t.Fatal("the executor never received SIGTERM")
410414 }
411415 }
416+
417+// fakeDocker serves the Docker API on a temp unix socket from the moment it is
418+// called; ping decides the /_ping answer.
419+func fakeDocker(t *testing.T, ping http.HandlerFunc) string {
420+ t.Helper()
421+ sock := filepath.Join(t.TempDir(), "docker.sock")
422+ l, err := net.Listen("unix", sock)
423+ if err != nil {
424+ t.Fatal(err)
425+ }
426+ mux := http.NewServeMux()
427+ mux.HandleFunc("/_ping", ping)
428+ srv := &http.Server{Handler: mux}
429+ go func() { _ = srv.Serve(l) }()
430+ t.Cleanup(func() { _ = srv.Close() })
431+ return sock
432+}
433+
434+func sleeper(t *testing.T) int {
435+ t.Helper()
436+ pid, err := spawn([]string{"/bin/sh", "-c", "sleep 60"}, nil)
437+ if err != nil {
438+ t.Fatal(err)
439+ }
440+ t.Cleanup(func() { _ = syscall.Kill(pid, syscall.SIGKILL); _, _ = wait4NoHang(pid) })
441+ return pid
442+}
443+
444+var testLog = zerowrap.FromCtx(context.Background())
445+
446+func TestWaitDockerReadyWaitsForPing(t *testing.T) {
447+ var calls atomic.Int32
448+ sock := fakeDocker(t, func(w http.ResponseWriter, _ *http.Request) {
449+ if calls.Add(1) < 4 { // dockerd is up but still starting
450+ http.Error(w, "starting", http.StatusServiceUnavailable)
451+ return
452+ }
453+ _, _ = w.Write([]byte("OK"))
454+ })
455+ start := time.Now()
456+ if err := waitDockerReady(context.Background(), testLog, sock, sleeper(t), 10*time.Second, nil); err != nil {
457+ t.Fatal(err)
458+ }
459+ if calls.Load() < 4 || time.Since(start) < 200*time.Millisecond {
460+ t.Fatalf("returned before the API answered OK (calls=%d)", calls.Load())
461+ }
462+}
463+
464+func TestWaitDockerReadyLateSocket(t *testing.T) {
465+ sock := filepath.Join(t.TempDir(), "docker.sock")
466+ go func() {
467+ time.Sleep(400 * time.Millisecond)
468+ l, err := net.Listen("unix", sock)
469+ if err != nil {
470+ return
471+ }
472+ srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte("OK")) })}
473+ go func() { _ = srv.Serve(l) }()
474+ time.AfterFunc(5*time.Second, func() { _ = srv.Close() })
475+ }()
476+ start := time.Now()
477+ if err := waitDockerReady(context.Background(), testLog, sock, sleeper(t), 10*time.Second, nil); err != nil {
478+ t.Fatal(err)
479+ }
480+ if time.Since(start) < 400*time.Millisecond {
481+ t.Fatal("returned before the socket existed")
482+ }
483+}
484+
485+func TestWaitDockerReadyTimesOut(t *testing.T) {
486+ sock := fakeDocker(t, func(w http.ResponseWriter, _ *http.Request) { http.Error(w, "no", http.StatusServiceUnavailable) })
487+ start := time.Now()
488+ err := waitDockerReady(context.Background(), testLog, sock, sleeper(t), 500*time.Millisecond, nil)
489+ if err == nil || time.Since(start) > 5*time.Second {
490+ t.Fatalf("want a timeout error, got %v after %s", err, time.Since(start))
491+ }
492+ var exited *dockerdExitError
493+ var abort *abortBootError
494+ if errors.As(err, &exited) || errors.As(err, &abort) {
495+ t.Fatalf("a timeout is neither a dockerd exit nor an abort: %v", err)
496+ }
497+}
498+
499+// A daemon that accepts the connection and never answers must not delay the
500+// deadline, nor a signal.
501+func TestWaitDockerReadyStalledServer(t *testing.T) {
502+ release := make(chan struct{})
503+ t.Cleanup(func() { close(release) })
504+ sock := fakeDocker(t, func(http.ResponseWriter, *http.Request) { <-release })
505+
506+ start := time.Now()
507+ err := waitDockerReady(context.Background(), testLog, sock, sleeper(t), 500*time.Millisecond, nil)
508+ var abort *abortBootError
509+ if err == nil || errors.As(err, &abort) || time.Since(start) > 1500*time.Millisecond {
510+ t.Fatalf("want the deadline error within ~0.5 s, got %v after %s", err, time.Since(start))
511+ }
512+
513+ sigs := make(chan os.Signal, 1)
514+ time.AfterFunc(200*time.Millisecond, func() { sigs <- syscall.SIGTERM })
515+ start = time.Now()
516+ err = waitDockerReady(context.Background(), testLog, sock, sleeper(t), 30*time.Second, sigs)
517+ if !errors.As(err, &abort) || time.Since(start) > 1500*time.Millisecond {
518+ t.Fatalf("want an abort right after the signal, got %v after %s", err, time.Since(start))
519+ }
520+}
521+
522+func TestWaitDockerReadyDockerdExits(t *testing.T) {
523+ pid, err := spawn([]string{"/bin/sh", "-c", "exit 3"}, nil)
524+ if err != nil {
525+ t.Fatal(err)
526+ }
527+ start := time.Now()
528+ err = waitDockerReady(context.Background(), testLog, filepath.Join(t.TempDir(), "none.sock"), pid, 30*time.Second, nil)
529+ var exited *dockerdExitError
530+ if !errors.As(err, &exited) || exited.status.ExitStatus() != 3 {
531+ t.Fatalf("want dockerd exit 3, got %v", err)
532+ }
533+ if time.Since(start) > 5*time.Second {
534+ t.Fatal("did not notice the exit promptly")
535+ }
536+}
537+
538+// Orphans that init adopts while it waits must be reaped at once, not left as
539+// zombies until the wait ends.
540+func TestWaitDockerReadyReapsAdoptedOrphans(t *testing.T) {
541+ // A child that exits immediately stands for an adopted orphan.
542+ orphan, err := spawn([]string{"/bin/sh", "-c", "exit 0"}, nil)
543+ if err != nil {
544+ t.Fatal(err)
545+ }
546+ zombie := func() bool {
547+ b, err := os.ReadFile("/proc/" + strconv.Itoa(orphan) + "/stat")
548+ return err == nil && strings.Contains(string(b), ") Z ")
549+ }
550+ deadline := time.Now().Add(5 * time.Second)
551+ for !zombie() && time.Now().Before(deadline) {
552+ time.Sleep(10 * time.Millisecond)
553+ }
554+ if !zombie() {
555+ t.Fatal("setup: the orphan did not become a zombie")
556+ }
557+ sigs := make(chan os.Signal, 1)
558+ time.AfterFunc(600*time.Millisecond, func() { sigs <- syscall.SIGTERM })
559+ done := make(chan error, 1)
560+ go func() {
561+ done <- waitDockerReady(context.Background(), testLog, filepath.Join(t.TempDir(), "none.sock"), sleeper(t), 30*time.Second, sigs)
562+ }()
563+ // The zombie must vanish while the wait is still running.
564+ deadline = time.Now().Add(400 * time.Millisecond)
565+ for zombie() && time.Now().Before(deadline) {
566+ time.Sleep(10 * time.Millisecond)
567+ }
568+ if zombie() {
569+ t.Fatal("the adopted orphan stayed a zombie during the wait")
570+ }
571+ <-done
572+}
573+
574+// A shutdown signal during the wait aborts the boot (exit 128+signal) instead
575+// of starting the executor; so does a cancelled context.
576+func TestWaitDockerReadyAbortsOnSignalAndContext(t *testing.T) {
577+ sock := filepath.Join(t.TempDir(), "none.sock")
578+ sigs := make(chan os.Signal, 1)
579+ sigs <- syscall.SIGINT
580+ var abort *abortBootError
581+ err := waitDockerReady(context.Background(), testLog, sock, sleeper(t), 30*time.Second, sigs)
582+ if !errors.As(err, &abort) || abort.code != 128+int(syscall.SIGINT) {
583+ t.Fatalf("err=%v", err)
584+ }
585+ ctx, cancel := context.WithCancel(context.Background())
586+ cancel()
587+ err = waitDockerReady(ctx, testLog, sock, sleeper(t), 30*time.Second, nil)
588+ if !errors.As(err, &abort) || abort.code != 128+int(syscall.SIGTERM) {
589+ t.Fatalf("err=%v", err)
590+ }
591+}
592+
593+func TestStopDockerdReapsAndZeroIsNoop(t *testing.T) {
594+ stopDockerd(0)
595+ pid := sleeper(t)
596+ stopDockerd(pid)
597+ if _, err := os.Stat("/proc/" + strconv.Itoa(pid)); err == nil {
598+ t.Fatal("dockerd is still there")
599+ }
600+}
601+
602+func TestDockerPingRejectsNonOK(t *testing.T) {
603+ sock := fakeDocker(t, func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte("nope")) })
604+ client := &http.Client{Transport: &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
605+ var d net.Dialer
606+ return d.DialContext(ctx, "unix", sock)
607+ }}}
608+ if dockerPing(context.Background(), client) {
609+ t.Fatal("a 200 without OK must not count as ready")
610+ }
611+}
612+
613+func initWithFakeDockerd(t *testing.T, sigs <-chan os.Signal, timeout time.Duration, dockerd string) (code int, marker string) {
614+ t.Helper()
615+ marker = filepath.Join(t.TempDir(), "executor-ran")
616+ code = Init(context.Background(), InitConfig{
617+ Cmdline: "ermite.sandbox=container ermite.connect=unix:/nonexistent",
618+ Getpid: func() int { return 4242 },
619+ Signals: sigs,
620+ DockerdArgs: []string{"/bin/sh", "-c", dockerd},
621+ DockerSocket: filepath.Join(t.TempDir(), "none.sock"),
622+ DockerReadyTimeout: timeout,
623+ ExecArgs: []string{"/bin/sh", "-c", "touch " + marker + "; exit 7"},
624+ })
625+ return code, marker
626+}
627+
628+// A shutdown signal while waiting for dockerd: no executor, dockerd stopped,
629+// exit 128+signal.
630+func TestInitSignalDuringDockerWaitDoesNotStartTheExecutor(t *testing.T) {
631+ sigs := make(chan os.Signal, 1)
632+ sigs <- syscall.SIGTERM
633+ code, marker := initWithFakeDockerd(t, sigs, 30*time.Second, "sleep 60")
634+ if code != 128+int(syscall.SIGTERM) {
635+ t.Fatalf("code=%d", code)
636+ }
637+ if _, err := os.Stat(marker); err == nil {
638+ t.Fatal("the executor was started")
639+ }
640+}
641+
642+// The deadline and a dying dockerd still start the executor.
643+func TestInitStartsTheExecutorWhenDockerNeverGetsReady(t *testing.T) {
644+ for name, tc := range map[string]struct {
645+ dockerd string
646+ timeout time.Duration
647+ }{"timeout": {"sleep 60", 400 * time.Millisecond}, "dockerd exits": {"exit 1", 30 * time.Second}} {
648+ t.Run(name, func(t *testing.T) {
649+ code, marker := initWithFakeDockerd(t, nil, tc.timeout, tc.dockerd)
650+ if code != 7 {
651+ t.Fatalf("want the executor's code 7, got %d", code)
652+ }
653+ if _, err := os.Stat(marker); err != nil {
654+ t.Fatal("the executor did not run")
655+ }
656+ })
657+ }
658+}
659+
660+// A dockerd that ignores SIGTERM goes through the SIGKILL path and is reaped
661+// (no zombie left).
662+func TestStopDockerdKillsAndReapsATermIgnoringDaemon(t *testing.T) {
663+ pid, err := spawn([]string{"/bin/sh", "-c", "trap '' TERM; while :; do sleep 0.1; done"}, nil)
664+ if err != nil {
665+ t.Fatal(err)
666+ }
667+ t.Cleanup(func() { _ = syscall.Kill(pid, syscall.SIGKILL) })
668+ time.Sleep(200 * time.Millisecond) // let the shell install its trap
669+ start := time.Now()
670+ stopDockerd(pid)
671+ if d := time.Since(start); d < 5*time.Second || d > 8*time.Second {
672+ t.Fatalf("expected the SIGKILL path after the 5 s grace, took %s", d)
673+ }
674+ if _, err := os.Stat("/proc/" + strconv.Itoa(pid)); err == nil {
675+ t.Fatal("dockerd was not reaped (zombie or alive)")
676+ }
677+}
@@ -4,8 +4,10 @@ package guest
4 4
5 import (5 import (
6 "context"6 "context"
7+ "errors"
7 "fmt"8 "fmt"
8 "net"9 "net"
10+ "net/http"
9 "os"11 "os"
10 "path/filepath"12 "path/filepath"
11 "reflect"13 "reflect"
@@ -17,6 +19,8 @@ import (
17 "testing"19 "testing"
18 "time"20 "time"
19 21
22+ "github.com/bnema/zerowrap"
23+
20 "bnema.dev/ermite-runner/internal/execstream"24 "bnema.dev/ermite-runner/internal/execstream"
21 "bnema.dev/ermite-runner/internal/execstream/execpb"25 "bnema.dev/ermite-runner/internal/execstream/execpb"
22 )26 )
@@ -409,3 +413,265 @@ func TestInitForwardsSIGTERMToTheExecutor(t *testing.T) {
409 t.Fatal("the executor never received SIGTERM")413 t.Fatal("the executor never received SIGTERM")
410 }414 }
411 }415 }
416+
417+// fakeDocker serves the Docker API on a temp unix socket from the moment it is
418+// called; ping decides the /_ping answer.
419+func fakeDocker(t *testing.T, ping http.HandlerFunc) string {
420+ t.Helper()
421+ sock := filepath.Join(t.TempDir(), "docker.sock")
422+ l, err := net.Listen("unix", sock)
423+ if err != nil {
424+ t.Fatal(err)
425+ }
426+ mux := http.NewServeMux()
427+ mux.HandleFunc("/_ping", ping)
428+ srv := &http.Server{Handler: mux}
429+ go func() { _ = srv.Serve(l) }()
430+ t.Cleanup(func() { _ = srv.Close() })
431+ return sock
432+}
433+
434+func sleeper(t *testing.T) int {
435+ t.Helper()
436+ pid, err := spawn([]string{"/bin/sh", "-c", "sleep 60"}, nil)
437+ if err != nil {
438+ t.Fatal(err)
439+ }
440+ t.Cleanup(func() { _ = syscall.Kill(pid, syscall.SIGKILL); _, _ = wait4NoHang(pid) })
441+ return pid
442+}
443+
444+var testLog = zerowrap.FromCtx(context.Background())
445+
446+func TestWaitDockerReadyWaitsForPing(t *testing.T) {
447+ var calls atomic.Int32
448+ sock := fakeDocker(t, func(w http.ResponseWriter, _ *http.Request) {
449+ if calls.Add(1) < 4 { // dockerd is up but still starting
450+ http.Error(w, "starting", http.StatusServiceUnavailable)
451+ return
452+ }
453+ _, _ = w.Write([]byte("OK"))
454+ })
455+ start := time.Now()
456+ if err := waitDockerReady(context.Background(), testLog, sock, sleeper(t), 10*time.Second, nil); err != nil {
457+ t.Fatal(err)
458+ }
459+ if calls.Load() < 4 || time.Since(start) < 200*time.Millisecond {
460+ t.Fatalf("returned before the API answered OK (calls=%d)", calls.Load())
461+ }
462+}
463+
464+func TestWaitDockerReadyLateSocket(t *testing.T) {
465+ sock := filepath.Join(t.TempDir(), "docker.sock")
466+ go func() {
467+ time.Sleep(400 * time.Millisecond)
468+ l, err := net.Listen("unix", sock)
469+ if err != nil {
470+ return
471+ }
472+ srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte("OK")) })}
473+ go func() { _ = srv.Serve(l) }()
474+ time.AfterFunc(5*time.Second, func() { _ = srv.Close() })
475+ }()
476+ start := time.Now()
477+ if err := waitDockerReady(context.Background(), testLog, sock, sleeper(t), 10*time.Second, nil); err != nil {
478+ t.Fatal(err)
479+ }
480+ if time.Since(start) < 400*time.Millisecond {
481+ t.Fatal("returned before the socket existed")
482+ }
483+}
484+
485+func TestWaitDockerReadyTimesOut(t *testing.T) {
486+ sock := fakeDocker(t, func(w http.ResponseWriter, _ *http.Request) { http.Error(w, "no", http.StatusServiceUnavailable) })
487+ start := time.Now()
488+ err := waitDockerReady(context.Background(), testLog, sock, sleeper(t), 500*time.Millisecond, nil)
489+ if err == nil || time.Since(start) > 5*time.Second {
490+ t.Fatalf("want a timeout error, got %v after %s", err, time.Since(start))
491+ }
492+ var exited *dockerdExitError
493+ var abort *abortBootError
494+ if errors.As(err, &exited) || errors.As(err, &abort) {
495+ t.Fatalf("a timeout is neither a dockerd exit nor an abort: %v", err)
496+ }
497+}
498+
499+// A daemon that accepts the connection and never answers must not delay the
500+// deadline, nor a signal.
501+func TestWaitDockerReadyStalledServer(t *testing.T) {
502+ release := make(chan struct{})
503+ t.Cleanup(func() { close(release) })
504+ sock := fakeDocker(t, func(http.ResponseWriter, *http.Request) { <-release })
505+
506+ start := time.Now()
507+ err := waitDockerReady(context.Background(), testLog, sock, sleeper(t), 500*time.Millisecond, nil)
508+ var abort *abortBootError
509+ if err == nil || errors.As(err, &abort) || time.Since(start) > 1500*time.Millisecond {
510+ t.Fatalf("want the deadline error within ~0.5 s, got %v after %s", err, time.Since(start))
511+ }
512+
513+ sigs := make(chan os.Signal, 1)
514+ time.AfterFunc(200*time.Millisecond, func() { sigs <- syscall.SIGTERM })
515+ start = time.Now()
516+ err = waitDockerReady(context.Background(), testLog, sock, sleeper(t), 30*time.Second, sigs)
517+ if !errors.As(err, &abort) || time.Since(start) > 1500*time.Millisecond {
518+ t.Fatalf("want an abort right after the signal, got %v after %s", err, time.Since(start))
519+ }
520+}
521+
522+func TestWaitDockerReadyDockerdExits(t *testing.T) {
523+ pid, err := spawn([]string{"/bin/sh", "-c", "exit 3"}, nil)
524+ if err != nil {
525+ t.Fatal(err)
526+ }
527+ start := time.Now()
528+ err = waitDockerReady(context.Background(), testLog, filepath.Join(t.TempDir(), "none.sock"), pid, 30*time.Second, nil)
529+ var exited *dockerdExitError
530+ if !errors.As(err, &exited) || exited.status.ExitStatus() != 3 {
531+ t.Fatalf("want dockerd exit 3, got %v", err)
532+ }
533+ if time.Since(start) > 5*time.Second {
534+ t.Fatal("did not notice the exit promptly")
535+ }
536+}
537+
538+// Orphans that init adopts while it waits must be reaped at once, not left as
539+// zombies until the wait ends.
540+func TestWaitDockerReadyReapsAdoptedOrphans(t *testing.T) {
541+ // A child that exits immediately stands for an adopted orphan.
542+ orphan, err := spawn([]string{"/bin/sh", "-c", "exit 0"}, nil)
543+ if err != nil {
544+ t.Fatal(err)
545+ }
546+ zombie := func() bool {
547+ b, err := os.ReadFile("/proc/" + strconv.Itoa(orphan) + "/stat")
548+ return err == nil && strings.Contains(string(b), ") Z ")
549+ }
550+ deadline := time.Now().Add(5 * time.Second)
551+ for !zombie() && time.Now().Before(deadline) {
552+ time.Sleep(10 * time.Millisecond)
553+ }
554+ if !zombie() {
555+ t.Fatal("setup: the orphan did not become a zombie")
556+ }
557+ sigs := make(chan os.Signal, 1)
558+ time.AfterFunc(600*time.Millisecond, func() { sigs <- syscall.SIGTERM })
559+ done := make(chan error, 1)
560+ go func() {
561+ done <- waitDockerReady(context.Background(), testLog, filepath.Join(t.TempDir(), "none.sock"), sleeper(t), 30*time.Second, sigs)
562+ }()
563+ // The zombie must vanish while the wait is still running.
564+ deadline = time.Now().Add(400 * time.Millisecond)
565+ for zombie() && time.Now().Before(deadline) {
566+ time.Sleep(10 * time.Millisecond)
567+ }
568+ if zombie() {
569+ t.Fatal("the adopted orphan stayed a zombie during the wait")
570+ }
571+ <-done
572+}
573+
574+// A shutdown signal during the wait aborts the boot (exit 128+signal) instead
575+// of starting the executor; so does a cancelled context.
576+func TestWaitDockerReadyAbortsOnSignalAndContext(t *testing.T) {
577+ sock := filepath.Join(t.TempDir(), "none.sock")
578+ sigs := make(chan os.Signal, 1)
579+ sigs <- syscall.SIGINT
580+ var abort *abortBootError
581+ err := waitDockerReady(context.Background(), testLog, sock, sleeper(t), 30*time.Second, sigs)
582+ if !errors.As(err, &abort) || abort.code != 128+int(syscall.SIGINT) {
583+ t.Fatalf("err=%v", err)
584+ }
585+ ctx, cancel := context.WithCancel(context.Background())
586+ cancel()
587+ err = waitDockerReady(ctx, testLog, sock, sleeper(t), 30*time.Second, nil)
588+ if !errors.As(err, &abort) || abort.code != 128+int(syscall.SIGTERM) {
589+ t.Fatalf("err=%v", err)
590+ }
591+}
592+
593+func TestStopDockerdReapsAndZeroIsNoop(t *testing.T) {
594+ stopDockerd(0)
595+ pid := sleeper(t)
596+ stopDockerd(pid)
597+ if _, err := os.Stat("/proc/" + strconv.Itoa(pid)); err == nil {
598+ t.Fatal("dockerd is still there")
599+ }
600+}
601+
602+func TestDockerPingRejectsNonOK(t *testing.T) {
603+ sock := fakeDocker(t, func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte("nope")) })
604+ client := &http.Client{Transport: &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
605+ var d net.Dialer
606+ return d.DialContext(ctx, "unix", sock)
607+ }}}
608+ if dockerPing(context.Background(), client) {
609+ t.Fatal("a 200 without OK must not count as ready")
610+ }
611+}
612+
613+func initWithFakeDockerd(t *testing.T, sigs <-chan os.Signal, timeout time.Duration, dockerd string) (code int, marker string) {
614+ t.Helper()
615+ marker = filepath.Join(t.TempDir(), "executor-ran")
616+ code = Init(context.Background(), InitConfig{
617+ Cmdline: "ermite.sandbox=container ermite.connect=unix:/nonexistent",
618+ Getpid: func() int { return 4242 },
619+ Signals: sigs,
620+ DockerdArgs: []string{"/bin/sh", "-c", dockerd},
621+ DockerSocket: filepath.Join(t.TempDir(), "none.sock"),
622+ DockerReadyTimeout: timeout,
623+ ExecArgs: []string{"/bin/sh", "-c", "touch " + marker + "; exit 7"},
624+ })
625+ return code, marker
626+}
627+
628+// A shutdown signal while waiting for dockerd: no executor, dockerd stopped,
629+// exit 128+signal.
630+func TestInitSignalDuringDockerWaitDoesNotStartTheExecutor(t *testing.T) {
631+ sigs := make(chan os.Signal, 1)
632+ sigs <- syscall.SIGTERM
633+ code, marker := initWithFakeDockerd(t, sigs, 30*time.Second, "sleep 60")
634+ if code != 128+int(syscall.SIGTERM) {
635+ t.Fatalf("code=%d", code)
636+ }
637+ if _, err := os.Stat(marker); err == nil {
638+ t.Fatal("the executor was started")
639+ }
640+}
641+
642+// The deadline and a dying dockerd still start the executor.
643+func TestInitStartsTheExecutorWhenDockerNeverGetsReady(t *testing.T) {
644+ for name, tc := range map[string]struct {
645+ dockerd string
646+ timeout time.Duration
647+ }{"timeout": {"sleep 60", 400 * time.Millisecond}, "dockerd exits": {"exit 1", 30 * time.Second}} {
648+ t.Run(name, func(t *testing.T) {
649+ code, marker := initWithFakeDockerd(t, nil, tc.timeout, tc.dockerd)
650+ if code != 7 {
651+ t.Fatalf("want the executor's code 7, got %d", code)
652+ }
653+ if _, err := os.Stat(marker); err != nil {
654+ t.Fatal("the executor did not run")
655+ }
656+ })
657+ }
658+}
659+
660+// A dockerd that ignores SIGTERM goes through the SIGKILL path and is reaped
661+// (no zombie left).
662+func TestStopDockerdKillsAndReapsATermIgnoringDaemon(t *testing.T) {
663+ pid, err := spawn([]string{"/bin/sh", "-c", "trap '' TERM; while :; do sleep 0.1; done"}, nil)
664+ if err != nil {
665+ t.Fatal(err)
666+ }
667+ t.Cleanup(func() { _ = syscall.Kill(pid, syscall.SIGKILL) })
668+ time.Sleep(200 * time.Millisecond) // let the shell install its trap
669+ start := time.Now()
670+ stopDockerd(pid)
671+ if d := time.Since(start); d < 5*time.Second || d > 8*time.Second {
672+ t.Fatalf("expected the SIGKILL path after the 5 s grace, took %s", d)
673+ }
674+ if _, err := os.Stat("/proc/" + strconv.Itoa(pid)); err == nil {
675+ t.Fatal("dockerd was not reaped (zombie or alive)")
676+ }
677+}
M internal/guest/init.go +242 -24
@@ -4,7 +4,11 @@ package guest
44
55 import (
66 "context"
7+ "errors"
78 "fmt"
9+ "io"
10+ "net"
11+ "net/http"
812 "os"
913 "os/signal"
1014 "strings"
@@ -33,6 +37,15 @@ type InitConfig struct {
3337 // ExecArgs is the executor command line; default
3438 // [GuestExecDir/GuestExecName exec].
3539 ExecArgs []string
40+ // DockerdArgs replaces the dockerd command line, and the check that
41+ // /usr/bin/dockerd exists (tests); SkipDocker still wins.
42+ DockerdArgs []string
43+ // DockerSocket is the Docker API socket dockerd listens on and init waits
44+ // for; default dockerSocket.
45+ DockerSocket string
46+ // DockerReadyTimeout bounds the wait for the Docker API before the executor
47+ // is started anyway; default dockerReadyTimeout.
48+ DockerReadyTimeout time.Duration
3649 }
3750
3851 // Init is `ermite-runner init`: the entrypoint of every sandbox.
@@ -45,7 +58,7 @@ type InitConfig struct {
4558 //
4659 // In a container the root and the pseudo filesystems are ready: it skips all of
4760 // that and only prepares cgroups. Then, in both: start dockerd with its data
48-// root on the scratch space, wait for its socket, run the executor
61+// root on the scratch space, wait (bounded) for its API to answer, run the executor
4962 // (`ermite-runner exec`) as a child, reap orphans, stop dockerd, and finish by
5063 // powering the VM off, or exiting with the executor's code in a container.
5164 func Init(ctx context.Context, cfg InitConfig) int {
@@ -116,10 +129,15 @@ func runPayload(ctx context.Context, cfg InitConfig, mode string, args map[strin
116129 }
117130
118131 var dockerPid int
119- if _, err := os.Stat("/usr/bin/dockerd"); err == nil && !cfg.SkipDocker {
132+ _, statErr := os.Stat("/usr/bin/dockerd")
133+ if !cfg.SkipDocker && (statErr == nil || cfg.DockerdArgs != nil) {
120134 _ = os.MkdirAll(sandbox.GuestScratch+"/docker", 0o711)
121- pid, err := spawn([]string{"/usr/bin/dockerd", "--data-root", sandbox.GuestScratch + "/docker",
122- "--storage-driver", "overlay2", "--host", "unix:///var/run/docker.sock"}, nil)
135+ argv := cfg.DockerdArgs
136+ if argv == nil {
137+ argv = []string{"/usr/bin/dockerd", "--data-root", sandbox.GuestScratch + "/docker",
138+ "--storage-driver", "overlay2", "--host", "unix:///var/run/docker.sock"}
139+ }
140+ pid, err := spawn(argv, nil)
123141 if err != nil {
124142 log.Error().Err(err).Msg("init: start dockerd")
125143 } else {
@@ -127,12 +145,6 @@ func runPayload(ctx context.Context, cfg InitConfig, mode string, args map[strin
127145 }
128146 }
129147
130- execPid, err := spawn(cfg.ExecArgs, append(os.Environ(),
131- "ERMITE_CONNECT="+target, "ERMITE_SANDBOX="+mode, "ERMITE_VERSION="+cfg.Version))
132- if err != nil {
133- log.Error().Err(err).Msg("init: start executor")
134- return 1
135- }
136148 // SIGTERM/SIGINT (docker stop, or the host stopping the VM) go to the
137149 // executor, which cancels its script with the usual escalation and reports
138150 // its Exit; init then finishes normally instead of dying with children.
@@ -143,35 +155,241 @@ func runPayload(ctx context.Context, cfg InitConfig, mode string, args map[strin
143155 defer signal.Stop(ch)
144156 sigs = ch
145157 }
158+
159+ // The first job-container step needs the Docker API, so the executor must
160+ // not start before dockerd answers. If it never does (or dies), start the
161+ // executor anyway: jobs that do not use Docker still run. A shutdown signal
162+ // during the wait aborts the boot instead: see abortBootError.
163+ if dockerPid != 0 {
164+ sock := cfg.DockerSocket
165+ if sock == "" {
166+ sock = dockerSocket
167+ }
168+ timeout := cfg.DockerReadyTimeout
169+ if timeout <= 0 {
170+ timeout = dockerReadyTimeout
171+ }
172+ err := waitDockerReady(ctx, log, sock, dockerPid, timeout, sigs)
173+ var exited *dockerdExitError
174+ var abort *abortBootError
175+ switch {
176+ case errors.As(err, &abort):
177+ stopDockerd(dockerPid)
178+ return abort.code
179+ case errors.As(err, &exited):
180+ dockerPid = 0 // already reaped
181+ }
182+ }
183+
184+ execPid, err := spawn(cfg.ExecArgs, append(os.Environ(),
185+ "ERMITE_CONNECT="+target, "ERMITE_SANDBOX="+mode, "ERMITE_VERSION="+cfg.Version))
186+ if err != nil {
187+ log.Error().Err(err).Msg("init: start executor")
188+ stopDockerd(dockerPid)
189+ return 1
190+ }
191+ forward := func(sig os.Signal) {
192+ if s, ok := sig.(syscall.Signal); ok {
193+ log.Info().Str("signal", s.String()).Msg("init: forwarding the signal to the executor")
194+ _ = syscall.Kill(-execPid, s) // the executor leads its own process group
195+ _ = syscall.Kill(execPid, s)
196+ }
197+ }
146198 stopFwd := make(chan struct{})
147199 defer close(stopFwd)
148200 go func() {
149201 for {
150202 select {
151203 case sig := <-sigs:
152- if s, ok := sig.(syscall.Signal); ok {
153- log.Info().Str("signal", s.String()).Msg("init: forwarding the signal to the executor")
154- _ = syscall.Kill(-execPid, s) // the executor leads its own process group
155- _ = syscall.Kill(execPid, s)
156- }
204+ forward(sig)
157205 case <-stopFwd:
158206 return
159207 }
160208 }
161209 }()
162210 code := reapUntil(execPid)
163- if dockerPid != 0 {
164- _ = syscall.Kill(dockerPid, syscall.SIGTERM)
165- deadline := time.Now().Add(5 * time.Second)
166- for time.Now().Before(deadline) {
167- if pid, _ := wait4NoHang(dockerPid); pid == dockerPid {
168- break
211+ stopDockerd(dockerPid)
212+ return code
213+}
214+
215+// stopDockerd stops dockerd (SIGTERM, then SIGKILL after 5 s) and reaps it (at
216+// most 5 s more after the SIGKILL); a zero pid is a no-op.
217+func stopDockerd(pid int) {
218+ if pid == 0 {
219+ return
220+ }
221+ _ = syscall.Kill(pid, syscall.SIGTERM)
222+ deadline := time.Now().Add(5 * time.Second)
223+ for time.Now().Before(deadline) {
224+ if got, _ := wait4NoHang(pid); got == pid {
225+ return
226+ }
227+ time.Sleep(100 * time.Millisecond)
228+ }
229+ _ = syscall.Kill(pid, syscall.SIGKILL)
230+ // SIGKILL cannot be ignored, so the exit is prompt; poll rather than block
231+ // for ever on a process stuck in the kernel (uninterruptible sleep).
232+ for end := time.Now().Add(5 * time.Second); time.Now().Before(end); time.Sleep(10 * time.Millisecond) {
233+ got, err := wait4NoHang(pid)
234+ if got == pid || (err != nil && err != syscall.EINTR) { // ECHILD: already reaped
235+ return
236+ }
237+ }
238+}
239+
240+const (
241+ // dockerSocket is the Docker API socket of the nested dockerd.
242+ dockerSocket = "/var/run/docker.sock"
243+ // dockerReadyTimeout bounds the wait for the Docker API at boot.
244+ dockerReadyTimeout = 60 * time.Second
245+ dockerPollInterval = 100 * time.Millisecond
246+ dockerPingTimeout = 2 * time.Second
247+)
248+
249+// dockerdExitError reports that dockerd exited (and was reaped) before its API
250+// answered.
251+type dockerdExitError struct{ status syscall.WaitStatus }
252+
253+func (e *dockerdExitError) Error() string {
254+ if e.status.Signaled() {
255+ return "dockerd killed by signal " + e.status.Signal().String()
256+ }
257+ return fmt.Sprintf("dockerd exited with code %d", e.status.ExitStatus())
258+}
259+
260+// abortBootError means a shutdown signal (or the cancelled context) arrived
261+// while waiting for the Docker API: init must not start the executor, stop
262+// dockerd and exit with code (128+signal, as for a killed executor).
263+//
264+// Not starting the executor is safe for the host: the sandbox ends before the
265+// executor connected ("container exited ... before the executor connected" /
266+// "firecracker exited before the executor connected"), and the host reports the
267+// job cancelled (not failed) when a cancel or shutdown was requested
268+// (jobRun.execute: startSandbox error with cancelRequested). A stop that the
269+// host did not ask for is reported as "the sandbox did not start".
270+type abortBootError struct{ code int }
271+
272+func (e *abortBootError) Error() string {
273+ return fmt.Sprintf("boot aborted by a shutdown signal (exit %d)", e.code)
274+}
275+
276+// reapChildren reaps every child that has exited (orphans adopted by init
277+// included) without blocking, and reports whether pid was among them, with its
278+// wait status.
279+func reapChildren(pid int) (bool, syscall.WaitStatus) {
280+ var found bool
281+ var status syscall.WaitStatus
282+ for {
283+ var ws syscall.WaitStatus
284+ got, err := syscall.Wait4(-1, &ws, syscall.WNOHANG, nil)
285+ if err == syscall.EINTR {
286+ continue
287+ }
288+ if err != nil || got <= 0 {
289+ return found, status
290+ }
291+ if got == pid {
292+ found, status = true, ws
293+ }
294+ }
295+}
296+
297+// waitDockerReady blocks until the Docker API on sock answers GET /_ping with
298+// "OK", for at most timeout. It returns nil when ready, and otherwise:
299+// - *dockerdExitError: dockerd (pid, a child of init) exited and was reaped;
300+// - a plain error: the deadline passed;
301+// - *abortBootError: a signal arrived on sigs or ctx was cancelled.
302+//
303+// The orphans that init adopts are reaped on every tick, and a ping in flight
304+// never delays the deadline, the signals or the exit check. Every outcome is
305+// logged; for the first two the caller starts the executor anyway.
306+func waitDockerReady(ctx context.Context, log zerowrap.Logger, sock string, pid int, timeout time.Duration, sigs <-chan os.Signal) error {
307+ start := time.Now()
308+ log.Info().Str("socket", sock).Dur("timeout", timeout).Msg("init: waiting for the Docker API")
309+ wctx, cancel := context.WithTimeout(ctx, timeout)
310+ defer cancel() // also aborts a ping in flight
311+ client := &http.Client{Transport: &http.Transport{
312+ DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
313+ var d net.Dialer
314+ return d.DialContext(ctx, "unix", sock)
315+ },
316+ DisableKeepAlives: true,
317+ }}
318+ defer client.CloseIdleConnections()
319+ ticker := time.NewTicker(dockerPollInterval)
320+ defer ticker.Stop()
321+
322+ var pinging <-chan bool // result of the ping in flight, nil if none
323+ startPing := func() {
324+ ch := make(chan bool, 1)
325+ pinging = ch
326+ go func() { ch <- dockerPing(wctx, client) }()
327+ }
328+ checkExit := func() error {
329+ if ok, ws := reapChildren(pid); ok {
330+ err := &dockerdExitError{status: ws}
331+ log.Warn().Err(err).Msg("init: dockerd died before its API was ready; starting the executor anyway (jobs using Docker will fail)")
332+ return err
333+ }
334+ return nil
335+ }
336+ if err := checkExit(); err != nil {
337+ return err
338+ }
339+ startPing()
340+ for {
341+ select {
342+ case ok := <-pinging:
343+ pinging = nil
344+ if ok {
345+ log.Info().Dur("after", time.Since(start).Round(time.Millisecond)).Msg("init: the Docker API is ready")
346+ return nil
347+ }
348+ case <-ticker.C:
349+ if err := checkExit(); err != nil {
350+ return err
351+ }
352+ if pinging == nil {
353+ startPing()
169354 }
170- time.Sleep(100 * time.Millisecond)
355+ case sig := <-sigs:
356+ return abortBoot(log, sig)
357+ case <-wctx.Done():
358+ if ctx.Err() != nil {
359+ return abortBoot(log, syscall.SIGTERM)
360+ }
361+ log.Warn().Dur("timeout", timeout).Msg("init: the Docker API did not answer in time; starting the executor anyway (jobs using Docker will fail)")
362+ return fmt.Errorf("docker API not ready after %s", timeout)
171363 }
172- _ = syscall.Kill(dockerPid, syscall.SIGKILL)
173364 }
174- return code
365+}
366+
367+func abortBoot(log zerowrap.Logger, sig os.Signal) error {
368+ code := 128 + int(syscall.SIGTERM)
369+ if s, ok := sig.(syscall.Signal); ok {
370+ code = 128 + int(s)
371+ }
372+ log.Warn().Str("signal", fmt.Sprint(sig)).Msg("init: shutdown signal while waiting for the Docker API; not starting the executor, stopping dockerd")
373+ return &abortBootError{code: code}
374+}
375+
376+// dockerPing reports whether GET /_ping answers 200 "OK".
377+func dockerPing(ctx context.Context, client *http.Client) bool {
378+ // One ping is bounded on its own so that a daemon that accepts and stalls
379+ // does not hold the others back; ctx (the readiness deadline) bounds it too.
380+ ctx, cancel := context.WithTimeout(ctx, dockerPingTimeout)
381+ defer cancel()
382+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://docker/_ping", nil)
383+ if err != nil {
384+ return false
385+ }
386+ resp, err := client.Do(req)
387+ if err != nil {
388+ return false
389+ }
390+ defer func() { _ = resp.Body.Close() }()
391+ body, _ := io.ReadAll(io.LimitReader(resp.Body, 16))
392+ return resp.StatusCode == http.StatusOK && strings.TrimSpace(string(body)) == "OK"
175393 }
176394
177395 // spawn starts argv[0] as a child (own process group) without os/exec, so that
@@ -4,7 +4,11 @@ package guest
4 4
5 import (5 import (
6 "context"6 "context"
7+ "errors"
7 "fmt"8 "fmt"
9+ "io"
10+ "net"
11+ "net/http"
8 "os"12 "os"
9 "os/signal"13 "os/signal"
10 "strings"14 "strings"
@@ -33,6 +37,15 @@ type InitConfig struct {
33 // ExecArgs is the executor command line; default37 // ExecArgs is the executor command line; default
34 // [GuestExecDir/GuestExecName exec].38 // [GuestExecDir/GuestExecName exec].
35 ExecArgs []string39 ExecArgs []string
40+ // DockerdArgs replaces the dockerd command line, and the check that
41+ // /usr/bin/dockerd exists (tests); SkipDocker still wins.
42+ DockerdArgs []string
43+ // DockerSocket is the Docker API socket dockerd listens on and init waits
44+ // for; default dockerSocket.
45+ DockerSocket string
46+ // DockerReadyTimeout bounds the wait for the Docker API before the executor
47+ // is started anyway; default dockerReadyTimeout.
48+ DockerReadyTimeout time.Duration
36 }49 }
37 50
38 // Init is `ermite-runner init`: the entrypoint of every sandbox.51 // Init is `ermite-runner init`: the entrypoint of every sandbox.
@@ -45,7 +58,7 @@ type InitConfig struct {
45 //58 //
46 // In a container the root and the pseudo filesystems are ready: it skips all of59 // In a container the root and the pseudo filesystems are ready: it skips all of
47 // that and only prepares cgroups. Then, in both: start dockerd with its data60 // that and only prepares cgroups. Then, in both: start dockerd with its data
48-// root on the scratch space, wait for its socket, run the executor61+// root on the scratch space, wait (bounded) for its API to answer, run the executor
49 // (`ermite-runner exec`) as a child, reap orphans, stop dockerd, and finish by62 // (`ermite-runner exec`) as a child, reap orphans, stop dockerd, and finish by
50 // powering the VM off, or exiting with the executor's code in a container.63 // powering the VM off, or exiting with the executor's code in a container.
51 func Init(ctx context.Context, cfg InitConfig) int {64 func Init(ctx context.Context, cfg InitConfig) int {
@@ -116,10 +129,15 @@ func runPayload(ctx context.Context, cfg InitConfig, mode string, args map[strin
116 }129 }
117 130
118 var dockerPid int131 var dockerPid int
119- if _, err := os.Stat("/usr/bin/dockerd"); err == nil && !cfg.SkipDocker {132+ _, statErr := os.Stat("/usr/bin/dockerd")
133+ if !cfg.SkipDocker && (statErr == nil || cfg.DockerdArgs != nil) {
120 _ = os.MkdirAll(sandbox.GuestScratch+"/docker", 0o711)134 _ = os.MkdirAll(sandbox.GuestScratch+"/docker", 0o711)
121- pid, err := spawn([]string{"/usr/bin/dockerd", "--data-root", sandbox.GuestScratch + "/docker",135+ argv := cfg.DockerdArgs
122- "--storage-driver", "overlay2", "--host", "unix:///var/run/docker.sock"}, nil)136+ if argv == nil {
137+ argv = []string{"/usr/bin/dockerd", "--data-root", sandbox.GuestScratch + "/docker",
138+ "--storage-driver", "overlay2", "--host", "unix:///var/run/docker.sock"}
139+ }
140+ pid, err := spawn(argv, nil)
123 if err != nil {141 if err != nil {
124 log.Error().Err(err).Msg("init: start dockerd")142 log.Error().Err(err).Msg("init: start dockerd")
125 } else {143 } else {
@@ -127,12 +145,6 @@ func runPayload(ctx context.Context, cfg InitConfig, mode string, args map[strin
127 }145 }
128 }146 }
129 147
130- execPid, err := spawn(cfg.ExecArgs, append(os.Environ(),
131- "ERMITE_CONNECT="+target, "ERMITE_SANDBOX="+mode, "ERMITE_VERSION="+cfg.Version))
132- if err != nil {
133- log.Error().Err(err).Msg("init: start executor")
134- return 1
135- }
136 // SIGTERM/SIGINT (docker stop, or the host stopping the VM) go to the148 // SIGTERM/SIGINT (docker stop, or the host stopping the VM) go to the
137 // executor, which cancels its script with the usual escalation and reports149 // executor, which cancels its script with the usual escalation and reports
138 // its Exit; init then finishes normally instead of dying with children.150 // its Exit; init then finishes normally instead of dying with children.
@@ -143,35 +155,241 @@ func runPayload(ctx context.Context, cfg InitConfig, mode string, args map[strin
143 defer signal.Stop(ch)155 defer signal.Stop(ch)
144 sigs = ch156 sigs = ch
145 }157 }
158+
159+ // The first job-container step needs the Docker API, so the executor must
160+ // not start before dockerd answers. If it never does (or dies), start the
161+ // executor anyway: jobs that do not use Docker still run. A shutdown signal
162+ // during the wait aborts the boot instead: see abortBootError.
163+ if dockerPid != 0 {
164+ sock := cfg.DockerSocket
165+ if sock == "" {
166+ sock = dockerSocket
167+ }
168+ timeout := cfg.DockerReadyTimeout
169+ if timeout <= 0 {
170+ timeout = dockerReadyTimeout
171+ }
172+ err := waitDockerReady(ctx, log, sock, dockerPid, timeout, sigs)
173+ var exited *dockerdExitError
174+ var abort *abortBootError
175+ switch {
176+ case errors.As(err, &abort):
177+ stopDockerd(dockerPid)
178+ return abort.code
179+ case errors.As(err, &exited):
180+ dockerPid = 0 // already reaped
181+ }
182+ }
183+
184+ execPid, err := spawn(cfg.ExecArgs, append(os.Environ(),
185+ "ERMITE_CONNECT="+target, "ERMITE_SANDBOX="+mode, "ERMITE_VERSION="+cfg.Version))
186+ if err != nil {
187+ log.Error().Err(err).Msg("init: start executor")
188+ stopDockerd(dockerPid)
189+ return 1
190+ }
191+ forward := func(sig os.Signal) {
192+ if s, ok := sig.(syscall.Signal); ok {
193+ log.Info().Str("signal", s.String()).Msg("init: forwarding the signal to the executor")
194+ _ = syscall.Kill(-execPid, s) // the executor leads its own process group
195+ _ = syscall.Kill(execPid, s)
196+ }
197+ }
146 stopFwd := make(chan struct{})198 stopFwd := make(chan struct{})
147 defer close(stopFwd)199 defer close(stopFwd)
148 go func() {200 go func() {
149 for {201 for {
150 select {202 select {
151 case sig := <-sigs:203 case sig := <-sigs:
152- if s, ok := sig.(syscall.Signal); ok {204+ forward(sig)
153- log.Info().Str("signal", s.String()).Msg("init: forwarding the signal to the executor")
154- _ = syscall.Kill(-execPid, s) // the executor leads its own process group
155- _ = syscall.Kill(execPid, s)
156- }
157 case <-stopFwd:205 case <-stopFwd:
158 return206 return
159 }207 }
160 }208 }
161 }()209 }()
162 code := reapUntil(execPid)210 code := reapUntil(execPid)
163- if dockerPid != 0 {211+ stopDockerd(dockerPid)
164- _ = syscall.Kill(dockerPid, syscall.SIGTERM)212+ return code
165- deadline := time.Now().Add(5 * time.Second)213+}
166- for time.Now().Before(deadline) {214+
167- if pid, _ := wait4NoHang(dockerPid); pid == dockerPid {215+// stopDockerd stops dockerd (SIGTERM, then SIGKILL after 5 s) and reaps it (at
168- break216+// most 5 s more after the SIGKILL); a zero pid is a no-op.
217+func stopDockerd(pid int) {
218+ if pid == 0 {
219+ return
220+ }
221+ _ = syscall.Kill(pid, syscall.SIGTERM)
222+ deadline := time.Now().Add(5 * time.Second)
223+ for time.Now().Before(deadline) {
224+ if got, _ := wait4NoHang(pid); got == pid {
225+ return
226+ }
227+ time.Sleep(100 * time.Millisecond)
228+ }
229+ _ = syscall.Kill(pid, syscall.SIGKILL)
230+ // SIGKILL cannot be ignored, so the exit is prompt; poll rather than block
231+ // for ever on a process stuck in the kernel (uninterruptible sleep).
232+ for end := time.Now().Add(5 * time.Second); time.Now().Before(end); time.Sleep(10 * time.Millisecond) {
233+ got, err := wait4NoHang(pid)
234+ if got == pid || (err != nil && err != syscall.EINTR) { // ECHILD: already reaped
235+ return
236+ }
237+ }
238+}
239+
240+const (
241+ // dockerSocket is the Docker API socket of the nested dockerd.
242+ dockerSocket = "/var/run/docker.sock"
243+ // dockerReadyTimeout bounds the wait for the Docker API at boot.
244+ dockerReadyTimeout = 60 * time.Second
245+ dockerPollInterval = 100 * time.Millisecond
246+ dockerPingTimeout = 2 * time.Second
247+)
248+
249+// dockerdExitError reports that dockerd exited (and was reaped) before its API
250+// answered.
251+type dockerdExitError struct{ status syscall.WaitStatus }
252+
253+func (e *dockerdExitError) Error() string {
254+ if e.status.Signaled() {
255+ return "dockerd killed by signal " + e.status.Signal().String()
256+ }
257+ return fmt.Sprintf("dockerd exited with code %d", e.status.ExitStatus())
258+}
259+
260+// abortBootError means a shutdown signal (or the cancelled context) arrived
261+// while waiting for the Docker API: init must not start the executor, stop
262+// dockerd and exit with code (128+signal, as for a killed executor).
263+//
264+// Not starting the executor is safe for the host: the sandbox ends before the
265+// executor connected ("container exited ... before the executor connected" /
266+// "firecracker exited before the executor connected"), and the host reports the
267+// job cancelled (not failed) when a cancel or shutdown was requested
268+// (jobRun.execute: startSandbox error with cancelRequested). A stop that the
269+// host did not ask for is reported as "the sandbox did not start".
270+type abortBootError struct{ code int }
271+
272+func (e *abortBootError) Error() string {
273+ return fmt.Sprintf("boot aborted by a shutdown signal (exit %d)", e.code)
274+}
275+
276+// reapChildren reaps every child that has exited (orphans adopted by init
277+// included) without blocking, and reports whether pid was among them, with its
278+// wait status.
279+func reapChildren(pid int) (bool, syscall.WaitStatus) {
280+ var found bool
281+ var status syscall.WaitStatus
282+ for {
283+ var ws syscall.WaitStatus
284+ got, err := syscall.Wait4(-1, &ws, syscall.WNOHANG, nil)
285+ if err == syscall.EINTR {
286+ continue
287+ }
288+ if err != nil || got <= 0 {
289+ return found, status
290+ }
291+ if got == pid {
292+ found, status = true, ws
293+ }
294+ }
295+}
296+
297+// waitDockerReady blocks until the Docker API on sock answers GET /_ping with
298+// "OK", for at most timeout. It returns nil when ready, and otherwise:
299+// - *dockerdExitError: dockerd (pid, a child of init) exited and was reaped;
300+// - a plain error: the deadline passed;
301+// - *abortBootError: a signal arrived on sigs or ctx was cancelled.
302+//
303+// The orphans that init adopts are reaped on every tick, and a ping in flight
304+// never delays the deadline, the signals or the exit check. Every outcome is
305+// logged; for the first two the caller starts the executor anyway.
306+func waitDockerReady(ctx context.Context, log zerowrap.Logger, sock string, pid int, timeout time.Duration, sigs <-chan os.Signal) error {
307+ start := time.Now()
308+ log.Info().Str("socket", sock).Dur("timeout", timeout).Msg("init: waiting for the Docker API")
309+ wctx, cancel := context.WithTimeout(ctx, timeout)
310+ defer cancel() // also aborts a ping in flight
311+ client := &http.Client{Transport: &http.Transport{
312+ DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
313+ var d net.Dialer
314+ return d.DialContext(ctx, "unix", sock)
315+ },
316+ DisableKeepAlives: true,
317+ }}
318+ defer client.CloseIdleConnections()
319+ ticker := time.NewTicker(dockerPollInterval)
320+ defer ticker.Stop()
321+
322+ var pinging <-chan bool // result of the ping in flight, nil if none
323+ startPing := func() {
324+ ch := make(chan bool, 1)
325+ pinging = ch
326+ go func() { ch <- dockerPing(wctx, client) }()
327+ }
328+ checkExit := func() error {
329+ if ok, ws := reapChildren(pid); ok {
330+ err := &dockerdExitError{status: ws}
331+ log.Warn().Err(err).Msg("init: dockerd died before its API was ready; starting the executor anyway (jobs using Docker will fail)")
332+ return err
333+ }
334+ return nil
335+ }
336+ if err := checkExit(); err != nil {
337+ return err
338+ }
339+ startPing()
340+ for {
341+ select {
342+ case ok := <-pinging:
343+ pinging = nil
344+ if ok {
345+ log.Info().Dur("after", time.Since(start).Round(time.Millisecond)).Msg("init: the Docker API is ready")
346+ return nil
347+ }
348+ case <-ticker.C:
349+ if err := checkExit(); err != nil {
350+ return err
351+ }
352+ if pinging == nil {
353+ startPing()
169 }354 }
170- time.Sleep(100 * time.Millisecond)355+ case sig := <-sigs:
356+ return abortBoot(log, sig)
357+ case <-wctx.Done():
358+ if ctx.Err() != nil {
359+ return abortBoot(log, syscall.SIGTERM)
360+ }
361+ log.Warn().Dur("timeout", timeout).Msg("init: the Docker API did not answer in time; starting the executor anyway (jobs using Docker will fail)")
362+ return fmt.Errorf("docker API not ready after %s", timeout)
171 }363 }
172- _ = syscall.Kill(dockerPid, syscall.SIGKILL)
173 }364 }
174- return code365+}
366+
367+func abortBoot(log zerowrap.Logger, sig os.Signal) error {
368+ code := 128 + int(syscall.SIGTERM)
369+ if s, ok := sig.(syscall.Signal); ok {
370+ code = 128 + int(s)
371+ }
372+ log.Warn().Str("signal", fmt.Sprint(sig)).Msg("init: shutdown signal while waiting for the Docker API; not starting the executor, stopping dockerd")
373+ return &abortBootError{code: code}
374+}
375+
376+// dockerPing reports whether GET /_ping answers 200 "OK".
377+func dockerPing(ctx context.Context, client *http.Client) bool {
378+ // One ping is bounded on its own so that a daemon that accepts and stalls
379+ // does not hold the others back; ctx (the readiness deadline) bounds it too.
380+ ctx, cancel := context.WithTimeout(ctx, dockerPingTimeout)
381+ defer cancel()
382+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://docker/_ping", nil)
383+ if err != nil {
384+ return false
385+ }
386+ resp, err := client.Do(req)
387+ if err != nil {
388+ return false
389+ }
390+ defer func() { _ = resp.Body.Close() }()
391+ body, _ := io.ReadAll(io.LimitReader(resp.Body, 16))
392+ return resp.StatusCode == http.StatusOK && strings.TrimSpace(string(body)) == "OK"
175 }393 }
176 394
177 // spawn starts argv[0] as a child (own process group) without os/exec, so that395 // spawn starts argv[0] as a child (own process group) without os/exec, so that

Keyboard shortcuts

Single-key shortcuts are on. ctrl k and esc work whatever this says.

ctrl k :
go to a section, branch, tag, file or repository
?
this list
j k
move the row cursor down / up
enter
open the row under the cursor; on Files, a directory opens or closes in place
↑ ↓ ← → home end
in the file tree (focus on a row): previous / next row, close / open a directory or go to its parent / first entry, first / last row. They never open a file
l h
on Files: open a directory or a file / close a directory or go up to its parent, then to the parent page. Elsewhere: into the side panel (on a merge request, its files) / back to the content; there j k move between its controls
g g G
first / last row
/
filter the list; on Files, find a file anywhere in the repository (esc clears, enter opens the best match)
b
switch branch or tag
g s g f g c g b g m g i g r g a
tabs: summary, files, commits, branches & tags, merge requests, issues, releases, actions (when available)
g h
home
w
wide mode: use the whole width of the screen, or not (remembered)
y
copy a permalink pinned to the commit
d u x
diff: split / unified / collapse or expand all
e E
diff: the whole file in view / every file, with the lines the hunks hide
n
merge request: next note in the diff
esc
close a dialog, leave a field

Branches 4

Tags 11

↑↓ moveenter openesc close