ca0f9b5 test(sandbox): find VMMs by id in the integration audit, emergency reap on failure, separate namespaces for the firewall test · ermite-runner

M internal/sandbox/integration/integration_test.go +37 -5


@@ -207,8 +207,17 @@ func leftovers(stateDir string) []string {
207	207		return nil
208	208		})
209	209		}
210		-	// Any process whose command line carries one of our VM ids, whatever the
211		-	// program is called (the same rule the driver's Reap uses).
	210	+	for pid, id := range vmmProcs() {
	211	+	left = append(left, fmt.Sprintf("process %d %s", pid, id))
	212	+	}
	213	+	return left
	214	+	}
	215	+
	216	+	// vmmProcs returns the processes whose command line carries one of our VM ids
	217	+	// (and --api-sock or --exec-file), whatever the program is called: the rule the
	218	+	// driver's Reap uses.
	219	+	func vmmProcs() map[int]string {
	220	+	out := map[int]string{}
212	221		procs, _ := filepath.Glob("/proc/[0-9]*/cmdline")
213	222		for _, p := range procs {
214	223		b, err := os.ReadFile(p)

@@ -228,11 +237,23 @@ func leftovers(stateDir string) []string {
228	237		marker = true
229	238		}
230	239		}
231		-	if id != "" && marker {
232		-	left = append(left, "process "+p+" "+id)
	240	+	if id == "" \|\| !marker {
	241	+	continue
	242	+	}
	243	+	var pid int
	244	+	if _, err := fmt.Sscanf(p, "/proc/%d/cmdline", &pid); err == nil {
	245	+	out[pid] = id
233	246		}
234	247		}
235		-	return left
	248	+	return out
	249	+	}
	250	+
	251	+	func vmmPIDs() []int {
	252	+	var pids []int
	253	+	for pid := range vmmProcs() {
	254	+	pids = append(pids, pid)
	255	+	}
	256	+	return pids
236	257		}
237	258
238	259		func checkStubOutput(t *testing.T, r result, wantSandbox string) {

@@ -262,6 +283,17 @@ func scenario(t *testing.T, ctx context.Context, cfg sandbox.Config, mode sandbo
262	283		if drv.Mode() != mode {
263	284		t.Fatalf("driver mode %s, want %s", drv.Mode(), mode)
264	285		}
	286	+	// Whatever happens below (a Fatal included), leave nothing behind on the host.
	287	+	t.Cleanup(func() {
	288	+	_ = drv.Close()
	289	+	d2, err := sandbox.Select(context.Background(), cfg, drivers.Factories(), sandbox.Prober{})
	290	+	if err != nil {
	291	+	t.Logf("emergency cleanup: %v", err)
	292	+	return
	293	+	}
	294	+	_ = d2.Reap(context.Background())
	295	+	_ = d2.Close()
	296	+	})
265	297		if err := drv.Reap(ctx); err != nil {
266	298		t.Fatalf("initial reap: %v", err)
267	299		}


@@ -207,8 +207,17 @@ func leftovers(stateDir string) []string {
207		return nil	207		return nil
208		})	208		})
209		}	209		}
210	-	// Any process whose command line carries one of our VM ids, whatever the	210	+	for pid, id := range vmmProcs() {
211	-	// program is called (the same rule the driver's Reap uses).	211	+	left = append(left, fmt.Sprintf("process %d %s", pid, id))
			212	+	}
			213	+	return left
			214	+	}
			215	+
			216	+	// vmmProcs returns the processes whose command line carries one of our VM ids
			217	+	// (and --api-sock or --exec-file), whatever the program is called: the rule the
			218	+	// driver's Reap uses.
			219	+	func vmmProcs() map[int]string {
			220	+	out := map[int]string{}
212		procs, _ := filepath.Glob("/proc/[0-9]*/cmdline")	221		procs, _ := filepath.Glob("/proc/[0-9]*/cmdline")
213		for _, p := range procs {	222		for _, p := range procs {
214		b, err := os.ReadFile(p)	223		b, err := os.ReadFile(p)

@@ -228,11 +237,23 @@ func leftovers(stateDir string) []string {
228		marker = true	237		marker = true
229		}	238		}
230		}	239		}
231	-	if id != "" && marker {	240	+	if id == "" \|\| !marker {
232	-	left = append(left, "process "+p+" "+id)	241	+	continue
			242	+	}
			243	+	var pid int
			244	+	if _, err := fmt.Sscanf(p, "/proc/%d/cmdline", &pid); err == nil {
			245	+	out[pid] = id
233		}	246		}
234		}	247		}
235	-	return left	248	+	return out
			249	+	}
			250	+
			251	+	func vmmPIDs() []int {
			252	+	var pids []int
			253	+	for pid := range vmmProcs() {
			254	+	pids = append(pids, pid)
			255	+	}
			256	+	return pids
236		}	257		}
237			258
238		func checkStubOutput(t *testing.T, r result, wantSandbox string) {	259		func checkStubOutput(t *testing.T, r result, wantSandbox string) {

@@ -262,6 +283,17 @@ func scenario(t *testing.T, ctx context.Context, cfg sandbox.Config, mode sandbo
262		if drv.Mode() != mode {	283		if drv.Mode() != mode {
263		t.Fatalf("driver mode %s, want %s", drv.Mode(), mode)	284		t.Fatalf("driver mode %s, want %s", drv.Mode(), mode)
264		}	285		}
			286	+	// Whatever happens below (a Fatal included), leave nothing behind on the host.
			287	+	t.Cleanup(func() {
			288	+	_ = drv.Close()
			289	+	d2, err := sandbox.Select(context.Background(), cfg, drivers.Factories(), sandbox.Prober{})
			290	+	if err != nil {
			291	+	t.Logf("emergency cleanup: %v", err)
			292	+	return
			293	+	}
			294	+	_ = d2.Reap(context.Background())
			295	+	_ = d2.Close()
			296	+	})
265		if err := drv.Reap(ctx); err != nil {	297		if err := drv.Reap(ctx); err != nil {
266		t.Fatalf("initial reap: %v", err)	298		t.Fatalf("initial reap: %v", err)
267		}	299		}

M internal/sandbox/integration/vm_test.go +5 -10


@@ -5,7 +5,6 @@ package integration
5	5		import (
6	6		"context"
7	7		"os"
8		-	"os/exec"
9	8		"strings"
10	9		"syscall"
11	10		"testing"

@@ -27,16 +26,12 @@ func TestVMStubExecutor(t *testing.T) {
27	26		}
28	27
29	28		scenario(t, ctx, cfg, sandbox.VM, func(t *testing.T, sb sandbox.Sandbox) {
30		-	// Kill the VMM process of the only running VM.
31		-	out, err := exec.Command("pgrep", "-f", "--api-sock .--id er\|--id er[0-9a-f]{8} .--api-sock\|--id er[0-9a-f]{8} --api-sock").Output()
32		-	if err != nil {
33		-	t.Fatalf("no firecracker to kill: %v", err)
	29	+	// Kill every VMM process of ours (found like Reap finds them: by id).
	30	+	pids := vmmPIDs()
	31	+	if len(pids) == 0 {
	32	+	t.Fatal("no VMM process to kill")
34	33		}
35		-	for _, f := range strings.Fields(string(out)) {
36		-	pid := 0
37		-	for _, c := range f {
38		-	pid = pid*10 + int(c-'0')
39		-	}
	34	+	for _, pid := range pids {
40	35		_ = syscall.Kill(pid, syscall.SIGKILL)
41	36		}
42	37		})

 import (
 	"context"
 	"os"
-	"os/exec"
 	"strings"
 	"syscall"
 	"testing"
 	}
 	scenario(t, ctx, cfg, sandbox.VM, func(t *testing.T, sb sandbox.Sandbox) {
-		// Kill the VMM process of the only running VM.
+		// Kill every VMM process of ours (found like Reap finds them: by id).
-		out, err := exec.Command("pgrep", "-f", "--api-sock .*--id er|--id er[0-9a-f]{8} .*--api-sock|--id er[0-9a-f]{8} --api-sock").Output()
+		pids := vmmPIDs()
-		if err != nil {
+		if len(pids) == 0 {
-			t.Fatalf("no firecracker to kill: %v", err)
+			t.Fatal("no VMM process to kill")
 		}
-		for _, f := range strings.Fields(string(out)) {
+		for _, pid := range pids {
-			pid := 0
-			for _, c := range f {
-				pid = pid*10 + int(c-'0')
-			}
 			_ = syscall.Kill(pid, syscall.SIGKILL)
 		}
 	})

M scripts/it-vm-userns.sh +12 -7


@@ -19,7 +19,10 @@ cd "$root"
19	19		fc=${FIRECRACKER:-$(command -v firecracker \|\| true)}
20	20		[ -x "${fc:-}" ] \|\| { echo "firecracker not found: set FIRECRACKER" >&2; exit 1; }
21	21		img=${IMAGE_DIR:-$root/tmp/image}
22		-	exec unshare --user --map-root-user --net bash -c '
	22	+	# Two separate namespaces: the VM tests leave TAP devices behind when they fail,
	23	+	# and the packet-level firewall test must start from a clean network stack.
	24	+	rc=0
	25	+	unshare --user --map-root-user --net bash -c '
23	26		ip link set lo up
24	27		# A default route, so that forwarded packets reach the firewall instead of
25	28		# being answered "network unreachable" by the routing table: the drops the

@@ -33,9 +36,11 @@ exec unshare --user --map-root-user --net bash -c '
33	36		ERMITE_RUNNER_FIRECRACKER="$2" ERMITE_RUNNER_JAILER_MODE=off \
34	37		ERMITE_RUNNER_LOCK_FILE="${TMPDIR:-/tmp}/ermite-runner-it-$$.lock"
35	38		shift 2
36		-	rc=0
37		-	go test -count=1 -v -timeout 10m ./internal/sandbox/integration "$@" \|\| rc=$?
38		-	# The packet-level firewall test needs only the namespace, not Firecracker.
39		-	ERMITE_RUNNER_IT_NET=1 go test -count=1 -v -timeout 10m -run TwoGuests ./internal/netpolicy \|\| rc=$?
40		-	exit $rc
41		-	' _ "$img" "$fc" "$@"
	39	+	exec go test -count=1 -v -timeout 10m ./internal/sandbox/integration "$@"
	40	+	' _ "$img" "$fc" "$@" \|\| rc=$?
	41	+	# The packet-level firewall test needs only a namespace, not Firecracker.
	42	+	unshare --user --map-root-user --net bash -c '
	43	+	ip link set lo up
	44	+	GOWORK=off ERMITE_RUNNER_IT_NET=1 exec go test -count=1 -v -timeout 10m -run TwoGuests ./internal/netpolicy
	45	+	' \|\| rc=$?
	46	+	exit $rc

 fc=${FIRECRACKER:-$(command -v firecracker || true)}
 [ -x "${fc:-}" ] || { echo "firecracker not found: set FIRECRACKER" >&2; exit 1; }
 img=${IMAGE_DIR:-$root/tmp/image}
-exec unshare --user --map-root-user --net bash -c '
+# Two separate namespaces: the VM tests leave TAP devices behind when they fail,
+# and the packet-level firewall test must start from a clean network stack.
+rc=0
+unshare --user --map-root-user --net bash -c '
   ip link set lo up
   # A default route, so that forwarded packets reach the firewall instead of
   # being answered "network unreachable" by the routing table: the drops the
     ERMITE_RUNNER_FIRECRACKER="$2" ERMITE_RUNNER_JAILER_MODE=off \
     ERMITE_RUNNER_LOCK_FILE="${TMPDIR:-/tmp}/ermite-runner-it-$$.lock"
   shift 2
-  rc=0
+  exec go test -count=1 -v -timeout 10m ./internal/sandbox/integration "$@"
-  go test -count=1 -v -timeout 10m ./internal/sandbox/integration "$@" || rc=$?
+' _ "$img" "$fc" "$@" || rc=$?
-  # The packet-level firewall test needs only the namespace, not Firecracker.
+# The packet-level firewall test needs only a namespace, not Firecracker.
-  ERMITE_RUNNER_IT_NET=1 go test -count=1 -v -timeout 10m -run TwoGuests ./internal/netpolicy || rc=$?
+unshare --user --map-root-user --net bash -c '
-  exit $rc
+  ip link set lo up
-' _ "$img" "$fc" "$@"
+  GOWORK=off ERMITE_RUNNER_IT_NET=1 exec go test -count=1 -v -timeout 10m -run TwoGuests ./internal/netpolicy
+' || rc=$?
+exit $rc