detached 1e5d59c

test(resilience): run harness Python tests and cover the SSH wrapper

make test runs the Python unit tests; cover readiness bounds, port rewrite and passthrough, a UDP route opened during a blackout, a longer build timeout, and a negative control that requires the compositor to keep capturing.
commit
1e5d59ca5420b81068f07d99797d5a8f05c5908e
author
brice · · 2026-10-07 07:01 UTC
parent
0d9258e
signature
GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email
4 files · +80 -3
Unified
UnifiedSplit
M Makefile +1 -0
@@ -9,6 +9,7 @@ install:
99
1010 test:
1111 go test ./... -race
12+ python3 -m unittest discover -s scripts/remote-resilience-harness -p '*_test.py'
1213
1314 test-installer:
1415 sh scripts/install_platform_test.sh
@@ -9,6 +9,7 @@ install:
9 9
10 test:10 test:
11 go test ./... -race11 go test ./... -race
12+ python3 -m unittest discover -s scripts/remote-resilience-harness -p '*_test.py'
12 13
13 test-installer:14 test-installer:
14 sh scripts/install_platform_test.sh15 sh scripts/install_platform_test.sh
M scripts/remote-resilience-harness/relay/main_test.go +31 -0
@@ -106,6 +106,37 @@ func TestTCPRelayBlackoutHoldsBytes(t *testing.T) {
106106 <-done
107107 }
108108
109+// TestRelayUDPProxyInheritsActiveBlackout opens a UDP route while a blackout
110+// is active and proves it drops datagrams until the blackout ends.
111+func TestRelayUDPProxyInheritsActiveBlackout(t *testing.T) {
112+ ctx, cancel := context.WithCancel(t.Context())
113+ defer cancel()
114+ r := &relay{ctx: ctx, cancel: cancel, remote: "127.0.0.1"}
115+ defer r.close()
116+ upstream, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1)})
117+ require.NoError(t, err)
118+ defer upstream.Close()
119+ _, err = r.apply(command{Op: "blackout", UDP: true, Active: true})
120+ require.NoError(t, err)
121+ port, err := r.apply(command{Op: "udp", Port: upstream.LocalAddr().(*net.UDPAddr).Port})
122+ require.NoError(t, err)
123+ client, err := net.DialUDP("udp", nil, &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: port.(int)})
124+ require.NoError(t, err)
125+ defer client.Close()
126+ _, err = client.Write([]byte("blocked"))
127+ require.NoError(t, err)
128+ require.Eventually(t, func() bool { return r.proxies[0].Stats().ToServer.BlackoutDrops == 1 }, time.Second, time.Millisecond)
129+ _, err = r.apply(command{Op: "blackout", UDP: true, Active: false})
130+ require.NoError(t, err)
131+ _, err = client.Write([]byte("open"))
132+ require.NoError(t, err)
133+ require.NoError(t, upstream.SetReadDeadline(time.Now().Add(time.Second)))
134+ data := make([]byte, 16)
135+ n, _, err := upstream.ReadFromUDP(data)
136+ require.NoError(t, err)
137+ require.Equal(t, "open", string(data[:n]))
138+}
139+
109140 func TestRelayDisconnectClosesConnections(t *testing.T) {
110141 ctx, cancel := context.WithCancel(t.Context())
111142 defer cancel()
@@ -106,6 +106,37 @@ func TestTCPRelayBlackoutHoldsBytes(t *testing.T) {
106 <-done106 <-done
107 }107 }
108 108
109+// TestRelayUDPProxyInheritsActiveBlackout opens a UDP route while a blackout
110+// is active and proves it drops datagrams until the blackout ends.
111+func TestRelayUDPProxyInheritsActiveBlackout(t *testing.T) {
112+ ctx, cancel := context.WithCancel(t.Context())
113+ defer cancel()
114+ r := &relay{ctx: ctx, cancel: cancel, remote: "127.0.0.1"}
115+ defer r.close()
116+ upstream, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1)})
117+ require.NoError(t, err)
118+ defer upstream.Close()
119+ _, err = r.apply(command{Op: "blackout", UDP: true, Active: true})
120+ require.NoError(t, err)
121+ port, err := r.apply(command{Op: "udp", Port: upstream.LocalAddr().(*net.UDPAddr).Port})
122+ require.NoError(t, err)
123+ client, err := net.DialUDP("udp", nil, &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: port.(int)})
124+ require.NoError(t, err)
125+ defer client.Close()
126+ _, err = client.Write([]byte("blocked"))
127+ require.NoError(t, err)
128+ require.Eventually(t, func() bool { return r.proxies[0].Stats().ToServer.BlackoutDrops == 1 }, time.Second, time.Millisecond)
129+ _, err = r.apply(command{Op: "blackout", UDP: true, Active: false})
130+ require.NoError(t, err)
131+ _, err = client.Write([]byte("open"))
132+ require.NoError(t, err)
133+ require.NoError(t, upstream.SetReadDeadline(time.Now().Add(time.Second)))
134+ data := make([]byte, 16)
135+ n, _, err := upstream.ReadFromUDP(data)
136+ require.NoError(t, err)
137+ require.Equal(t, "open", string(data[:n]))
138+}
139+
109 func TestRelayDisconnectClosesConnections(t *testing.T) {140 func TestRelayDisconnectClosesConnections(t *testing.T) {
110 ctx, cancel := context.WithCancel(t.Context())141 ctx, cancel := context.WithCancel(t.Context())
111 defer cancel()142 defer cancel()
M scripts/remote-resilience-harness/ssh_test.py +37 -0
@@ -26,5 +26,42 @@ class ControlTests(unittest.TestCase):
2626 self.assertEqual(wrapper.control({"id": 1, "op": "udp", "port": 1234}), 4321)
2727
2828
29+class MainTests(unittest.TestCase):
30+ BOOTSTRAP = ["-T", "--", "remote", "'vev' '_broker-mux-quic-bootstrap'"]
31+
32+ def run_main(self, readiness, rewritten=4321):
33+ process = MagicMock()
34+ process.stdout.readline.return_value = readiness
35+ process.stdout.read1.return_value = b""
36+ process.wait.return_value = 0
37+ process.poll.return_value = 0
38+ out = MagicMock()
39+ with patch.object(wrapper.sys, "argv", ["ssh"] + self.BOOTSTRAP), \
40+ patch.object(wrapper.subprocess, "Popen", return_value=process), \
41+ patch.object(wrapper, "control", return_value=rewritten) as control, \
42+ patch.object(wrapper.sys, "stdout", MagicMock(buffer=out)):
43+ code = wrapper.main()
44+ return code, control, out
45+
46+ def test_rewrites_only_the_port(self):
47+ code, control, out = self.run_main(b'{"port":1234,"fingerprint":"f"}\n')
48+ self.assertEqual(code, 0)
49+ control.assert_called_once_with({"id": 1, "op": "udp", "port": 1234})
50+ self.assertEqual(out.write.call_args_list[0].args[0], b'{"port":4321,"fingerprint":"f"}\n')
51+
52+ def test_rejects_invalid_readiness(self):
53+ for line in (b'{"port":1}', b"x" * 4097 + b"\n", b'{"port":0}\n', b'{"port":65536}\n', b'{"port":"22"}\n'):
54+ with self.subTest(line=line[:20]):
55+ with self.assertRaises(RuntimeError):
56+ self.run_main(line)
57+
58+ def test_other_commands_exec_ssh_unchanged(self):
59+ with patch.object(wrapper.sys, "argv", ["ssh", "remote", "true"]), \
60+ patch.object(wrapper.os, "execv", side_effect=SystemExit(0)) as execv:
61+ with self.assertRaises(SystemExit):
62+ wrapper.main()
63+ execv.assert_called_once_with("/usr/bin/ssh-real", ["/usr/bin/ssh-real", "-p", "2222", "-o", "HostKeyAlias=remote", "remote", "true"])
64+
65+
2966 if __name__ == "__main__":
3067 unittest.main()
@@ -26,5 +26,42 @@ class ControlTests(unittest.TestCase):
26 self.assertEqual(wrapper.control({"id": 1, "op": "udp", "port": 1234}), 4321)26 self.assertEqual(wrapper.control({"id": 1, "op": "udp", "port": 1234}), 4321)
27 27
28 28
29+class MainTests(unittest.TestCase):
30+ BOOTSTRAP = ["-T", "--", "remote", "'vev' '_broker-mux-quic-bootstrap'"]
31+
32+ def run_main(self, readiness, rewritten=4321):
33+ process = MagicMock()
34+ process.stdout.readline.return_value = readiness
35+ process.stdout.read1.return_value = b""
36+ process.wait.return_value = 0
37+ process.poll.return_value = 0
38+ out = MagicMock()
39+ with patch.object(wrapper.sys, "argv", ["ssh"] + self.BOOTSTRAP), \
40+ patch.object(wrapper.subprocess, "Popen", return_value=process), \
41+ patch.object(wrapper, "control", return_value=rewritten) as control, \
42+ patch.object(wrapper.sys, "stdout", MagicMock(buffer=out)):
43+ code = wrapper.main()
44+ return code, control, out
45+
46+ def test_rewrites_only_the_port(self):
47+ code, control, out = self.run_main(b'{"port":1234,"fingerprint":"f"}\n')
48+ self.assertEqual(code, 0)
49+ control.assert_called_once_with({"id": 1, "op": "udp", "port": 1234})
50+ self.assertEqual(out.write.call_args_list[0].args[0], b'{"port":4321,"fingerprint":"f"}\n')
51+
52+ def test_rejects_invalid_readiness(self):
53+ for line in (b'{"port":1}', b"x" * 4097 + b"\n", b'{"port":0}\n', b'{"port":65536}\n', b'{"port":"22"}\n'):
54+ with self.subTest(line=line[:20]):
55+ with self.assertRaises(RuntimeError):
56+ self.run_main(line)
57+
58+ def test_other_commands_exec_ssh_unchanged(self):
59+ with patch.object(wrapper.sys, "argv", ["ssh", "remote", "true"]), \
60+ patch.object(wrapper.os, "execv", side_effect=SystemExit(0)) as execv:
61+ with self.assertRaises(SystemExit):
62+ wrapper.main()
63+ execv.assert_called_once_with("/usr/bin/ssh-real", ["/usr/bin/ssh-real", "-p", "2222", "-o", "HostKeyAlias=remote", "remote", "true"])
64+
65+
29 if __name__ == "__main__":66 if __name__ == "__main__":
30 unittest.main()67 unittest.main()
M scripts/remote-resilience-harness/visual.py +11 -3
@@ -39,15 +39,15 @@ def main():
3939 for tool in ("docker", "foot", "ssh-keygen", "go"):
4040 if shutil.which(tool) is None:
4141 raise RuntimeError("missing prerequisite: " + tool)
42- run("go", "build", "-o", str(ROOT / "build/vev"), ".", cwd=ROOT)
42+ run("go", "build", "-o", str(ROOT / "build/vev"), ".", cwd=ROOT, timeout=600)
4343 for helper in ("relay", "pixels"):
44- run("go", "build", "-o", str(ROOT / f"build/resilience-{helper}"), f"./scripts/remote-resilience-harness/{helper}", cwd=ROOT)
44+ run("go", "build", "-o", str(ROOT / f"build/resilience-{helper}"), f"./scripts/remote-resilience-harness/{helper}", cwd=ROOT, timeout=600)
4545 nefer = os.environ.get("NEFERWL_BIN")
4646 if not nefer:
4747 nefer = str(private / "neferwl")
4848 common = run("git", "rev-parse", "--path-format=absolute", "--git-common-dir", cwd=ROOT, capture_output=True, text=True).stdout.strip()
4949 source = Path(os.environ.get("NEFERWL_SOURCE", str(Path(common).parent.parent / "neferwl")))
50- run("go", "build", "-o", nefer, "./cmd/neferwl", cwd=source, env={**os.environ, "CGO_ENABLED": "0"})
50+ run("go", "build", "-o", nefer, "./cmd/neferwl", cwd=source, env={**os.environ, "CGO_ENABLED": "0"}, timeout=600)
5151 docker("build", "-f", str(ROOT / "scripts/demo/Dockerfile"), "-t", name, str(ROOT), timeout=600)
5252 docker("network", "create", name)
5353 network = True
@@ -258,7 +258,15 @@ def main():
258258 frozen = artifacts / "frozen.png"
259259 shutil.copyfile(shots / "latest.png", frozen)
260260 try:
261+ frozen_at = (shots / "latest.png").stat().st_mtime_ns
261262 key("key Alt+space")
263+ # The compositor must keep capturing, or an unchanged frame
264+ # proves nothing about the stopped client.
265+ deadline = time.monotonic() + 3
266+ while (shots / "latest.png").stat().st_mtime_ns == frozen_at:
267+ if time.monotonic() > deadline:
268+ raise RuntimeError("compositor stopped capturing during the negative control")
269+ time.sleep(0.05)
262270 time.sleep(0.3)
263271 check = subprocess.run([str(ROOT / "build/resilience-pixels"), str(frozen), str(shots / "latest.png")], capture_output=True, text=True)
264272 assert check.returncode == 1, "visual gate accepted a stopped client"
@@ -39,15 +39,15 @@ def main():
39 for tool in ("docker", "foot", "ssh-keygen", "go"):39 for tool in ("docker", "foot", "ssh-keygen", "go"):
40 if shutil.which(tool) is None:40 if shutil.which(tool) is None:
41 raise RuntimeError("missing prerequisite: " + tool)41 raise RuntimeError("missing prerequisite: " + tool)
42- run("go", "build", "-o", str(ROOT / "build/vev"), ".", cwd=ROOT)42+ run("go", "build", "-o", str(ROOT / "build/vev"), ".", cwd=ROOT, timeout=600)
43 for helper in ("relay", "pixels"):43 for helper in ("relay", "pixels"):
44- run("go", "build", "-o", str(ROOT / f"build/resilience-{helper}"), f"./scripts/remote-resilience-harness/{helper}", cwd=ROOT)44+ run("go", "build", "-o", str(ROOT / f"build/resilience-{helper}"), f"./scripts/remote-resilience-harness/{helper}", cwd=ROOT, timeout=600)
45 nefer = os.environ.get("NEFERWL_BIN")45 nefer = os.environ.get("NEFERWL_BIN")
46 if not nefer:46 if not nefer:
47 nefer = str(private / "neferwl")47 nefer = str(private / "neferwl")
48 common = run("git", "rev-parse", "--path-format=absolute", "--git-common-dir", cwd=ROOT, capture_output=True, text=True).stdout.strip()48 common = run("git", "rev-parse", "--path-format=absolute", "--git-common-dir", cwd=ROOT, capture_output=True, text=True).stdout.strip()
49 source = Path(os.environ.get("NEFERWL_SOURCE", str(Path(common).parent.parent / "neferwl")))49 source = Path(os.environ.get("NEFERWL_SOURCE", str(Path(common).parent.parent / "neferwl")))
50- run("go", "build", "-o", nefer, "./cmd/neferwl", cwd=source, env={**os.environ, "CGO_ENABLED": "0"})50+ run("go", "build", "-o", nefer, "./cmd/neferwl", cwd=source, env={**os.environ, "CGO_ENABLED": "0"}, timeout=600)
51 docker("build", "-f", str(ROOT / "scripts/demo/Dockerfile"), "-t", name, str(ROOT), timeout=600)51 docker("build", "-f", str(ROOT / "scripts/demo/Dockerfile"), "-t", name, str(ROOT), timeout=600)
52 docker("network", "create", name)52 docker("network", "create", name)
53 network = True53 network = True
@@ -258,7 +258,15 @@ def main():
258 frozen = artifacts / "frozen.png"258 frozen = artifacts / "frozen.png"
259 shutil.copyfile(shots / "latest.png", frozen)259 shutil.copyfile(shots / "latest.png", frozen)
260 try:260 try:
261+ frozen_at = (shots / "latest.png").stat().st_mtime_ns
261 key("key Alt+space")262 key("key Alt+space")
263+ # The compositor must keep capturing, or an unchanged frame
264+ # proves nothing about the stopped client.
265+ deadline = time.monotonic() + 3
266+ while (shots / "latest.png").stat().st_mtime_ns == frozen_at:
267+ if time.monotonic() > deadline:
268+ raise RuntimeError("compositor stopped capturing during the negative control")
269+ time.sleep(0.05)
262 time.sleep(0.3)270 time.sleep(0.3)
263 check = subprocess.run([str(ROOT / "build/resilience-pixels"), str(frozen), str(shots / "latest.png")], capture_output=True, text=True)271 check = subprocess.run([str(ROOT / "build/resilience-pixels"), str(frozen), str(shots / "latest.png")], capture_output=True, text=True)
264 assert check.returncode == 1, "visual gate accepted a stopped client"272 assert check.returncode == 1, "visual gate accepted a stopped client"

Keyboard shortcuts

Single-key shortcuts are on. ctrl k and esc work whatever this says.

ctrl k :
go to a section, branch, tag, file or repository
?
this list
j k
move the row cursor down / up
enter
open the row under the cursor; on Files, a directory opens or closes in place
↑ ↓ ← → home end
in the file tree (focus on a row): previous / next row, close / open a directory or go to its parent / first entry, first / last row. They never open a file
l h
on Files: open a directory or a file / close a directory or go up to its parent, then to the parent page. Elsewhere: into the side panel (on a merge request, its files) / back to the content; there j k move between its controls
g g G
first / last row
/
filter the list; on Files, find a file anywhere in the repository (esc clears, enter opens the best match)
b
switch branch or tag
g s g f g c g b g m g i g r g a
tabs: summary, files, commits, branches & tags, merge requests, issues, releases, actions (when available)
g h
home
w
wide mode: use the whole width of the screen, or not (remembered)
y
copy a permalink pinned to the commit
d u x
diff: split / unified / collapse or expand all
e E
diff: the whole file in view / every file, with the lines the hunks hide
n
merge request: next note in the diff
esc
close a dialog, leave a field

Branches 12

Tags 11

↑↓ moveenter openesc close