1// SPDX-License-Identifier: MIT
2
3package netpolicy
4
5import (
6 "context"
7 "fmt"
8 "os"
9 "os/exec"
10 "strings"
11 "testing"
12 "time"
13
14 "bnema.dev/ermite-runner/internal/cmdrun"
15)
16
17// TestRulesetBetweenTwoGuestsAndAForgedSource loads the real ruleset into the
18// kernel and sends real packets: two guests (ertap0, ertap1) and an outside
19// host hang off veth pairs, each in its own network namespace. It needs
20// CAP_NET_ADMIN, which scripts/it-vm-userns.sh provides without root:
21//
22// ERMITE_RUNNER_IT_NET=1 unshare --user --map-root-user --net go test -run Netns ./internal/netpolicy
23func TestRulesetBetweenTwoGuestsAndAForgedSource(t *testing.T) {
24 if os.Getenv("ERMITE_RUNNER_IT_NET") != "1" {
25 t.Skip("set ERMITE_RUNNER_IT_NET=1 inside a user+network namespace (scripts/it-vm-userns.sh)")
26 }
27 for _, c := range []string{"ip", "nft", "nsenter", "unshare", "python3"} {
28 if _, err := exec.LookPath(c); err != nil {
29 t.Skipf("%s not found", c)
30 }
31 }
32 sh := func(args ...string) string {
33 t.Helper()
34 out, err := exec.Command(args[0], args[1:]...).CombinedOutput()
35 if err != nil {
36 t.Fatalf("%s: %v: %s", strings.Join(args, " "), err, out)
37 }
38 return strings.TrimSpace(string(out))
39 }
40 if out, err := exec.Command("ip", "link", "add", "ertapprobe", "type", "dummy").CombinedOutput(); err != nil {
41 t.Skipf("no CAP_NET_ADMIN: %v: %s", err, out)
42 }
43 sh("ip", "link", "del", "ertapprobe")
44 sh("ip", "link", "set", "lo", "up")
45
46 // Three machines: guest A, guest B, an outside host.
47 pid := map[string]int{}
48 for _, n := range []string{"a", "b", "ext"} {
49 c := exec.Command("unshare", "--net", "sleep", "120")
50 if err := c.Start(); err != nil {
51 t.Fatal(err)
52 }
53 pid[n] = c.Process.Pid
54 t.Cleanup(func() { _ = c.Process.Kill(); _ = c.Wait() })
55 }
56 time.Sleep(300 * time.Millisecond)
57 in := func(n string, args ...string) string {
58 t.Helper()
59 return sh(append([]string{"nsenter", fmt.Sprintf("--net=/proc/%d/ns/net", pid[n])}, args...)...)
60 }
61 link := func(host, n string) {
62 sh("ip", "link", "add", host, "type", "veth", "peer", "name", "v0", "netns", fmt.Sprint(pid[n]))
63 in(n, "ip", "link", "set", "lo", "up")
64 in(n, "ip", "link", "set", "v0", "up")
65 sh("sysctl", "-qw", "net.ipv4.conf."+host+".rp_filter=0")
66 }
67 link("ertap0", "a")
68 link("ertap1", "b")
69 link("extif", "ext")
70 sh("sysctl", "-qw", "net.ipv4.conf.all.rp_filter=0")
71 sh("sysctl", "-qw", "net.ipv4.ip_forward=1")
72
73 gA := guest(t, "ertap0", 0) // 10.200.0.1 / .2
74 gB := guest(t, "ertap1", 1) // 10.200.0.5 / .6
75 gA.ForgePort = 8088 // guest A has the guest forge exception, guest B has not
76 for _, g := range []Guest{gA, gB} {
77 sh("ip", "addr", "add", g.Gateway.String()+"/30", "dev", g.Iface)
78 sh("ip", "link", "set", g.Iface, "up")
79 }
80 sh("ip", "addr", "add", "203.0.114.1/24", "dev", "extif")
81 sh("ip", "link", "set", "extif", "up")
82 in("a", "ip", "addr", "add", gA.Addr.String()+"/30", "dev", "v0")
83 in("a", "ip", "route", "add", "default", "via", gA.Gateway.String())
84 in("b", "ip", "addr", "add", gB.Addr.String()+"/30", "dev", "v0")
85 in("b", "ip", "route", "add", "default", "via", gB.Gateway.String())
86 in("ext", "ip", "addr", "add", "203.0.114.2/24", "dev", "v0")
87 in("ext", "ip", "route", "add", "default", "via", "203.0.114.1")
88 // Guest A holds guest B's address too, to be able to forge it.
89 in("a", "ip", "addr", "add", gB.Addr.String()+"/32", "dev", "lo")
90
91 m := NewManager(cmdrun.Exec{}, "nft")
92 ctx := context.Background()
93 t.Cleanup(func() { _ = m.Reset(ctx) })
94 for _, g := range []Guest{gA, gB} {
95 if err := m.Add(ctx, g); err != nil {
96 t.Fatal(err)
97 }
98 }
99
100 // An outside UDP server: logs each payload with the source it saw, and
101 // answers "pong".
102 logf := t.TempDir() + "/ext.log"
103 const server = `
104import socket,sys
105s=socket.socket(socket.AF_INET,socket.SOCK_DGRAM); s.bind(("0.0.0.0",9999))
106f=open(sys.argv[1],"a",buffering=1)
107while True:
108 d,a=s.recvfrom(100); f.write(d.decode()+" from "+a[0]+"\n"); s.sendto(b"pong",a)
109`
110 srv := exec.Command("nsenter", fmt.Sprintf("--net=/proc/%d/ns/net", pid["ext"]), "python3", "-c", server, logf)
111 if err := srv.Start(); err != nil {
112 t.Fatal(err)
113 }
114 t.Cleanup(func() { _ = srv.Process.Kill(); _ = srv.Wait() })
115 time.Sleep(500 * time.Millisecond)
116
117 // udp sends one datagram from machine n (optionally bound to src:sport) and
118 // reports whether an answer came back.
119 const client = `
120import socket,sys
121dst,payload,src,sport=sys.argv[1],sys.argv[2],sys.argv[3],int(sys.argv[4])
122s=socket.socket(socket.AF_INET,socket.SOCK_DGRAM); s.settimeout(2)
123if src: s.bind((src,sport))
124s.sendto(payload.encode(),(dst,int(sys.argv[5]) if len(sys.argv)>5 else 9999))
125try: s.recv(10); print("answer")
126except Exception: print("silence")
127`
128 udp := func(n, dst, payload, src string, sport int, dport int) string {
129 return in(n, "python3", "-c", client, dst, payload, src, fmt.Sprint(sport), fmt.Sprint(dport))
130 }
131 seen := func() string { b, _ := os.ReadFile(logf); return string(b) }
132
133 // Control: a guest reaches the outside host and gets the reply (NAT and the
134 // established path work, so the drops below are the policy, not a broken rig).
135 if got := udp("a", "203.0.114.2", "ctl-a", "", 0, 9999); got != "answer" {
136 t.Fatalf("control: guest A cannot reach the outside host: %s", got)
137 }
138
139 // Guests cannot reach each other, directly or through the other's gateway.
140 if got := udp("a", gB.Addr.String(), "a-to-b", "", 0, 9999); got != "silence" {
141 t.Errorf("guest A reached guest B: %s", got)
142 }
143 // Forged reply (S1): B opens a flow to the outside host from port 5555 and
144 // keeps listening. Guest A then forges a reply of that flow: source = the
145 // outside host's address and port, destination = the host's masquerade
146 // address and B's NAT port. Conntrack would map it onto B's flow and deliver
147 // it to B as "established" traffic if an established-accept came before the
148 // source check.
149 in("a", "ip", "addr", "add", "203.0.114.2/32", "dev", "lo")
150 bLog := t.TempDir() + "/b.log"
151 const listener = `
152import socket,sys
153s=socket.socket(socket.AF_INET,socket.SOCK_DGRAM); s.settimeout(6)
154s.bind(("10.200.0.6",5555)); s.sendto(b"open-flow",("203.0.114.2",9999))
155f=open(sys.argv[1],"a",buffering=1)
156try:
157 while True:
158 d,a=s.recvfrom(100); f.write(d.decode()+" from "+a[0]+"\n")
159except Exception: pass
160`
161 bProc := exec.Command("nsenter", fmt.Sprintf("--net=/proc/%d/ns/net", pid["b"]), "python3", "-c", listener, bLog)
162 if err := bProc.Start(); err != nil {
163 t.Fatal(err)
164 }
165 time.Sleep(1500 * time.Millisecond)
166 if got := udp("a", "203.0.114.1", "forged-reply-to-b", "203.0.114.2", 9999, 5555); got != "silence" {
167 t.Errorf("unexpected answer to the forgery: %s", got)
168 }
169 _ = bProc.Wait()
170 if b, _ := os.ReadFile(bLog); strings.Contains(string(b), "forged-reply-to-b") {
171 t.Errorf("guest B received a reply forged by guest A through conntrack:\n%s", b)
172 }
173 // A forged source address on a plain forwarded packet is dropped as well.
174 if got := udp("a", "203.0.114.2", "forged-src", gB.Addr.String(), 7777, 9999); got != "silence" {
175 t.Errorf("a packet with another guest's source address got through: %s", got)
176 }
177 // S6: guest A cannot use guest B's DNS gateway, nor the host's other ports.
178 if got := udp("a", gB.Gateway.String(), "dns-a-to-b-gw", "", 0, 53); got != "silence" {
179 t.Errorf("guest A reached guest B's gateway on 53: %s", got)
180 }
181 // The outside host cannot open connections to a guest.
182 if got := in("ext", "python3", "-c", client, gA.Addr.String(), "ext-to-a", "", "0", "9999"); got != "silence" {
183 t.Errorf("the outside host reached guest A: %s", got)
184 }
185 // Special-use ranges (S10) are dropped even though the rig could route them.
186 in("ext", "ip", "addr", "add", "198.51.100.7/32", "dev", "lo")
187 sh("ip", "route", "add", "198.51.100.7/32", "via", "203.0.114.2")
188 if got := udp("a", "198.51.100.7", "doc-range", "", 0, 9999); got != "silence" {
189 t.Errorf("TEST-NET-2 reached: %s", got)
190 }
191
192 // Guest forge access: the host listens on 8088 and 8089 on every address
193 // (like a reverse proxy on *:8088). Guest A, which has the exception for
194 // 8088, reaches its own gateway there and nowhere else; guest B, which has
195 // none, reaches nothing.
196 const tcpServer = `
197import socket,threading
198def serve(port):
199 s=socket.socket(); s.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1); s.bind(("0.0.0.0",port)); s.listen(8)
200 while True:
201 c,_=s.accept(); c.sendall(b"hello"); c.close()
202for p in (8088,8089): threading.Thread(target=serve,args=(p,),daemon=True).start()
203threading.Event().wait()
204`
205 tsrv := exec.Command("python3", "-c", tcpServer)
206 if err := tsrv.Start(); err != nil {
207 t.Fatal(err)
208 }
209 t.Cleanup(func() { _ = tsrv.Process.Kill(); _ = tsrv.Wait() })
210 time.Sleep(500 * time.Millisecond)
211 const tcpClient = `
212import socket,sys
213s=socket.socket(); s.settimeout(2)
214try:
215 s.connect((sys.argv[1],int(sys.argv[2]))); print(s.recv(10).decode())
216except Exception: print("closed")
217`
218 tcp := func(n, dst string, port int) string {
219 return in(n, "python3", "-c", tcpClient, dst, fmt.Sprint(port))
220 }
221 if got := tcp("a", gA.Gateway.String(), 8088); got != "hello" {
222 t.Errorf("guest A cannot reach its gateway on the forge port: %s", got)
223 }
224 if got := tcp("a", gA.Gateway.String(), 8089); got != "closed" {
225 t.Errorf("guest A reached its gateway on another port: %s", got)
226 }
227 if got := tcp("a", gB.Gateway.String(), 8088); got != "closed" {
228 t.Errorf("guest A reached guest B's gateway on the forge port: %s", got)
229 }
230 if got := tcp("a", "203.0.114.1", 8088); got != "closed" {
231 t.Errorf("guest A reached the host's LAN address on the forge port: %s", got)
232 }
233 if got := tcp("b", gB.Gateway.String(), 8088); got != "closed" {
234 t.Errorf("guest B, without the exception, reached its gateway: %s", got)
235 }
236 if got := tcp("b", gA.Gateway.String(), 8088); got != "closed" {
237 t.Errorf("guest B reached guest A's gateway on the forge port: %s", got)
238 }
239
240 out := seen()
241 for _, bad := range []string{"forged-src", "a-to-b", "ext-to-a", "doc-range"} {
242 if strings.Contains(out, bad) {
243 t.Errorf("outside host saw %q:\n%s", bad, out)
244 }
245 }
246 for _, good := range []string{"ctl-a", "open-flow"} {
247 if !strings.Contains(out, good) {
248 t.Errorf("outside host did not see %q:\n%s", good, out)
249 }
250 }
251}