1// SPDX-License-Identifier: MIT
2
3package netpolicy
4
5import (
6 "context"
7 "fmt"
8 "os"
9 "os/exec"
10 "strings"
11 "testing"
12 "time"
13
14 "bnema.dev/ermite-runner/internal/cmdrun"
15)
16
17// TestRulesetBetweenTwoGuestsAndAForgedSource loads the real ruleset into the
18// kernel and sends real packets: two guests (ertap0, ertap1) and an outside
19// host hang off veth pairs, each in its own network namespace. It needs
20// CAP_NET_ADMIN, which scripts/it-vm-userns.sh provides without root:
21//
22// ERMITE_RUNNER_IT_NET=1 unshare --user --map-root-user --net go test -run Netns ./internal/netpolicy
23func TestRulesetBetweenTwoGuestsAndAForgedSource(t *testing.T) {
24 if os.Getenv("ERMITE_RUNNER_IT_NET") != "1" {
25 t.Skip("set ERMITE_RUNNER_IT_NET=1 inside a user+network namespace (scripts/it-vm-userns.sh)")
26 }
27 for _, c := range []string{"ip", "nft", "nsenter", "unshare", "python3"} {
28 if _, err := exec.LookPath(c); err != nil {
29 t.Skipf("%s not found", c)
30 }
31 }
32 sh := func(args ...string) string {
33 t.Helper()
34 out, err := exec.Command(args[0], args[1:]...).CombinedOutput()
35 if err != nil {
36 t.Fatalf("%s: %v: %s", strings.Join(args, " "), err, out)
37 }
38 return strings.TrimSpace(string(out))
39 }
40 if out, err := exec.Command("ip", "link", "add", "ertapprobe", "type", "dummy").CombinedOutput(); err != nil {
41 t.Skipf("no CAP_NET_ADMIN: %v: %s", err, out)
42 }
43 sh("ip", "link", "del", "ertapprobe")
44 sh("ip", "link", "set", "lo", "up")
45
46 // Three machines: guest A, guest B, an outside host.
47 pid := map[string]int{}
48 for _, n := range []string{"a", "b", "ext"} {
49 c := exec.Command("unshare", "--net", "sleep", "120")
50 if err := c.Start(); err != nil {
51 t.Fatal(err)
52 }
53 pid[n] = c.Process.Pid
54 t.Cleanup(func() { _ = c.Process.Kill(); _ = c.Wait() })
55 }
56 time.Sleep(300 * time.Millisecond)
57 in := func(n string, args ...string) string {
58 t.Helper()
59 return sh(append([]string{"nsenter", fmt.Sprintf("--net=/proc/%d/ns/net", pid[n])}, args...)...)
60 }
61 link := func(host, n string) {
62 sh("ip", "link", "add", host, "type", "veth", "peer", "name", "v0", "netns", fmt.Sprint(pid[n]))
63 in(n, "ip", "link", "set", "lo", "up")
64 in(n, "ip", "link", "set", "v0", "up")
65 sh("sysctl", "-qw", "net.ipv4.conf."+host+".rp_filter=0")
66 }
67 link("ertap0", "a")
68 link("ertap1", "b")
69 link("extif", "ext")
70 sh("sysctl", "-qw", "net.ipv4.conf.all.rp_filter=0")
71 sh("sysctl", "-qw", "net.ipv4.ip_forward=1")
72
73 gA := guest(t, "ertap0", 0) // 10.200.0.1 / .2
74 gB := guest(t, "ertap1", 1) // 10.200.0.5 / .6
75 for _, g := range []Guest{gA, gB} {
76 sh("ip", "addr", "add", g.Gateway.String()+"/30", "dev", g.Iface)
77 sh("ip", "link", "set", g.Iface, "up")
78 }
79 sh("ip", "addr", "add", "203.0.114.1/24", "dev", "extif")
80 sh("ip", "link", "set", "extif", "up")
81 in("a", "ip", "addr", "add", gA.Addr.String()+"/30", "dev", "v0")
82 in("a", "ip", "route", "add", "default", "via", gA.Gateway.String())
83 in("b", "ip", "addr", "add", gB.Addr.String()+"/30", "dev", "v0")
84 in("b", "ip", "route", "add", "default", "via", gB.Gateway.String())
85 in("ext", "ip", "addr", "add", "203.0.114.2/24", "dev", "v0")
86 in("ext", "ip", "route", "add", "default", "via", "203.0.114.1")
87 // Guest A holds guest B's address too, to be able to forge it.
88 in("a", "ip", "addr", "add", gB.Addr.String()+"/32", "dev", "lo")
89
90 m := NewManager(cmdrun.Exec{}, "nft")
91 ctx := context.Background()
92 t.Cleanup(func() { _ = m.Reset(ctx) })
93 for _, g := range []Guest{gA, gB} {
94 if err := m.Add(ctx, g); err != nil {
95 t.Fatal(err)
96 }
97 }
98
99 // An outside UDP server: logs each payload with the source it saw, and
100 // answers "pong".
101 logf := t.TempDir() + "/ext.log"
102 const server = `
103import socket,sys
104s=socket.socket(socket.AF_INET,socket.SOCK_DGRAM); s.bind(("0.0.0.0",9999))
105f=open(sys.argv[1],"a",buffering=1)
106while True:
107 d,a=s.recvfrom(100); f.write(d.decode()+" from "+a[0]+"\n"); s.sendto(b"pong",a)
108`
109 srv := exec.Command("nsenter", fmt.Sprintf("--net=/proc/%d/ns/net", pid["ext"]), "python3", "-c", server, logf)
110 if err := srv.Start(); err != nil {
111 t.Fatal(err)
112 }
113 t.Cleanup(func() { _ = srv.Process.Kill(); _ = srv.Wait() })
114 time.Sleep(500 * time.Millisecond)
115
116 // udp sends one datagram from machine n (optionally bound to src:sport) and
117 // reports whether an answer came back.
118 const client = `
119import socket,sys
120dst,payload,src,sport=sys.argv[1],sys.argv[2],sys.argv[3],int(sys.argv[4])
121s=socket.socket(socket.AF_INET,socket.SOCK_DGRAM); s.settimeout(2)
122if src: s.bind((src,sport))
123s.sendto(payload.encode(),(dst,int(sys.argv[5]) if len(sys.argv)>5 else 9999))
124try: s.recv(10); print("answer")
125except Exception: print("silence")
126`
127 udp := func(n, dst, payload, src string, sport int, dport int) string {
128 return in(n, "python3", "-c", client, dst, payload, src, fmt.Sprint(sport), fmt.Sprint(dport))
129 }
130 seen := func() string { b, _ := os.ReadFile(logf); return string(b) }
131
132 // Control: a guest reaches the outside host and gets the reply (NAT and the
133 // established path work, so the drops below are the policy, not a broken rig).
134 if got := udp("a", "203.0.114.2", "ctl-a", "", 0, 9999); got != "answer" {
135 t.Fatalf("control: guest A cannot reach the outside host: %s", got)
136 }
137
138 // Guests cannot reach each other, directly or through the other's gateway.
139 if got := udp("a", gB.Addr.String(), "a-to-b", "", 0, 9999); got != "silence" {
140 t.Errorf("guest A reached guest B: %s", got)
141 }
142 // Forged reply (S1): B opens a flow to the outside host from port 5555 and
143 // keeps listening. Guest A then forges a reply of that flow: source = the
144 // outside host's address and port, destination = the host's masquerade
145 // address and B's NAT port. Conntrack would map it onto B's flow and deliver
146 // it to B as "established" traffic if an established-accept came before the
147 // source check.
148 in("a", "ip", "addr", "add", "203.0.114.2/32", "dev", "lo")
149 bLog := t.TempDir() + "/b.log"
150 const listener = `
151import socket,sys
152s=socket.socket(socket.AF_INET,socket.SOCK_DGRAM); s.settimeout(6)
153s.bind(("10.200.0.6",5555)); s.sendto(b"open-flow",("203.0.114.2",9999))
154f=open(sys.argv[1],"a",buffering=1)
155try:
156 while True:
157 d,a=s.recvfrom(100); f.write(d.decode()+" from "+a[0]+"\n")
158except Exception: pass
159`
160 bProc := exec.Command("nsenter", fmt.Sprintf("--net=/proc/%d/ns/net", pid["b"]), "python3", "-c", listener, bLog)
161 if err := bProc.Start(); err != nil {
162 t.Fatal(err)
163 }
164 time.Sleep(1500 * time.Millisecond)
165 if got := udp("a", "203.0.114.1", "forged-reply-to-b", "203.0.114.2", 9999, 5555); got != "silence" {
166 t.Errorf("unexpected answer to the forgery: %s", got)
167 }
168 _ = bProc.Wait()
169 if b, _ := os.ReadFile(bLog); strings.Contains(string(b), "forged-reply-to-b") {
170 t.Errorf("guest B received a reply forged by guest A through conntrack:\n%s", b)
171 }
172 // A forged source address on a plain forwarded packet is dropped as well.
173 if got := udp("a", "203.0.114.2", "forged-src", gB.Addr.String(), 7777, 9999); got != "silence" {
174 t.Errorf("a packet with another guest's source address got through: %s", got)
175 }
176 // S6: guest A cannot use guest B's DNS gateway, nor the host's other ports.
177 if got := udp("a", gB.Gateway.String(), "dns-a-to-b-gw", "", 0, 53); got != "silence" {
178 t.Errorf("guest A reached guest B's gateway on 53: %s", got)
179 }
180 // The outside host cannot open connections to a guest.
181 if got := in("ext", "python3", "-c", client, gA.Addr.String(), "ext-to-a", "", "0", "9999"); got != "silence" {
182 t.Errorf("the outside host reached guest A: %s", got)
183 }
184 // Special-use ranges (S10) are dropped even though the rig could route them.
185 in("ext", "ip", "addr", "add", "198.51.100.7/32", "dev", "lo")
186 sh("ip", "route", "add", "198.51.100.7/32", "via", "203.0.114.2")
187 if got := udp("a", "198.51.100.7", "doc-range", "", 0, 9999); got != "silence" {
188 t.Errorf("TEST-NET-2 reached: %s", got)
189 }
190
191 out := seen()
192 for _, bad := range []string{"forged-src", "a-to-b", "ext-to-a", "doc-range"} {
193 if strings.Contains(out, bad) {
194 t.Errorf("outside host saw %q:\n%s", bad, out)
195 }
196 }
197 for _, good := range []string{"ctl-a", "open-flow"} {
198 if !strings.Contains(out, good) {
199 t.Errorf("outside host did not see %q:\n%s", good, out)
200 }
201 }
202}