1// SPDX-License-Identifier: MIT
2
3// Package netpolicy generates and applies the nftables policy of job sandboxes.
4//
5// All rules live in one dedicated table, "inet ermite_runner", that is replaced
6// atomically (one `nft -f -` transaction) whenever a sandbox is added or
7// removed, and deleted when none is left. Nothing outside that table is touched.
8//
9// Policy for every guest interface (a TAP in a micro-VM, a bridge in a
10// container):
11// - to the host itself (input hook): only DNS (udp/tcp 53) to the gateway
12// address of that guest, where the runner's DNS forwarder listens; every
13// other packet is dropped, and port 53 of a gateway is closed to every
14// other interface;
15// - optionally (Guest.ForgePort, the opt-in guest forge access, staging
16// only): TCP from the guest's source address to its own gateway address on
17// that one port. The host's own listener on that port receives the traffic;
18// nothing is proxied, and every other host port stays closed;
19// - IPv6 from the guest is dropped (the guest also disables IPv6);
20// - forwarded traffic must carry the guest's own source address;
21// - forwarded traffic to RFC 1918, CGNAT 100.64.0.0/10 (VPN overlays),
22// link-local 169.254.0.0/16 (cloud metadata 169.254.169.254), loopback,
23// "this network", multicast and reserved ranges is dropped;
24// - traffic towards a guest interface that does not belong to an established
25// connection is dropped, so guests cannot reach each other;
26// - everything else is NATed (masquerade) to the internet.
27package netpolicy
28
29import (
30 "context"
31 "errors"
32 "fmt"
33 "net/netip"
34 "sort"
35 "strings"
36 "sync"
37
38 "bnema.dev/ermite-runner/internal/cmdrun"
39)
40
41// Table is the nftables table owned by the runner.
42const Table = "ermite_runner"
43
44// IfacePrefixes are the interface name prefixes of sandboxes; wildcard rules
45// are generated for them. TAP devices of micro-VMs, bridges of containers.
46var IfacePrefixes = []string{"ertap", "erbr"}
47
48// blockedV4 are the destinations a guest may never reach.
49var blockedV4 = []string{
50 "0.0.0.0/8", // "this network"
51 "10.0.0.0/8", // RFC 1918
52 "100.64.0.0/10", // RFC 6598 shared address space (CGNAT, VPN overlays)
53 "127.0.0.0/8", // loopback
54 "169.254.0.0/16", // link-local, incl. the 169.254.169.254 metadata service
55 "172.16.0.0/12", // RFC 1918
56 "192.0.0.0/24", // IETF protocol assignments
57 "192.0.2.0/24", // TEST-NET-1
58 "192.168.0.0/16", // RFC 1918
59 "198.18.0.0/15", // benchmarking
60 "198.51.100.0/24", // TEST-NET-2
61 "203.0.113.0/24", // TEST-NET-3
62 "224.0.0.0/4", // multicast
63 "240.0.0.0/4", // reserved, incl. limited broadcast
64}
65
66// prefixSet renders IfacePrefixes as an nft anonymous set of interface wildcards.
67func prefixSet() string {
68 q := make([]string, len(IfacePrefixes))
69 for i, p := range IfacePrefixes {
70 q[i] = `"` + p + `*"`
71 }
72 return "{ " + strings.Join(q, ", ") + " }"
73}
74
75// Guest is one sandbox network attachment.
76type Guest struct {
77 Iface string // host-side interface: TAP or bridge name
78 Gateway netip.Addr // host address on Iface; the DNS forwarder listens here
79 Addr netip.Addr // the guest's only allowed source address
80 // ForgePort, when not 0, opens TCP from Addr to Gateway on that one port
81 // (guest forge access). The accept is in the input chain, scoped to this
82 // guest's interface, source and gateway, so it neither reaches other
83 // addresses of the host nor other guests.
84 ForgePort uint16
85}
86
87// Validate checks the fields used in generated rules.
88func (g Guest) Validate() error {
89 if !validIface(g.Iface) {
90 return fmt.Errorf("netpolicy: invalid interface name %q", g.Iface)
91 }
92 if !g.Gateway.Is4() || !g.Addr.Is4() {
93 return errors.New("netpolicy: guest and gateway must be IPv4 addresses")
94 }
95 return nil
96}
97
98func validIface(s string) bool {
99 if s == "" || len(s) > 15 {
100 return false
101 }
102 for _, r := range s {
103 if (r < 'a' || r > 'z') && (r < '0' || r > '9') && r != '-' && r != '_' {
104 return false
105 }
106 }
107 return true
108}
109
110// Net is the /30 given to one sandbox.
111type Net struct {
112 Prefix netip.Prefix // the /30
113 Gateway netip.Addr // host side (.1)
114 Guest netip.Addr // guest side (.2)
115}
116
117// Subnet returns the slot-th /30 of base (an IPv4 prefix of at most /30).
118func Subnet(base netip.Prefix, slot int) (Net, error) {
119 if !base.Addr().Is4() || base.Bits() > 30 || slot < 0 {
120 return Net{}, fmt.Errorf("netpolicy: bad base %s or slot %d", base, slot)
121 }
122 if slot >= 1<<(30-base.Bits()) {
123 return Net{}, fmt.Errorf("netpolicy: slot %d does not fit in %s", slot, base)
124 }
125 b := base.Masked().Addr().As4()
126 v := uint32(b[0])<<24 | uint32(b[1])<<16 | uint32(b[2])<<8 | uint32(b[3])
127 v += uint32(slot) * 4
128 addr := func(off uint32) netip.Addr {
129 x := v + off
130 return netip.AddrFrom4([4]byte{byte(x >> 24), byte(x >> 16), byte(x >> 8), byte(x)})
131 }
132 return Net{Prefix: netip.PrefixFrom(addr(0), 30), Gateway: addr(1), Guest: addr(2)}, nil
133}
134
135// Ruleset returns the nft script that atomically replaces the runner table with
136// the policy for guests. With no guest it only deletes the table. The output is
137// deterministic (guests sorted by interface).
138func Ruleset(guests []Guest) string {
139 var b strings.Builder
140 fmt.Fprintf(&b, "table inet %s\ndelete table inet %s\n", Table, Table)
141 if len(guests) == 0 {
142 return b.String()
143 }
144 gs := append([]Guest(nil), guests...)
145 sort.Slice(gs, func(i, j int) bool { return gs[i].Iface < gs[j].Iface })
146
147 fmt.Fprintf(&b, "table inet %s {\n", Table)
148 b.WriteString("\tset blocked_v4 {\n\t\ttype ipv4_addr\n\t\tflags interval\n\t\telements = { ")
149 b.WriteString(strings.Join(blockedV4, ", "))
150 b.WriteString(" }\n\t}\n")
151
152 b.WriteString("\tchain input {\n\t\ttype filter hook input priority filter; policy accept;\n")
153 for _, p := range IfacePrefixes {
154 fmt.Fprintf(&b, "\t\tiifname \"%s*\" meta nfproto ipv6 drop\n", p)
155 }
156 // The DNS forwarder of a guest answers that guest only: port 53 of a gateway
157 // address is dropped from every other interface (other guests, the LAN).
158 for _, g := range gs {
159 for _, proto := range []string{"udp", "tcp"} {
160 fmt.Fprintf(&b, "\t\tip daddr %s %s dport 53 iifname != \"%s\" drop\n", g.Gateway, proto, g.Iface)
161 }
162 }
163 for _, g := range gs {
164 for _, proto := range []string{"udp", "tcp"} {
165 fmt.Fprintf(&b, "\t\tiifname \"%s\" ip saddr %s ip daddr %s %s dport 53 accept\n", g.Iface, g.Addr, g.Gateway, proto)
166 }
167 if g.ForgePort != 0 {
168 // New connections only: the replies leave through the output hook,
169 // which this table does not touch.
170 fmt.Fprintf(&b, "\t\tiifname \"%s\" ip saddr %s ip daddr %s tcp dport %d accept\n", g.Iface, g.Addr, g.Gateway, g.ForgePort)
171 }
172 fmt.Fprintf(&b, "\t\tiifname \"%s\" drop\n", g.Iface)
173 }
174 // An interface with our prefix that no live guest owns (a leftover) is
175 // dropped as well.
176 for _, p := range IfacePrefixes {
177 fmt.Fprintf(&b, "\t\tiifname \"%s*\" drop\n", p)
178 }
179 b.WriteString("\t}\n")
180
181 // Forward chain. Order matters: everything a guest sends is checked (family,
182 // source address, destination ranges) BEFORE any "established" accept, or a
183 // guest could forge the source of another guest's flow and have it accepted
184 // as a reply.
185 b.WriteString("\tchain forward {\n\t\ttype filter hook forward priority filter; policy accept;\n")
186 for _, p := range IfacePrefixes {
187 fmt.Fprintf(&b, "\t\tiifname \"%s*\" meta nfproto ipv6 drop\n", p)
188 }
189 for _, g := range gs {
190 fmt.Fprintf(&b, "\t\tiifname \"%s\" ip saddr != %s drop\n", g.Iface, g.Addr)
191 fmt.Fprintf(&b, "\t\tiifname \"%s\" ip daddr @blocked_v4 drop\n", g.Iface)
192 }
193 // Replies from outside to a guest; never for packets that come from a guest.
194 b.WriteString("\t\tiifname != " + prefixSet() + " oifname " + prefixSet() + " ct state established,related accept\n")
195 for _, p := range IfacePrefixes {
196 fmt.Fprintf(&b, "\t\toifname \"%s*\" drop\n", p)
197 }
198 for _, g := range gs {
199 fmt.Fprintf(&b, "\t\tiifname \"%s\" accept\n", g.Iface)
200 }
201 for _, p := range IfacePrefixes {
202 fmt.Fprintf(&b, "\t\tiifname \"%s*\" drop\n", p)
203 }
204 b.WriteString("\t}\n")
205
206 b.WriteString("\tchain postrouting {\n\t\ttype nat hook postrouting priority srcnat; policy accept;\n")
207 for _, g := range gs {
208 fmt.Fprintf(&b, "\t\tip saddr %s oifname != \"%s\" masquerade\n", g.Addr, g.Iface)
209 }
210 b.WriteString("\t}\n}\n")
211 return b.String()
212}
213
214// Manager applies the policy for the set of live guests.
215type Manager struct {
216 run cmdrun.Runner
217 nft string
218 mu sync.Mutex
219 guests map[string]Guest
220}
221
222// NewManager returns a Manager that runs nft (path or name) through run.
223func NewManager(run cmdrun.Runner, nft string) *Manager {
224 if nft == "" {
225 nft = "nft"
226 }
227 return &Manager{run: run, nft: nft, guests: map[string]Guest{}}
228}
229
230// Add installs (or replaces) the policy of g.
231func (m *Manager) Add(ctx context.Context, g Guest) error {
232 if err := g.Validate(); err != nil {
233 return err
234 }
235 m.mu.Lock()
236 defer m.mu.Unlock()
237 prev, had := m.guests[g.Iface]
238 m.guests[g.Iface] = g
239 if err := m.applyLocked(ctx); err != nil {
240 if had {
241 m.guests[g.Iface] = prev
242 } else {
243 delete(m.guests, g.Iface)
244 }
245 return err
246 }
247 return nil
248}
249
250// Remove drops the policy of iface; the table disappears with the last guest.
251// Removing an unknown interface is not an error.
252func (m *Manager) Remove(ctx context.Context, iface string) error {
253 m.mu.Lock()
254 defer m.mu.Unlock()
255 if _, ok := m.guests[iface]; !ok {
256 return nil
257 }
258 delete(m.guests, iface)
259 return m.applyLocked(ctx)
260}
261
262// Reset deletes the runner table whatever its content: used at start to reap
263// the table of a previous run. Safe when the table does not exist.
264func (m *Manager) Reset(ctx context.Context) error {
265 m.mu.Lock()
266 defer m.mu.Unlock()
267 m.guests = map[string]Guest{}
268 return m.applyLocked(ctx)
269}
270
271func (m *Manager) applyLocked(ctx context.Context) error {
272 gs := make([]Guest, 0, len(m.guests))
273 for _, g := range m.guests {
274 gs = append(gs, g)
275 }
276 // `table X` creates the table if missing so that the following delete cannot
277 // fail; the whole script is one transaction.
278 if _, err := m.run.Run(ctx, strings.NewReader(Ruleset(gs)), m.nft, "-f", "-"); err != nil {
279 return fmt.Errorf("netpolicy: apply ruleset: %w", err)
280 }
281 return nil
282}