1// SPDX-License-Identifier: MIT
2
3// Package netpolicy generates and applies the nftables policy of job sandboxes.
4//
5// All rules live in one dedicated table, "inet ermite_runner", that is replaced
6// atomically (one `nft -f -` transaction) whenever a sandbox is added or
7// removed, and deleted when none is left. Nothing outside that table is touched.
8//
9// Policy for every guest interface (a TAP in a micro-VM, a bridge in a
10// container):
11// - to the host itself (input hook): only DNS (udp/tcp 53) to the gateway
12// address of that guest, where the runner's DNS forwarder listens; every
13// other packet is dropped, and port 53 of a gateway is closed to every
14// other interface;
15// - IPv6 from the guest is dropped (the guest also disables IPv6);
16// - forwarded traffic must carry the guest's own source address;
17// - forwarded traffic to RFC 1918, CGNAT 100.64.0.0/10 (VPN overlays),
18// link-local 169.254.0.0/16 (cloud metadata 169.254.169.254), loopback,
19// "this network", multicast and reserved ranges is dropped;
20// - traffic towards a guest interface that does not belong to an established
21// connection is dropped, so guests cannot reach each other;
22// - everything else is NATed (masquerade) to the internet.
23package netpolicy
24
25import (
26 "context"
27 "errors"
28 "fmt"
29 "net/netip"
30 "sort"
31 "strings"
32 "sync"
33
34 "bnema.dev/ermite-runner/internal/cmdrun"
35)
36
37// Table is the nftables table owned by the runner.
38const Table = "ermite_runner"
39
40// IfacePrefixes are the interface name prefixes of sandboxes; wildcard rules
41// are generated for them. TAP devices of micro-VMs, bridges of containers.
42var IfacePrefixes = []string{"ertap", "erbr"}
43
44// blockedV4 are the destinations a guest may never reach.
45var blockedV4 = []string{
46 "0.0.0.0/8", // "this network"
47 "10.0.0.0/8", // RFC 1918
48 "100.64.0.0/10", // RFC 6598 shared address space (CGNAT, VPN overlays)
49 "127.0.0.0/8", // loopback
50 "169.254.0.0/16", // link-local, incl. the 169.254.169.254 metadata service
51 "172.16.0.0/12", // RFC 1918
52 "192.0.0.0/24", // IETF protocol assignments
53 "192.0.2.0/24", // TEST-NET-1
54 "192.168.0.0/16", // RFC 1918
55 "198.18.0.0/15", // benchmarking
56 "198.51.100.0/24", // TEST-NET-2
57 "203.0.113.0/24", // TEST-NET-3
58 "224.0.0.0/4", // multicast
59 "240.0.0.0/4", // reserved, incl. limited broadcast
60}
61
62// prefixSet renders IfacePrefixes as an nft anonymous set of interface wildcards.
63func prefixSet() string {
64 q := make([]string, len(IfacePrefixes))
65 for i, p := range IfacePrefixes {
66 q[i] = `"` + p + `*"`
67 }
68 return "{ " + strings.Join(q, ", ") + " }"
69}
70
71// Guest is one sandbox network attachment.
72type Guest struct {
73 Iface string // host-side interface: TAP or bridge name
74 Gateway netip.Addr // host address on Iface; the DNS forwarder listens here
75 Addr netip.Addr // the guest's only allowed source address
76}
77
78// Validate checks the fields used in generated rules.
79func (g Guest) Validate() error {
80 if !validIface(g.Iface) {
81 return fmt.Errorf("netpolicy: invalid interface name %q", g.Iface)
82 }
83 if !g.Gateway.Is4() || !g.Addr.Is4() {
84 return errors.New("netpolicy: guest and gateway must be IPv4 addresses")
85 }
86 return nil
87}
88
89func validIface(s string) bool {
90 if s == "" || len(s) > 15 {
91 return false
92 }
93 for _, r := range s {
94 if (r < 'a' || r > 'z') && (r < '0' || r > '9') && r != '-' && r != '_' {
95 return false
96 }
97 }
98 return true
99}
100
101// Net is the /30 given to one sandbox.
102type Net struct {
103 Prefix netip.Prefix // the /30
104 Gateway netip.Addr // host side (.1)
105 Guest netip.Addr // guest side (.2)
106}
107
108// Subnet returns the slot-th /30 of base (an IPv4 prefix of at most /30).
109func Subnet(base netip.Prefix, slot int) (Net, error) {
110 if !base.Addr().Is4() || base.Bits() > 30 || slot < 0 {
111 return Net{}, fmt.Errorf("netpolicy: bad base %s or slot %d", base, slot)
112 }
113 if slot >= 1<<(30-base.Bits()) {
114 return Net{}, fmt.Errorf("netpolicy: slot %d does not fit in %s", slot, base)
115 }
116 b := base.Masked().Addr().As4()
117 v := uint32(b[0])<<24 | uint32(b[1])<<16 | uint32(b[2])<<8 | uint32(b[3])
118 v += uint32(slot) * 4
119 addr := func(off uint32) netip.Addr {
120 x := v + off
121 return netip.AddrFrom4([4]byte{byte(x >> 24), byte(x >> 16), byte(x >> 8), byte(x)})
122 }
123 return Net{Prefix: netip.PrefixFrom(addr(0), 30), Gateway: addr(1), Guest: addr(2)}, nil
124}
125
126// Ruleset returns the nft script that atomically replaces the runner table with
127// the policy for guests. With no guest it only deletes the table. The output is
128// deterministic (guests sorted by interface).
129func Ruleset(guests []Guest) string {
130 var b strings.Builder
131 fmt.Fprintf(&b, "table inet %s\ndelete table inet %s\n", Table, Table)
132 if len(guests) == 0 {
133 return b.String()
134 }
135 gs := append([]Guest(nil), guests...)
136 sort.Slice(gs, func(i, j int) bool { return gs[i].Iface < gs[j].Iface })
137
138 fmt.Fprintf(&b, "table inet %s {\n", Table)
139 b.WriteString("\tset blocked_v4 {\n\t\ttype ipv4_addr\n\t\tflags interval\n\t\telements = { ")
140 b.WriteString(strings.Join(blockedV4, ", "))
141 b.WriteString(" }\n\t}\n")
142
143 b.WriteString("\tchain input {\n\t\ttype filter hook input priority filter; policy accept;\n")
144 for _, p := range IfacePrefixes {
145 fmt.Fprintf(&b, "\t\tiifname \"%s*\" meta nfproto ipv6 drop\n", p)
146 }
147 // The DNS forwarder of a guest answers that guest only: port 53 of a gateway
148 // address is dropped from every other interface (other guests, the LAN).
149 for _, g := range gs {
150 for _, proto := range []string{"udp", "tcp"} {
151 fmt.Fprintf(&b, "\t\tip daddr %s %s dport 53 iifname != \"%s\" drop\n", g.Gateway, proto, g.Iface)
152 }
153 }
154 for _, g := range gs {
155 for _, proto := range []string{"udp", "tcp"} {
156 fmt.Fprintf(&b, "\t\tiifname \"%s\" ip saddr %s ip daddr %s %s dport 53 accept\n", g.Iface, g.Addr, g.Gateway, proto)
157 }
158 fmt.Fprintf(&b, "\t\tiifname \"%s\" drop\n", g.Iface)
159 }
160 // An interface with our prefix that no live guest owns (a leftover) is
161 // dropped as well.
162 for _, p := range IfacePrefixes {
163 fmt.Fprintf(&b, "\t\tiifname \"%s*\" drop\n", p)
164 }
165 b.WriteString("\t}\n")
166
167 // Forward chain. Order matters: everything a guest sends is checked (family,
168 // source address, destination ranges) BEFORE any "established" accept, or a
169 // guest could forge the source of another guest's flow and have it accepted
170 // as a reply.
171 b.WriteString("\tchain forward {\n\t\ttype filter hook forward priority filter; policy accept;\n")
172 for _, p := range IfacePrefixes {
173 fmt.Fprintf(&b, "\t\tiifname \"%s*\" meta nfproto ipv6 drop\n", p)
174 }
175 for _, g := range gs {
176 fmt.Fprintf(&b, "\t\tiifname \"%s\" ip saddr != %s drop\n", g.Iface, g.Addr)
177 fmt.Fprintf(&b, "\t\tiifname \"%s\" ip daddr @blocked_v4 drop\n", g.Iface)
178 }
179 // Replies from outside to a guest; never for packets that come from a guest.
180 b.WriteString("\t\tiifname != " + prefixSet() + " oifname " + prefixSet() + " ct state established,related accept\n")
181 for _, p := range IfacePrefixes {
182 fmt.Fprintf(&b, "\t\toifname \"%s*\" drop\n", p)
183 }
184 for _, g := range gs {
185 fmt.Fprintf(&b, "\t\tiifname \"%s\" accept\n", g.Iface)
186 }
187 for _, p := range IfacePrefixes {
188 fmt.Fprintf(&b, "\t\tiifname \"%s*\" drop\n", p)
189 }
190 b.WriteString("\t}\n")
191
192 b.WriteString("\tchain postrouting {\n\t\ttype nat hook postrouting priority srcnat; policy accept;\n")
193 for _, g := range gs {
194 fmt.Fprintf(&b, "\t\tip saddr %s oifname != \"%s\" masquerade\n", g.Addr, g.Iface)
195 }
196 b.WriteString("\t}\n}\n")
197 return b.String()
198}
199
200// Manager applies the policy for the set of live guests.
201type Manager struct {
202 run cmdrun.Runner
203 nft string
204 mu sync.Mutex
205 guests map[string]Guest
206}
207
208// NewManager returns a Manager that runs nft (path or name) through run.
209func NewManager(run cmdrun.Runner, nft string) *Manager {
210 if nft == "" {
211 nft = "nft"
212 }
213 return &Manager{run: run, nft: nft, guests: map[string]Guest{}}
214}
215
216// Add installs (or replaces) the policy of g.
217func (m *Manager) Add(ctx context.Context, g Guest) error {
218 if err := g.Validate(); err != nil {
219 return err
220 }
221 m.mu.Lock()
222 defer m.mu.Unlock()
223 prev, had := m.guests[g.Iface]
224 m.guests[g.Iface] = g
225 if err := m.applyLocked(ctx); err != nil {
226 if had {
227 m.guests[g.Iface] = prev
228 } else {
229 delete(m.guests, g.Iface)
230 }
231 return err
232 }
233 return nil
234}
235
236// Remove drops the policy of iface; the table disappears with the last guest.
237// Removing an unknown interface is not an error.
238func (m *Manager) Remove(ctx context.Context, iface string) error {
239 m.mu.Lock()
240 defer m.mu.Unlock()
241 if _, ok := m.guests[iface]; !ok {
242 return nil
243 }
244 delete(m.guests, iface)
245 return m.applyLocked(ctx)
246}
247
248// Reset deletes the runner table whatever its content: used at start to reap
249// the table of a previous run. Safe when the table does not exist.
250func (m *Manager) Reset(ctx context.Context) error {
251 m.mu.Lock()
252 defer m.mu.Unlock()
253 m.guests = map[string]Guest{}
254 return m.applyLocked(ctx)
255}
256
257func (m *Manager) applyLocked(ctx context.Context) error {
258 gs := make([]Guest, 0, len(m.guests))
259 for _, g := range m.guests {
260 gs = append(gs, g)
261 }
262 // `table X` creates the table if missing so that the following delete cannot
263 // fail; the whole script is one transaction.
264 if _, err := m.run.Run(ctx, strings.NewReader(Ruleset(gs)), m.nft, "-f", "-"); err != nil {
265 return fmt.Errorf("netpolicy: apply ruleset: %w", err)
266 }
267 return nil
268}