1// SPDX-License-Identifier: MIT
2
3// Package netpolicy generates and applies the nftables policy of job sandboxes.
4//
5// All rules live in one dedicated table, "inet ermite_runner", that is replaced
6// atomically (one `nft -f -` transaction) whenever a sandbox is added or
7// removed, and deleted when none is left. Nothing outside that table is touched.
8//
9// Policy for every guest interface (a TAP in a micro-VM, a bridge in a
10// container):
11// - to the host itself (input hook): only DNS (udp/tcp 53) to the gateway
12// address of that guest, where the runner's DNS forwarder listens; every
13// other packet is dropped, and port 53 of a gateway is closed to every
14// other interface;
15// - optionally (Guest.ForgePort, the opt-in guest forge access, staging
16// only): TCP from the guest's source address to its own gateway address on
17// that one port. The host's own listener on that port receives the traffic;
18// nothing is proxied, and every other host port stays closed;
19// the GitHub Actions cache has no rule here: it is not on the network (the
20// guest reaches it on its own loopback, through the host channel);
21// - IPv6 from the guest is dropped (the guest also disables IPv6);
22// - forwarded traffic must carry the guest's own source address;
23// - forwarded traffic to RFC 1918, CGNAT 100.64.0.0/10 (VPN overlays),
24// link-local 169.254.0.0/16 (cloud metadata 169.254.169.254), loopback,
25// "this network", multicast and reserved ranges is dropped;
26// - traffic towards a guest interface that does not belong to an established
27// connection is dropped, so guests cannot reach each other;
28// - everything else is NATed (masquerade) to the internet.
29package netpolicy
30
31import (
32 "context"
33 "errors"
34 "fmt"
35 "net/netip"
36 "sort"
37 "strings"
38 "sync"
39
40 "bnema.dev/ermite-runner/internal/cmdrun"
41)
42
43// Table is the nftables table owned by the runner.
44const Table = "ermite_runner"
45
46// IfacePrefixes are the interface name prefixes of sandboxes; wildcard rules
47// are generated for them. TAP devices of micro-VMs, bridges of containers.
48var IfacePrefixes = []string{"ertap", "erbr"}
49
50// blockedV4 are the destinations a guest may never reach.
51var blockedV4 = []string{
52 "0.0.0.0/8", // "this network"
53 "10.0.0.0/8", // RFC 1918
54 "100.64.0.0/10", // RFC 6598 shared address space (CGNAT, VPN overlays)
55 "127.0.0.0/8", // loopback
56 "169.254.0.0/16", // link-local, incl. the 169.254.169.254 metadata service
57 "172.16.0.0/12", // RFC 1918
58 "192.0.0.0/24", // IETF protocol assignments
59 "192.0.2.0/24", // TEST-NET-1
60 "192.168.0.0/16", // RFC 1918
61 "198.18.0.0/15", // benchmarking
62 "198.51.100.0/24", // TEST-NET-2
63 "203.0.113.0/24", // TEST-NET-3
64 "224.0.0.0/4", // multicast
65 "240.0.0.0/4", // reserved, incl. limited broadcast
66}
67
68// prefixSet renders IfacePrefixes as an nft anonymous set of interface wildcards.
69func prefixSet() string {
70 q := make([]string, len(IfacePrefixes))
71 for i, p := range IfacePrefixes {
72 q[i] = `"` + p + `*"`
73 }
74 return "{ " + strings.Join(q, ", ") + " }"
75}
76
77// Guest is one sandbox network attachment.
78type Guest struct {
79 Iface string // host-side interface: TAP or bridge name
80 Gateway netip.Addr // host address on Iface; the DNS forwarder listens here
81 Addr netip.Addr // the guest's only allowed source address
82 // ForgePort, when not 0, opens TCP from Addr to Gateway on that one port
83 // (guest forge access). The accept is in the input chain, scoped to this
84 // guest's interface, source and gateway, so it neither reaches other
85 // addresses of the host nor other guests.
86 ForgePort uint16
87}
88
89// Validate checks the fields used in generated rules.
90func (g Guest) Validate() error {
91 if !validIface(g.Iface) {
92 return fmt.Errorf("netpolicy: invalid interface name %q", g.Iface)
93 }
94 if !g.Gateway.Is4() || !g.Addr.Is4() {
95 return errors.New("netpolicy: guest and gateway must be IPv4 addresses")
96 }
97 return nil
98}
99
100func validIface(s string) bool {
101 if s == "" || len(s) > 15 {
102 return false
103 }
104 for _, r := range s {
105 if (r < 'a' || r > 'z') && (r < '0' || r > '9') && r != '-' && r != '_' {
106 return false
107 }
108 }
109 return true
110}
111
112// Net is the /30 given to one sandbox.
113type Net struct {
114 Prefix netip.Prefix // the /30
115 Gateway netip.Addr // host side (.1)
116 Guest netip.Addr // guest side (.2)
117}
118
119// Subnet returns the slot-th /30 of base (an IPv4 prefix of at most /30).
120func Subnet(base netip.Prefix, slot int) (Net, error) {
121 if !base.Addr().Is4() || base.Bits() > 30 || slot < 0 {
122 return Net{}, fmt.Errorf("netpolicy: bad base %s or slot %d", base, slot)
123 }
124 if slot >= 1<<(30-base.Bits()) {
125 return Net{}, fmt.Errorf("netpolicy: slot %d does not fit in %s", slot, base)
126 }
127 b := base.Masked().Addr().As4()
128 v := uint32(b[0])<<24 | uint32(b[1])<<16 | uint32(b[2])<<8 | uint32(b[3])
129 v += uint32(slot) * 4
130 addr := func(off uint32) netip.Addr {
131 x := v + off
132 return netip.AddrFrom4([4]byte{byte(x >> 24), byte(x >> 16), byte(x >> 8), byte(x)})
133 }
134 return Net{Prefix: netip.PrefixFrom(addr(0), 30), Gateway: addr(1), Guest: addr(2)}, nil
135}
136
137// Ruleset returns the nft script that atomically replaces the runner table with
138// the policy for guests. With no guest it only deletes the table. The output is
139// deterministic (guests sorted by interface).
140func Ruleset(guests []Guest) string {
141 var b strings.Builder
142 fmt.Fprintf(&b, "table inet %s\ndelete table inet %s\n", Table, Table)
143 if len(guests) == 0 {
144 return b.String()
145 }
146 gs := append([]Guest(nil), guests...)
147 sort.Slice(gs, func(i, j int) bool { return gs[i].Iface < gs[j].Iface })
148
149 fmt.Fprintf(&b, "table inet %s {\n", Table)
150 b.WriteString("\tset blocked_v4 {\n\t\ttype ipv4_addr\n\t\tflags interval\n\t\telements = { ")
151 b.WriteString(strings.Join(blockedV4, ", "))
152 b.WriteString(" }\n\t}\n")
153
154 b.WriteString("\tchain input {\n\t\ttype filter hook input priority filter; policy accept;\n")
155 for _, p := range IfacePrefixes {
156 fmt.Fprintf(&b, "\t\tiifname \"%s*\" meta nfproto ipv6 drop\n", p)
157 }
158 // The DNS forwarder of a guest answers that guest only: port 53 of a gateway
159 // address is dropped from every other interface (other guests, the LAN).
160 for _, g := range gs {
161 for _, proto := range []string{"udp", "tcp"} {
162 fmt.Fprintf(&b, "\t\tip daddr %s %s dport 53 iifname != \"%s\" drop\n", g.Gateway, proto, g.Iface)
163 }
164 }
165 for _, g := range gs {
166 for _, proto := range []string{"udp", "tcp"} {
167 fmt.Fprintf(&b, "\t\tiifname \"%s\" ip saddr %s ip daddr %s %s dport 53 accept\n", g.Iface, g.Addr, g.Gateway, proto)
168 }
169 if g.ForgePort != 0 {
170 // New connections only: the replies leave through the output hook,
171 // which this table does not touch.
172 fmt.Fprintf(&b, "\t\tiifname \"%s\" ip saddr %s ip daddr %s tcp dport %d accept\n", g.Iface, g.Addr, g.Gateway, g.ForgePort)
173 }
174 fmt.Fprintf(&b, "\t\tiifname \"%s\" drop\n", g.Iface)
175 }
176 // An interface with our prefix that no live guest owns (a leftover) is
177 // dropped as well.
178 for _, p := range IfacePrefixes {
179 fmt.Fprintf(&b, "\t\tiifname \"%s*\" drop\n", p)
180 }
181 b.WriteString("\t}\n")
182
183 // Forward chain. Order matters: everything a guest sends is checked (family,
184 // source address, destination ranges) BEFORE any "established" accept, or a
185 // guest could forge the source of another guest's flow and have it accepted
186 // as a reply.
187 b.WriteString("\tchain forward {\n\t\ttype filter hook forward priority filter; policy accept;\n")
188 for _, p := range IfacePrefixes {
189 fmt.Fprintf(&b, "\t\tiifname \"%s*\" meta nfproto ipv6 drop\n", p)
190 }
191 for _, g := range gs {
192 fmt.Fprintf(&b, "\t\tiifname \"%s\" ip saddr != %s drop\n", g.Iface, g.Addr)
193 fmt.Fprintf(&b, "\t\tiifname \"%s\" ip daddr @blocked_v4 drop\n", g.Iface)
194 }
195 // Replies from outside to a guest; never for packets that come from a guest.
196 b.WriteString("\t\tiifname != " + prefixSet() + " oifname " + prefixSet() + " ct state established,related accept\n")
197 for _, p := range IfacePrefixes {
198 fmt.Fprintf(&b, "\t\toifname \"%s*\" drop\n", p)
199 }
200 for _, g := range gs {
201 fmt.Fprintf(&b, "\t\tiifname \"%s\" accept\n", g.Iface)
202 }
203 for _, p := range IfacePrefixes {
204 fmt.Fprintf(&b, "\t\tiifname \"%s*\" drop\n", p)
205 }
206 b.WriteString("\t}\n")
207
208 b.WriteString("\tchain postrouting {\n\t\ttype nat hook postrouting priority srcnat; policy accept;\n")
209 for _, g := range gs {
210 fmt.Fprintf(&b, "\t\tip saddr %s oifname != \"%s\" masquerade\n", g.Addr, g.Iface)
211 }
212 b.WriteString("\t}\n}\n")
213 return b.String()
214}
215
216// Manager applies the policy for the set of live guests.
217type Manager struct {
218 run cmdrun.Runner
219 nft string
220 mu sync.Mutex
221 guests map[string]Guest
222}
223
224// NewManager returns a Manager that runs nft (path or name) through run.
225func NewManager(run cmdrun.Runner, nft string) *Manager {
226 if nft == "" {
227 nft = "nft"
228 }
229 return &Manager{run: run, nft: nft, guests: map[string]Guest{}}
230}
231
232// Add installs (or replaces) the policy of g.
233func (m *Manager) Add(ctx context.Context, g Guest) error {
234 if err := g.Validate(); err != nil {
235 return err
236 }
237 m.mu.Lock()
238 defer m.mu.Unlock()
239 prev, had := m.guests[g.Iface]
240 m.guests[g.Iface] = g
241 if err := m.applyLocked(ctx); err != nil {
242 if had {
243 m.guests[g.Iface] = prev
244 } else {
245 delete(m.guests, g.Iface)
246 }
247 return err
248 }
249 return nil
250}
251
252// Remove drops the policy of iface; the table disappears with the last guest.
253// Removing an unknown interface is not an error.
254func (m *Manager) Remove(ctx context.Context, iface string) error {
255 m.mu.Lock()
256 defer m.mu.Unlock()
257 if _, ok := m.guests[iface]; !ok {
258 return nil
259 }
260 delete(m.guests, iface)
261 return m.applyLocked(ctx)
262}
263
264// Reset deletes the runner table whatever its content: used at start to reap
265// the table of a previous run. Safe when the table does not exist.
266func (m *Manager) Reset(ctx context.Context) error {
267 m.mu.Lock()
268 defer m.mu.Unlock()
269 m.guests = map[string]Guest{}
270 return m.applyLocked(ctx)
271}
272
273func (m *Manager) applyLocked(ctx context.Context) error {
274 gs := make([]Guest, 0, len(m.guests))
275 for _, g := range m.guests {
276 gs = append(gs, g)
277 }
278 // `table X` creates the table if missing so that the following delete cannot
279 // fail; the whole script is one transaction.
280 if _, err := m.run.Run(ctx, strings.NewReader(Ruleset(gs)), m.nft, "-f", "-"); err != nil {
281 return fmt.Errorf("netpolicy: apply ruleset: %w", err)
282 }
283 return nil
284}