feat(netpolicy): generate and apply the nftables policy of job sandboxes
- commit
- 2e4b251fe93d9dbbd55d182b0632c2a775edcdf4
- author
- brice · · 2026-10-12 15:35 UTC
- parent
- b44fbc4
- signature
- GPG Verified
CD88F9E564E0102646FB6077A1A07C7C7C3E349D· arch-gpg Verified for the committer email
6 files · +553 -0
A internal/cmdrun/cmdrun.go +75 -0
| @@ -0,0 +1,75 @@ | |||
| 1 | + | // SPDX-License-Identifier: MIT | |
| 2 | + | ||
| 3 | + | // Package cmdrun runs host commands (ip, nft, mkfs.ext4) behind a small | |
| 4 | + | // interface so that callers can be tested with a fake. | |
| 5 | + | package cmdrun | |
| 6 | + | ||
| 7 | + | import ( | |
| 8 | + | "bytes" | |
| 9 | + | "context" | |
| 10 | + | "fmt" | |
| 11 | + | "io" | |
| 12 | + | "os/exec" | |
| 13 | + | "strings" | |
| 14 | + | ) | |
| 15 | + | ||
| 16 | + | // Runner runs a command to completion and returns its combined output. | |
| 17 | + | type Runner interface { | |
| 18 | + | Run(ctx context.Context, stdin io.Reader, name string, args ...string) ([]byte, error) | |
| 19 | + | } | |
| 20 | + | ||
| 21 | + | // Exec runs real commands. | |
| 22 | + | type Exec struct{} | |
| 23 | + | ||
| 24 | + | // Run implements Runner. A non-zero exit becomes an error carrying the command | |
| 25 | + | // line and its output. | |
| 26 | + | func (Exec) Run(ctx context.Context, stdin io.Reader, name string, args ...string) ([]byte, error) { | |
| 27 | + | cmd := exec.CommandContext(ctx, name, args...) | |
| 28 | + | cmd.Stdin = stdin | |
| 29 | + | var out bytes.Buffer | |
| 30 | + | cmd.Stdout = &out | |
| 31 | + | cmd.Stderr = &out | |
| 32 | + | if err := cmd.Run(); err != nil { | |
| 33 | + | return out.Bytes(), fmt.Errorf("%s %s: %w: %s", name, strings.Join(args, " "), err, strings.TrimSpace(out.String())) | |
| 34 | + | } | |
| 35 | + | return out.Bytes(), nil | |
| 36 | + | } | |
| 37 | + | ||
| 38 | + | // Call is one recorded invocation of a Fake. | |
| 39 | + | type Call struct { | |
| 40 | + | Name string | |
| 41 | + | Args []string | |
| 42 | + | Stdin string | |
| 43 | + | } | |
| 44 | + | ||
| 45 | + | // String renders the call as a shell-like line. | |
| 46 | + | func (c Call) String() string { return strings.TrimSpace(c.Name + " " + strings.Join(c.Args, " ")) } | |
| 47 | + | ||
| 48 | + | // Fake records calls and answers them with Handle (nil means success). | |
| 49 | + | type Fake struct { | |
| 50 | + | Calls []Call | |
| 51 | + | Handle func(Call) ([]byte, error) | |
| 52 | + | } | |
| 53 | + | ||
| 54 | + | // Run implements Runner. | |
| 55 | + | func (f *Fake) Run(_ context.Context, stdin io.Reader, name string, args ...string) ([]byte, error) { | |
| 56 | + | c := Call{Name: name, Args: append([]string(nil), args...)} | |
| 57 | + | if stdin != nil { | |
| 58 | + | b, _ := io.ReadAll(stdin) | |
| 59 | + | c.Stdin = string(b) | |
| 60 | + | } | |
| 61 | + | f.Calls = append(f.Calls, c) | |
| 62 | + | if f.Handle != nil { | |
| 63 | + | return f.Handle(c) | |
| 64 | + | } | |
| 65 | + | return nil, nil | |
| 66 | + | } | |
| 67 | + | ||
| 68 | + | // Lines returns the recorded calls as strings. | |
| 69 | + | func (f *Fake) Lines() []string { | |
| 70 | + | out := make([]string, len(f.Calls)) | |
| 71 | + | for i, c := range f.Calls { | |
| 72 | + | out[i] = c.String() | |
| 73 | + | } | |
| 74 | + | return out | |
| 75 | + | } | |
| @@ -0,0 +1,75 @@ | |||||
| 1 | + | // SPDX-License-Identifier: MIT | |||
| 2 | + | ||||
| 3 | + | // Package cmdrun runs host commands (ip, nft, mkfs.ext4) behind a small | |||
| 4 | + | // interface so that callers can be tested with a fake. | |||
| 5 | + | package cmdrun | |||
| 6 | + | ||||
| 7 | + | import ( | |||
| 8 | + | "bytes" | |||
| 9 | + | "context" | |||
| 10 | + | "fmt" | |||
| 11 | + | "io" | |||
| 12 | + | "os/exec" | |||
| 13 | + | "strings" | |||
| 14 | + | ) | |||
| 15 | + | ||||
| 16 | + | // Runner runs a command to completion and returns its combined output. | |||
| 17 | + | type Runner interface { | |||
| 18 | + | Run(ctx context.Context, stdin io.Reader, name string, args ...string) ([]byte, error) | |||
| 19 | + | } | |||
| 20 | + | ||||
| 21 | + | // Exec runs real commands. | |||
| 22 | + | type Exec struct{} | |||
| 23 | + | ||||
| 24 | + | // Run implements Runner. A non-zero exit becomes an error carrying the command | |||
| 25 | + | // line and its output. | |||
| 26 | + | func (Exec) Run(ctx context.Context, stdin io.Reader, name string, args ...string) ([]byte, error) { | |||
| 27 | + | cmd := exec.CommandContext(ctx, name, args...) | |||
| 28 | + | cmd.Stdin = stdin | |||
| 29 | + | var out bytes.Buffer | |||
| 30 | + | cmd.Stdout = &out | |||
| 31 | + | cmd.Stderr = &out | |||
| 32 | + | if err := cmd.Run(); err != nil { | |||
| 33 | + | return out.Bytes(), fmt.Errorf("%s %s: %w: %s", name, strings.Join(args, " "), err, strings.TrimSpace(out.String())) | |||
| 34 | + | } | |||
| 35 | + | return out.Bytes(), nil | |||
| 36 | + | } | |||
| 37 | + | ||||
| 38 | + | // Call is one recorded invocation of a Fake. | |||
| 39 | + | type Call struct { | |||
| 40 | + | Name string | |||
| 41 | + | Args []string | |||
| 42 | + | Stdin string | |||
| 43 | + | } | |||
| 44 | + | ||||
| 45 | + | // String renders the call as a shell-like line. | |||
| 46 | + | func (c Call) String() string { return strings.TrimSpace(c.Name + " " + strings.Join(c.Args, " ")) } | |||
| 47 | + | ||||
| 48 | + | // Fake records calls and answers them with Handle (nil means success). | |||
| 49 | + | type Fake struct { | |||
| 50 | + | Calls []Call | |||
| 51 | + | Handle func(Call) ([]byte, error) | |||
| 52 | + | } | |||
| 53 | + | ||||
| 54 | + | // Run implements Runner. | |||
| 55 | + | func (f *Fake) Run(_ context.Context, stdin io.Reader, name string, args ...string) ([]byte, error) { | |||
| 56 | + | c := Call{Name: name, Args: append([]string(nil), args...)} | |||
| 57 | + | if stdin != nil { | |||
| 58 | + | b, _ := io.ReadAll(stdin) | |||
| 59 | + | c.Stdin = string(b) | |||
| 60 | + | } | |||
| 61 | + | f.Calls = append(f.Calls, c) | |||
| 62 | + | if f.Handle != nil { | |||
| 63 | + | return f.Handle(c) | |||
| 64 | + | } | |||
| 65 | + | return nil, nil | |||
| 66 | + | } | |||
| 67 | + | ||||
| 68 | + | // Lines returns the recorded calls as strings. | |||
| 69 | + | func (f *Fake) Lines() []string { | |||
| 70 | + | out := make([]string, len(f.Calls)) | |||
| 71 | + | for i, c := range f.Calls { | |||
| 72 | + | out[i] = c.String() | |||
| 73 | + | } | |||
| 74 | + | return out | |||
| 75 | + | } | |||
A internal/netpolicy/netpolicy.go +230 -0
| @@ -0,0 +1,230 @@ | |||
| 1 | + | // SPDX-License-Identifier: MIT | |
| 2 | + | ||
| 3 | + | // Package netpolicy generates and applies the nftables policy of job sandboxes. | |
| 4 | + | // | |
| 5 | + | // All rules live in one dedicated table, "inet ermite_runner", that is replaced | |
| 6 | + | // atomically (one `nft -f -` transaction) whenever a sandbox is added or | |
| 7 | + | // removed, and deleted when none is left. Nothing outside that table is touched. | |
| 8 | + | // | |
| 9 | + | // Policy for every guest interface (a TAP in a micro-VM, a bridge in a | |
| 10 | + | // container): | |
| 11 | + | // - to the host itself (input hook): only DNS (udp/tcp 53) to the gateway | |
| 12 | + | // address of that guest, where the runner's DNS forwarder listens; every | |
| 13 | + | // other packet is dropped; | |
| 14 | + | // - IPv6 from the guest is dropped (the guest also disables IPv6); | |
| 15 | + | // - forwarded traffic must carry the guest's own source address; | |
| 16 | + | // - forwarded traffic to RFC 1918, CGNAT 100.64.0.0/10 (VPN overlays), | |
| 17 | + | // link-local 169.254.0.0/16 (cloud metadata 169.254.169.254), loopback, | |
| 18 | + | // "this network", multicast and reserved ranges is dropped; | |
| 19 | + | // - traffic towards a guest interface that does not belong to an established | |
| 20 | + | // connection is dropped, so guests cannot reach each other; | |
| 21 | + | // - everything else is NATed (masquerade) to the internet. | |
| 22 | + | package netpolicy | |
| 23 | + | ||
| 24 | + | import ( | |
| 25 | + | "context" | |
| 26 | + | "errors" | |
| 27 | + | "fmt" | |
| 28 | + | "net/netip" | |
| 29 | + | "sort" | |
| 30 | + | "strings" | |
| 31 | + | "sync" | |
| 32 | + | ||
| 33 | + | "bnema.dev/ermite-runner/internal/cmdrun" | |
| 34 | + | ) | |
| 35 | + | ||
| 36 | + | // Table is the nftables table owned by the runner. | |
| 37 | + | const Table = "ermite_runner" | |
| 38 | + | ||
| 39 | + | // IfacePrefixes are the interface name prefixes of sandboxes; wildcard rules | |
| 40 | + | // are generated for them. TAP devices of micro-VMs, bridges of containers. | |
| 41 | + | var IfacePrefixes = []string{"ertap", "erbr"} | |
| 42 | + | ||
| 43 | + | // blockedV4 are the destinations a guest may never reach. | |
| 44 | + | var blockedV4 = []string{ | |
| 45 | + | "0.0.0.0/8", // "this network" | |
| 46 | + | "10.0.0.0/8", // RFC 1918 | |
| 47 | + | "100.64.0.0/10", // RFC 6598 shared address space (CGNAT, VPN overlays) | |
| 48 | + | "127.0.0.0/8", // loopback | |
| 49 | + | "169.254.0.0/16", // link-local, incl. the 169.254.169.254 metadata service | |
| 50 | + | "172.16.0.0/12", // RFC 1918 | |
| 51 | + | "192.168.0.0/16", // RFC 1918 | |
| 52 | + | "224.0.0.0/4", // multicast | |
| 53 | + | "240.0.0.0/4", // reserved, incl. limited broadcast | |
| 54 | + | } | |
| 55 | + | ||
| 56 | + | // Guest is one sandbox network attachment. | |
| 57 | + | type Guest struct { | |
| 58 | + | Iface string // host-side interface: TAP or bridge name | |
| 59 | + | Gateway netip.Addr // host address on Iface; the DNS forwarder listens here | |
| 60 | + | Addr netip.Addr // the guest's only allowed source address | |
| 61 | + | } | |
| 62 | + | ||
| 63 | + | // Validate checks the fields used in generated rules. | |
| 64 | + | func (g Guest) Validate() error { | |
| 65 | + | if !validIface(g.Iface) { | |
| 66 | + | return fmt.Errorf("netpolicy: invalid interface name %q", g.Iface) | |
| 67 | + | } | |
| 68 | + | if !g.Gateway.Is4() || !g.Addr.Is4() { | |
| 69 | + | return errors.New("netpolicy: guest and gateway must be IPv4 addresses") | |
| 70 | + | } | |
| 71 | + | return nil | |
| 72 | + | } | |
| 73 | + | ||
| 74 | + | func validIface(s string) bool { | |
| 75 | + | if s == "" || len(s) > 15 { | |
| 76 | + | return false | |
| 77 | + | } | |
| 78 | + | for _, r := range s { | |
| 79 | + | if (r < 'a' || r > 'z') && (r < '0' || r > '9') && r != '-' && r != '_' { | |
| 80 | + | return false | |
| 81 | + | } | |
| 82 | + | } | |
| 83 | + | return true | |
| 84 | + | } | |
| 85 | + | ||
| 86 | + | // Net is the /30 given to one sandbox. | |
| 87 | + | type Net struct { | |
| 88 | + | Prefix netip.Prefix // the /30 | |
| 89 | + | Gateway netip.Addr // host side (.1) | |
| 90 | + | Guest netip.Addr // guest side (.2) | |
| 91 | + | } | |
| 92 | + | ||
| 93 | + | // Subnet returns the slot-th /30 of base (an IPv4 prefix of at most /30). | |
| 94 | + | func Subnet(base netip.Prefix, slot int) (Net, error) { | |
| 95 | + | if !base.Addr().Is4() || base.Bits() > 30 || slot < 0 { | |
| 96 | + | return Net{}, fmt.Errorf("netpolicy: bad base %s or slot %d", base, slot) | |
| 97 | + | } | |
| 98 | + | if slot >= 1<<(30-base.Bits()) { | |
| 99 | + | return Net{}, fmt.Errorf("netpolicy: slot %d does not fit in %s", slot, base) | |
| 100 | + | } | |
| 101 | + | b := base.Masked().Addr().As4() | |
| 102 | + | v := uint32(b[0])<<24 | uint32(b[1])<<16 | uint32(b[2])<<8 | uint32(b[3]) | |
| 103 | + | v += uint32(slot) * 4 | |
| 104 | + | addr := func(off uint32) netip.Addr { | |
| 105 | + | x := v + off | |
| 106 | + | return netip.AddrFrom4([4]byte{byte(x >> 24), byte(x >> 16), byte(x >> 8), byte(x)}) | |
| 107 | + | } | |
| 108 | + | return Net{Prefix: netip.PrefixFrom(addr(0), 30), Gateway: addr(1), Guest: addr(2)}, nil | |
| 109 | + | } | |
| 110 | + | ||
| 111 | + | // Ruleset returns the nft script that atomically replaces the runner table with | |
| 112 | + | // the policy for guests. With no guest it only deletes the table. The output is | |
| 113 | + | // deterministic (guests sorted by interface). | |
| 114 | + | func Ruleset(guests []Guest) string { | |
| 115 | + | var b strings.Builder | |
| 116 | + | fmt.Fprintf(&b, "table inet %s\ndelete table inet %s\n", Table, Table) | |
| 117 | + | if len(guests) == 0 { | |
| 118 | + | return b.String() | |
| 119 | + | } | |
| 120 | + | gs := append([]Guest(nil), guests...) | |
| 121 | + | sort.Slice(gs, func(i, j int) bool { return gs[i].Iface < gs[j].Iface }) | |
| 122 | + | ||
| 123 | + | fmt.Fprintf(&b, "table inet %s {\n", Table) | |
| 124 | + | b.WriteString("\tset blocked_v4 {\n\t\ttype ipv4_addr\n\t\tflags interval\n\t\telements = { ") | |
| 125 | + | b.WriteString(strings.Join(blockedV4, ", ")) | |
| 126 | + | b.WriteString(" }\n\t}\n") | |
| 127 | + | ||
| 128 | + | b.WriteString("\tchain input {\n\t\ttype filter hook input priority filter; policy accept;\n") | |
| 129 | + | for _, p := range IfacePrefixes { | |
| 130 | + | fmt.Fprintf(&b, "\t\tiifname \"%s*\" meta nfproto ipv6 drop\n", p) | |
| 131 | + | } | |
| 132 | + | for _, g := range gs { | |
| 133 | + | for _, proto := range []string{"udp", "tcp"} { | |
| 134 | + | fmt.Fprintf(&b, "\t\tiifname \"%s\" ip saddr %s ip daddr %s %s dport 53 accept\n", g.Iface, g.Addr, g.Gateway, proto) | |
| 135 | + | } | |
| 136 | + | fmt.Fprintf(&b, "\t\tiifname \"%s\" drop\n", g.Iface) | |
| 137 | + | } | |
| 138 | + | b.WriteString("\t}\n") | |
| 139 | + | ||
| 140 | + | b.WriteString("\tchain forward {\n\t\ttype filter hook forward priority filter; policy accept;\n") | |
| 141 | + | for _, p := range IfacePrefixes { | |
| 142 | + | fmt.Fprintf(&b, "\t\toifname \"%s*\" ct state established,related accept\n", p) | |
| 143 | + | fmt.Fprintf(&b, "\t\toifname \"%s*\" drop\n", p) | |
| 144 | + | } | |
| 145 | + | for _, p := range IfacePrefixes { | |
| 146 | + | fmt.Fprintf(&b, "\t\tiifname \"%s*\" meta nfproto ipv6 drop\n", p) | |
| 147 | + | } | |
| 148 | + | for _, g := range gs { | |
| 149 | + | fmt.Fprintf(&b, "\t\tiifname \"%s\" ip saddr != %s drop\n", g.Iface, g.Addr) | |
| 150 | + | fmt.Fprintf(&b, "\t\tiifname \"%s\" ip daddr @blocked_v4 drop\n", g.Iface) | |
| 151 | + | } | |
| 152 | + | b.WriteString("\t}\n") | |
| 153 | + | ||
| 154 | + | b.WriteString("\tchain postrouting {\n\t\ttype nat hook postrouting priority srcnat; policy accept;\n") | |
| 155 | + | for _, g := range gs { | |
| 156 | + | fmt.Fprintf(&b, "\t\tip saddr %s oifname != \"%s\" masquerade\n", g.Addr, g.Iface) | |
| 157 | + | } | |
| 158 | + | b.WriteString("\t}\n}\n") | |
| 159 | + | return b.String() | |
| 160 | + | } | |
| 161 | + | ||
| 162 | + | // Manager applies the policy for the set of live guests. | |
| 163 | + | type Manager struct { | |
| 164 | + | run cmdrun.Runner | |
| 165 | + | nft string | |
| 166 | + | mu sync.Mutex | |
| 167 | + | guests map[string]Guest | |
| 168 | + | } | |
| 169 | + | ||
| 170 | + | // NewManager returns a Manager that runs nft (path or name) through run. | |
| 171 | + | func NewManager(run cmdrun.Runner, nft string) *Manager { | |
| 172 | + | if nft == "" { | |
| 173 | + | nft = "nft" | |
| 174 | + | } | |
| 175 | + | return &Manager{run: run, nft: nft, guests: map[string]Guest{}} | |
| 176 | + | } | |
| 177 | + | ||
| 178 | + | // Add installs (or replaces) the policy of g. | |
| 179 | + | func (m *Manager) Add(ctx context.Context, g Guest) error { | |
| 180 | + | if err := g.Validate(); err != nil { | |
| 181 | + | return err | |
| 182 | + | } | |
| 183 | + | m.mu.Lock() | |
| 184 | + | defer m.mu.Unlock() | |
| 185 | + | prev, had := m.guests[g.Iface] | |
| 186 | + | m.guests[g.Iface] = g | |
| 187 | + | if err := m.applyLocked(ctx); err != nil { | |
| 188 | + | if had { | |
| 189 | + | m.guests[g.Iface] = prev | |
| 190 | + | } else { | |
| 191 | + | delete(m.guests, g.Iface) | |
| 192 | + | } | |
| 193 | + | return err | |
| 194 | + | } | |
| 195 | + | return nil | |
| 196 | + | } | |
| 197 | + | ||
| 198 | + | // Remove drops the policy of iface; the table disappears with the last guest. | |
| 199 | + | // Removing an unknown interface is not an error. | |
| 200 | + | func (m *Manager) Remove(ctx context.Context, iface string) error { | |
| 201 | + | m.mu.Lock() | |
| 202 | + | defer m.mu.Unlock() | |
| 203 | + | if _, ok := m.guests[iface]; !ok { | |
| 204 | + | return nil | |
| 205 | + | } | |
| 206 | + | delete(m.guests, iface) | |
| 207 | + | return m.applyLocked(ctx) | |
| 208 | + | } | |
| 209 | + | ||
| 210 | + | // Reset deletes the runner table whatever its content: used at start to reap | |
| 211 | + | // the table of a previous run. Safe when the table does not exist. | |
| 212 | + | func (m *Manager) Reset(ctx context.Context) error { | |
| 213 | + | m.mu.Lock() | |
| 214 | + | defer m.mu.Unlock() | |
| 215 | + | m.guests = map[string]Guest{} | |
| 216 | + | return m.applyLocked(ctx) | |
| 217 | + | } | |
| 218 | + | ||
| 219 | + | func (m *Manager) applyLocked(ctx context.Context) error { | |
| 220 | + | gs := make([]Guest, 0, len(m.guests)) | |
| 221 | + | for _, g := range m.guests { | |
| 222 | + | gs = append(gs, g) | |
| 223 | + | } | |
| 224 | + | // `table X` creates the table if missing so that the following delete cannot | |
| 225 | + | // fail; the whole script is one transaction. | |
| 226 | + | if _, err := m.run.Run(ctx, strings.NewReader(Ruleset(gs)), m.nft, "-f", "-"); err != nil { | |
| 227 | + | return fmt.Errorf("netpolicy: apply ruleset: %w", err) | |
| 228 | + | } | |
| 229 | + | return nil | |
| 230 | + | } | |
| @@ -0,0 +1,230 @@ | |||||
| 1 | + | // SPDX-License-Identifier: MIT | |||
| 2 | + | ||||
| 3 | + | // Package netpolicy generates and applies the nftables policy of job sandboxes. | |||
| 4 | + | // | |||
| 5 | + | // All rules live in one dedicated table, "inet ermite_runner", that is replaced | |||
| 6 | + | // atomically (one `nft -f -` transaction) whenever a sandbox is added or | |||
| 7 | + | // removed, and deleted when none is left. Nothing outside that table is touched. | |||
| 8 | + | // | |||
| 9 | + | // Policy for every guest interface (a TAP in a micro-VM, a bridge in a | |||
| 10 | + | // container): | |||
| 11 | + | // - to the host itself (input hook): only DNS (udp/tcp 53) to the gateway | |||
| 12 | + | // address of that guest, where the runner's DNS forwarder listens; every | |||
| 13 | + | // other packet is dropped; | |||
| 14 | + | // - IPv6 from the guest is dropped (the guest also disables IPv6); | |||
| 15 | + | // - forwarded traffic must carry the guest's own source address; | |||
| 16 | + | // - forwarded traffic to RFC 1918, CGNAT 100.64.0.0/10 (VPN overlays), | |||
| 17 | + | // link-local 169.254.0.0/16 (cloud metadata 169.254.169.254), loopback, | |||
| 18 | + | // "this network", multicast and reserved ranges is dropped; | |||
| 19 | + | // - traffic towards a guest interface that does not belong to an established | |||
| 20 | + | // connection is dropped, so guests cannot reach each other; | |||
| 21 | + | // - everything else is NATed (masquerade) to the internet. | |||
| 22 | + | package netpolicy | |||
| 23 | + | ||||
| 24 | + | import ( | |||
| 25 | + | "context" | |||
| 26 | + | "errors" | |||
| 27 | + | "fmt" | |||
| 28 | + | "net/netip" | |||
| 29 | + | "sort" | |||
| 30 | + | "strings" | |||
| 31 | + | "sync" | |||
| 32 | + | ||||
| 33 | + | "bnema.dev/ermite-runner/internal/cmdrun" | |||
| 34 | + | ) | |||
| 35 | + | ||||
| 36 | + | // Table is the nftables table owned by the runner. | |||
| 37 | + | const Table = "ermite_runner" | |||
| 38 | + | ||||
| 39 | + | // IfacePrefixes are the interface name prefixes of sandboxes; wildcard rules | |||
| 40 | + | // are generated for them. TAP devices of micro-VMs, bridges of containers. | |||
| 41 | + | var IfacePrefixes = []string{"ertap", "erbr"} | |||
| 42 | + | ||||
| 43 | + | // blockedV4 are the destinations a guest may never reach. | |||
| 44 | + | var blockedV4 = []string{ | |||
| 45 | + | "0.0.0.0/8", // "this network" | |||
| 46 | + | "10.0.0.0/8", // RFC 1918 | |||
| 47 | + | "100.64.0.0/10", // RFC 6598 shared address space (CGNAT, VPN overlays) | |||
| 48 | + | "127.0.0.0/8", // loopback | |||
| 49 | + | "169.254.0.0/16", // link-local, incl. the 169.254.169.254 metadata service | |||
| 50 | + | "172.16.0.0/12", // RFC 1918 | |||
| 51 | + | "192.168.0.0/16", // RFC 1918 | |||
| 52 | + | "224.0.0.0/4", // multicast | |||
| 53 | + | "240.0.0.0/4", // reserved, incl. limited broadcast | |||
| 54 | + | } | |||
| 55 | + | ||||
| 56 | + | // Guest is one sandbox network attachment. | |||
| 57 | + | type Guest struct { | |||
| 58 | + | Iface string // host-side interface: TAP or bridge name | |||
| 59 | + | Gateway netip.Addr // host address on Iface; the DNS forwarder listens here | |||
| 60 | + | Addr netip.Addr // the guest's only allowed source address | |||
| 61 | + | } | |||
| 62 | + | ||||
| 63 | + | // Validate checks the fields used in generated rules. | |||
| 64 | + | func (g Guest) Validate() error { | |||
| 65 | + | if !validIface(g.Iface) { | |||
| 66 | + | return fmt.Errorf("netpolicy: invalid interface name %q", g.Iface) | |||
| 67 | + | } | |||
| 68 | + | if !g.Gateway.Is4() || !g.Addr.Is4() { | |||
| 69 | + | return errors.New("netpolicy: guest and gateway must be IPv4 addresses") | |||
| 70 | + | } | |||
| 71 | + | return nil | |||
| 72 | + | } | |||
| 73 | + | ||||
| 74 | + | func validIface(s string) bool { | |||
| 75 | + | if s == "" || len(s) > 15 { | |||
| 76 | + | return false | |||
| 77 | + | } | |||
| 78 | + | for _, r := range s { | |||
| 79 | + | if (r < 'a' || r > 'z') && (r < '0' || r > '9') && r != '-' && r != '_' { | |||
| 80 | + | return false | |||
| 81 | + | } | |||
| 82 | + | } | |||
| 83 | + | return true | |||
| 84 | + | } | |||
| 85 | + | ||||
| 86 | + | // Net is the /30 given to one sandbox. | |||
| 87 | + | type Net struct { | |||
| 88 | + | Prefix netip.Prefix // the /30 | |||
| 89 | + | Gateway netip.Addr // host side (.1) | |||
| 90 | + | Guest netip.Addr // guest side (.2) | |||
| 91 | + | } | |||
| 92 | + | ||||
| 93 | + | // Subnet returns the slot-th /30 of base (an IPv4 prefix of at most /30). | |||
| 94 | + | func Subnet(base netip.Prefix, slot int) (Net, error) { | |||
| 95 | + | if !base.Addr().Is4() || base.Bits() > 30 || slot < 0 { | |||
| 96 | + | return Net{}, fmt.Errorf("netpolicy: bad base %s or slot %d", base, slot) | |||
| 97 | + | } | |||
| 98 | + | if slot >= 1<<(30-base.Bits()) { | |||
| 99 | + | return Net{}, fmt.Errorf("netpolicy: slot %d does not fit in %s", slot, base) | |||
| 100 | + | } | |||
| 101 | + | b := base.Masked().Addr().As4() | |||
| 102 | + | v := uint32(b[0])<<24 | uint32(b[1])<<16 | uint32(b[2])<<8 | uint32(b[3]) | |||
| 103 | + | v += uint32(slot) * 4 | |||
| 104 | + | addr := func(off uint32) netip.Addr { | |||
| 105 | + | x := v + off | |||
| 106 | + | return netip.AddrFrom4([4]byte{byte(x >> 24), byte(x >> 16), byte(x >> 8), byte(x)}) | |||
| 107 | + | } | |||
| 108 | + | return Net{Prefix: netip.PrefixFrom(addr(0), 30), Gateway: addr(1), Guest: addr(2)}, nil | |||
| 109 | + | } | |||
| 110 | + | ||||
| 111 | + | // Ruleset returns the nft script that atomically replaces the runner table with | |||
| 112 | + | // the policy for guests. With no guest it only deletes the table. The output is | |||
| 113 | + | // deterministic (guests sorted by interface). | |||
| 114 | + | func Ruleset(guests []Guest) string { | |||
| 115 | + | var b strings.Builder | |||
| 116 | + | fmt.Fprintf(&b, "table inet %s\ndelete table inet %s\n", Table, Table) | |||
| 117 | + | if len(guests) == 0 { | |||
| 118 | + | return b.String() | |||
| 119 | + | } | |||
| 120 | + | gs := append([]Guest(nil), guests...) | |||
| 121 | + | sort.Slice(gs, func(i, j int) bool { return gs[i].Iface < gs[j].Iface }) | |||
| 122 | + | ||||
| 123 | + | fmt.Fprintf(&b, "table inet %s {\n", Table) | |||
| 124 | + | b.WriteString("\tset blocked_v4 {\n\t\ttype ipv4_addr\n\t\tflags interval\n\t\telements = { ") | |||
| 125 | + | b.WriteString(strings.Join(blockedV4, ", ")) | |||
| 126 | + | b.WriteString(" }\n\t}\n") | |||
| 127 | + | ||||
| 128 | + | b.WriteString("\tchain input {\n\t\ttype filter hook input priority filter; policy accept;\n") | |||
| 129 | + | for _, p := range IfacePrefixes { | |||
| 130 | + | fmt.Fprintf(&b, "\t\tiifname \"%s*\" meta nfproto ipv6 drop\n", p) | |||
| 131 | + | } | |||
| 132 | + | for _, g := range gs { | |||
| 133 | + | for _, proto := range []string{"udp", "tcp"} { | |||
| 134 | + | fmt.Fprintf(&b, "\t\tiifname \"%s\" ip saddr %s ip daddr %s %s dport 53 accept\n", g.Iface, g.Addr, g.Gateway, proto) | |||
| 135 | + | } | |||
| 136 | + | fmt.Fprintf(&b, "\t\tiifname \"%s\" drop\n", g.Iface) | |||
| 137 | + | } | |||
| 138 | + | b.WriteString("\t}\n") | |||
| 139 | + | ||||
| 140 | + | b.WriteString("\tchain forward {\n\t\ttype filter hook forward priority filter; policy accept;\n") | |||
| 141 | + | for _, p := range IfacePrefixes { | |||
| 142 | + | fmt.Fprintf(&b, "\t\toifname \"%s*\" ct state established,related accept\n", p) | |||
| 143 | + | fmt.Fprintf(&b, "\t\toifname \"%s*\" drop\n", p) | |||
| 144 | + | } | |||
| 145 | + | for _, p := range IfacePrefixes { | |||
| 146 | + | fmt.Fprintf(&b, "\t\tiifname \"%s*\" meta nfproto ipv6 drop\n", p) | |||
| 147 | + | } | |||
| 148 | + | for _, g := range gs { | |||
| 149 | + | fmt.Fprintf(&b, "\t\tiifname \"%s\" ip saddr != %s drop\n", g.Iface, g.Addr) | |||
| 150 | + | fmt.Fprintf(&b, "\t\tiifname \"%s\" ip daddr @blocked_v4 drop\n", g.Iface) | |||
| 151 | + | } | |||
| 152 | + | b.WriteString("\t}\n") | |||
| 153 | + | ||||
| 154 | + | b.WriteString("\tchain postrouting {\n\t\ttype nat hook postrouting priority srcnat; policy accept;\n") | |||
| 155 | + | for _, g := range gs { | |||
| 156 | + | fmt.Fprintf(&b, "\t\tip saddr %s oifname != \"%s\" masquerade\n", g.Addr, g.Iface) | |||
| 157 | + | } | |||
| 158 | + | b.WriteString("\t}\n}\n") | |||
| 159 | + | return b.String() | |||
| 160 | + | } | |||
| 161 | + | ||||
| 162 | + | // Manager applies the policy for the set of live guests. | |||
| 163 | + | type Manager struct { | |||
| 164 | + | run cmdrun.Runner | |||
| 165 | + | nft string | |||
| 166 | + | mu sync.Mutex | |||
| 167 | + | guests map[string]Guest | |||
| 168 | + | } | |||
| 169 | + | ||||
| 170 | + | // NewManager returns a Manager that runs nft (path or name) through run. | |||
| 171 | + | func NewManager(run cmdrun.Runner, nft string) *Manager { | |||
| 172 | + | if nft == "" { | |||
| 173 | + | nft = "nft" | |||
| 174 | + | } | |||
| 175 | + | return &Manager{run: run, nft: nft, guests: map[string]Guest{}} | |||
| 176 | + | } | |||
| 177 | + | ||||
| 178 | + | // Add installs (or replaces) the policy of g. | |||
| 179 | + | func (m *Manager) Add(ctx context.Context, g Guest) error { | |||
| 180 | + | if err := g.Validate(); err != nil { | |||
| 181 | + | return err | |||
| 182 | + | } | |||
| 183 | + | m.mu.Lock() | |||
| 184 | + | defer m.mu.Unlock() | |||
| 185 | + | prev, had := m.guests[g.Iface] | |||
| 186 | + | m.guests[g.Iface] = g | |||
| 187 | + | if err := m.applyLocked(ctx); err != nil { | |||
| 188 | + | if had { | |||
| 189 | + | m.guests[g.Iface] = prev | |||
| 190 | + | } else { | |||
| 191 | + | delete(m.guests, g.Iface) | |||
| 192 | + | } | |||
| 193 | + | return err | |||
| 194 | + | } | |||
| 195 | + | return nil | |||
| 196 | + | } | |||
| 197 | + | ||||
| 198 | + | // Remove drops the policy of iface; the table disappears with the last guest. | |||
| 199 | + | // Removing an unknown interface is not an error. | |||
| 200 | + | func (m *Manager) Remove(ctx context.Context, iface string) error { | |||
| 201 | + | m.mu.Lock() | |||
| 202 | + | defer m.mu.Unlock() | |||
| 203 | + | if _, ok := m.guests[iface]; !ok { | |||
| 204 | + | return nil | |||
| 205 | + | } | |||
| 206 | + | delete(m.guests, iface) | |||
| 207 | + | return m.applyLocked(ctx) | |||
| 208 | + | } | |||
| 209 | + | ||||
| 210 | + | // Reset deletes the runner table whatever its content: used at start to reap | |||
| 211 | + | // the table of a previous run. Safe when the table does not exist. | |||
| 212 | + | func (m *Manager) Reset(ctx context.Context) error { | |||
| 213 | + | m.mu.Lock() | |||
| 214 | + | defer m.mu.Unlock() | |||
| 215 | + | m.guests = map[string]Guest{} | |||
| 216 | + | return m.applyLocked(ctx) | |||
| 217 | + | } | |||
| 218 | + | ||||
| 219 | + | func (m *Manager) applyLocked(ctx context.Context) error { | |||
| 220 | + | gs := make([]Guest, 0, len(m.guests)) | |||
| 221 | + | for _, g := range m.guests { | |||
| 222 | + | gs = append(gs, g) | |||
| 223 | + | } | |||
| 224 | + | // `table X` creates the table if missing so that the following delete cannot | |||
| 225 | + | // fail; the whole script is one transaction. | |||
| 226 | + | if _, err := m.run.Run(ctx, strings.NewReader(Ruleset(gs)), m.nft, "-f", "-"); err != nil { | |||
| 227 | + | return fmt.Errorf("netpolicy: apply ruleset: %w", err) | |||
| 228 | + | } | |||
| 229 | + | return nil | |||
| 230 | + | } | |||
A internal/netpolicy/netpolicy_test.go +176 -0
| @@ -0,0 +1,176 @@ | |||
| 1 | + | // SPDX-License-Identifier: MIT | |
| 2 | + | ||
| 3 | + | package netpolicy | |
| 4 | + | ||
| 5 | + | import ( | |
| 6 | + | "context" | |
| 7 | + | "errors" | |
| 8 | + | "flag" | |
| 9 | + | "io" | |
| 10 | + | "net/netip" | |
| 11 | + | "os" | |
| 12 | + | "path/filepath" | |
| 13 | + | "strings" | |
| 14 | + | "testing" | |
| 15 | + | ||
| 16 | + | "bnema.dev/ermite-runner/internal/cmdrun" | |
| 17 | + | ) | |
| 18 | + | ||
| 19 | + | var update = flag.Bool("update", false, "rewrite golden files") | |
| 20 | + | ||
| 21 | + | func golden(t *testing.T, name, got string) { | |
| 22 | + | t.Helper() | |
| 23 | + | p := filepath.Join("testdata", name) | |
| 24 | + | if *update { | |
| 25 | + | if err := os.MkdirAll("testdata", 0o755); err != nil { | |
| 26 | + | t.Fatal(err) | |
| 27 | + | } | |
| 28 | + | if err := os.WriteFile(p, []byte(got), 0o644); err != nil { | |
| 29 | + | t.Fatal(err) | |
| 30 | + | } | |
| 31 | + | } | |
| 32 | + | want, err := os.ReadFile(p) | |
| 33 | + | if err != nil { | |
| 34 | + | t.Fatal(err) | |
| 35 | + | } | |
| 36 | + | if string(want) != got { | |
| 37 | + | t.Fatalf("%s differs from golden (run with -update to refresh):\n%s", name, got) | |
| 38 | + | } | |
| 39 | + | } | |
| 40 | + | ||
| 41 | + | func guest(t *testing.T, iface string, slot int) Guest { | |
| 42 | + | t.Helper() | |
| 43 | + | n, err := Subnet(netip.MustParsePrefix("10.200.0.0/16"), slot) | |
| 44 | + | if err != nil { | |
| 45 | + | t.Fatal(err) | |
| 46 | + | } | |
| 47 | + | return Guest{Iface: iface, Gateway: n.Gateway, Addr: n.Guest} | |
| 48 | + | } | |
| 49 | + | ||
| 50 | + | func TestRulesetGolden(t *testing.T) { | |
| 51 | + | golden(t, "empty.nft", Ruleset(nil)) | |
| 52 | + | golden(t, "one.nft", Ruleset([]Guest{guest(t, "ertap0", 0)})) | |
| 53 | + | // Order of input must not matter. | |
| 54 | + | two := Ruleset([]Guest{guest(t, "erbr1", 1), guest(t, "ertap0", 0)}) | |
| 55 | + | if two != Ruleset([]Guest{guest(t, "ertap0", 0), guest(t, "erbr1", 1)}) { | |
| 56 | + | t.Fatal("ruleset depends on guest order") | |
| 57 | + | } | |
| 58 | + | golden(t, "two.nft", two) | |
| 59 | + | } | |
| 60 | + | ||
| 61 | + | func TestRulesetPolicyInvariants(t *testing.T) { | |
| 62 | + | rs := Ruleset([]Guest{guest(t, "ertap0", 0)}) | |
| 63 | + | for _, want := range []string{ | |
| 64 | + | "169.254.0.0/16", "100.64.0.0/10", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", | |
| 65 | + | `iifname "ertap0" drop`, "dport 53 accept", "masquerade", `iifname "ertap*" meta nfproto ipv6 drop`, | |
| 66 | + | } { | |
| 67 | + | if !strings.Contains(rs, want) { | |
| 68 | + | t.Errorf("ruleset lacks %q", want) | |
| 69 | + | } | |
| 70 | + | } | |
| 71 | + | // The drop for the host must come after the DNS accepts and the guest | |
| 72 | + | // table must never use a drop policy (it would break other tables' traffic). | |
| 73 | + | if strings.Contains(rs, "policy drop") { | |
| 74 | + | t.Error("base chains must keep policy accept") | |
| 75 | + | } | |
| 76 | + | if strings.Index(rs, "dport 53 accept") > strings.Index(rs, `iifname "ertap0" drop`) { | |
| 77 | + | t.Error("DNS accept must precede the catch-all drop") | |
| 78 | + | } | |
| 79 | + | } | |
| 80 | + | ||
| 81 | + | func TestSubnet(t *testing.T) { | |
| 82 | + | base := netip.MustParsePrefix("10.200.0.0/16") | |
| 83 | + | n, err := Subnet(base, 3) | |
| 84 | + | if err != nil { | |
| 85 | + | t.Fatal(err) | |
| 86 | + | } | |
| 87 | + | if n.Prefix.String() != "10.200.0.12/30" || n.Gateway.String() != "10.200.0.13" || n.Guest.String() != "10.200.0.14" { | |
| 88 | + | t.Fatalf("got %+v", n) | |
| 89 | + | } | |
| 90 | + | n, _ = Subnet(base, 64) | |
| 91 | + | if n.Gateway.String() != "10.200.1.1" { | |
| 92 | + | t.Fatalf("carry: %+v", n) | |
| 93 | + | } | |
| 94 | + | if _, err := Subnet(base, 1<<14); err == nil { | |
| 95 | + | t.Fatal("slot beyond the prefix accepted") | |
| 96 | + | } | |
| 97 | + | if _, err := Subnet(netip.MustParsePrefix("10.0.0.0/31"), 0); err == nil { | |
| 98 | + | t.Fatal("prefix longer than /30 accepted") | |
| 99 | + | } | |
| 100 | + | if _, err := Subnet(base, -1); err == nil { | |
| 101 | + | t.Fatal("negative slot accepted") | |
| 102 | + | } | |
| 103 | + | } | |
| 104 | + | ||
| 105 | + | func TestGuestValidate(t *testing.T) { | |
| 106 | + | g := guest(t, "ertap0", 0) | |
| 107 | + | if err := g.Validate(); err != nil { | |
| 108 | + | t.Fatal(err) | |
| 109 | + | } | |
| 110 | + | for _, bad := range []string{"", "a b", `x"y`, "a;b", "waytoolonginterfacename", "x\ny"} { | |
| 111 | + | g.Iface = bad | |
| 112 | + | if g.Validate() == nil { | |
| 113 | + | t.Errorf("iface %q accepted", bad) | |
| 114 | + | } | |
| 115 | + | } | |
| 116 | + | g.Iface = "ok" | |
| 117 | + | g.Addr = netip.MustParseAddr("::1") | |
| 118 | + | if g.Validate() == nil { | |
| 119 | + | t.Error("IPv6 guest accepted") | |
| 120 | + | } | |
| 121 | + | } | |
| 122 | + | ||
| 123 | + | func TestManager(t *testing.T) { | |
| 124 | + | f := &cmdrun.Fake{} | |
| 125 | + | m := NewManager(f, "nft") | |
| 126 | + | ctx := context.Background() | |
| 127 | + | if err := m.Add(ctx, guest(t, "ertap0", 0)); err != nil { | |
| 128 | + | t.Fatal(err) | |
| 129 | + | } | |
| 130 | + | if err := m.Add(ctx, guest(t, "ertap1", 1)); err != nil { | |
| 131 | + | t.Fatal(err) | |
| 132 | + | } | |
| 133 | + | if err := m.Remove(ctx, "ertap0"); err != nil { | |
| 134 | + | t.Fatal(err) | |
| 135 | + | } | |
| 136 | + | if err := m.Remove(ctx, "unknown"); err != nil { | |
| 137 | + | t.Fatal(err) | |
| 138 | + | } | |
| 139 | + | if len(f.Calls) != 3 { | |
| 140 | + | t.Fatalf("want 3 nft calls (unknown iface is a no-op), got %v", f.Lines()) | |
| 141 | + | } | |
| 142 | + | last := f.Calls[2] | |
| 143 | + | if last.Name != "nft" || strings.Join(last.Args, " ") != "-f -" { | |
| 144 | + | t.Fatalf("bad command %v", last) | |
| 145 | + | } | |
| 146 | + | if strings.Contains(last.Stdin, "ertap0") || !strings.Contains(last.Stdin, "ertap1") { | |
| 147 | + | t.Fatalf("after removing ertap0 only ertap1 must remain:\n%s", last.Stdin) | |
| 148 | + | } | |
| 149 | + | if err := m.Remove(ctx, "ertap1"); err != nil { | |
| 150 | + | t.Fatal(err) | |
| 151 | + | } | |
| 152 | + | if got := f.Calls[3].Stdin; got != Ruleset(nil) { | |
| 153 | + | t.Fatalf("last removal must delete the table, got:\n%s", got) | |
| 154 | + | } | |
| 155 | + | if err := m.Reset(ctx); err != nil { | |
| 156 | + | t.Fatal(err) | |
| 157 | + | } | |
| 158 | + | } | |
| 159 | + | ||
| 160 | + | func TestManagerRollsBackOnError(t *testing.T) { | |
| 161 | + | boom := errors.New("boom") | |
| 162 | + | f := &cmdrun.Fake{Handle: func(cmdrun.Call) ([]byte, error) { return nil, boom }} | |
| 163 | + | m := NewManager(f, "") | |
| 164 | + | if err := m.Add(context.Background(), guest(t, "ertap0", 0)); !errors.Is(err, boom) { | |
| 165 | + | t.Fatalf("got %v", err) | |
| 166 | + | } | |
| 167 | + | f.Handle = nil | |
| 168 | + | if err := m.Remove(context.Background(), "ertap0"); err != nil { | |
| 169 | + | t.Fatal(err) | |
| 170 | + | } | |
| 171 | + | if len(f.Calls) != 1 { | |
| 172 | + | t.Fatalf("a guest whose Add failed must not be tracked: %v", f.Lines()) | |
| 173 | + | } | |
| 174 | + | } | |
| 175 | + | ||
| 176 | + | var _ io.Reader = strings.NewReader("") | |
| @@ -0,0 +1,176 @@ | |||||
| 1 | + | // SPDX-License-Identifier: MIT | |||
| 2 | + | ||||
| 3 | + | package netpolicy | |||
| 4 | + | ||||
| 5 | + | import ( | |||
| 6 | + | "context" | |||
| 7 | + | "errors" | |||
| 8 | + | "flag" | |||
| 9 | + | "io" | |||
| 10 | + | "net/netip" | |||
| 11 | + | "os" | |||
| 12 | + | "path/filepath" | |||
| 13 | + | "strings" | |||
| 14 | + | "testing" | |||
| 15 | + | ||||
| 16 | + | "bnema.dev/ermite-runner/internal/cmdrun" | |||
| 17 | + | ) | |||
| 18 | + | ||||
| 19 | + | var update = flag.Bool("update", false, "rewrite golden files") | |||
| 20 | + | ||||
| 21 | + | func golden(t *testing.T, name, got string) { | |||
| 22 | + | t.Helper() | |||
| 23 | + | p := filepath.Join("testdata", name) | |||
| 24 | + | if *update { | |||
| 25 | + | if err := os.MkdirAll("testdata", 0o755); err != nil { | |||
| 26 | + | t.Fatal(err) | |||
| 27 | + | } | |||
| 28 | + | if err := os.WriteFile(p, []byte(got), 0o644); err != nil { | |||
| 29 | + | t.Fatal(err) | |||
| 30 | + | } | |||
| 31 | + | } | |||
| 32 | + | want, err := os.ReadFile(p) | |||
| 33 | + | if err != nil { | |||
| 34 | + | t.Fatal(err) | |||
| 35 | + | } | |||
| 36 | + | if string(want) != got { | |||
| 37 | + | t.Fatalf("%s differs from golden (run with -update to refresh):\n%s", name, got) | |||
| 38 | + | } | |||
| 39 | + | } | |||
| 40 | + | ||||
| 41 | + | func guest(t *testing.T, iface string, slot int) Guest { | |||
| 42 | + | t.Helper() | |||
| 43 | + | n, err := Subnet(netip.MustParsePrefix("10.200.0.0/16"), slot) | |||
| 44 | + | if err != nil { | |||
| 45 | + | t.Fatal(err) | |||
| 46 | + | } | |||
| 47 | + | return Guest{Iface: iface, Gateway: n.Gateway, Addr: n.Guest} | |||
| 48 | + | } | |||
| 49 | + | ||||
| 50 | + | func TestRulesetGolden(t *testing.T) { | |||
| 51 | + | golden(t, "empty.nft", Ruleset(nil)) | |||
| 52 | + | golden(t, "one.nft", Ruleset([]Guest{guest(t, "ertap0", 0)})) | |||
| 53 | + | // Order of input must not matter. | |||
| 54 | + | two := Ruleset([]Guest{guest(t, "erbr1", 1), guest(t, "ertap0", 0)}) | |||
| 55 | + | if two != Ruleset([]Guest{guest(t, "ertap0", 0), guest(t, "erbr1", 1)}) { | |||
| 56 | + | t.Fatal("ruleset depends on guest order") | |||
| 57 | + | } | |||
| 58 | + | golden(t, "two.nft", two) | |||
| 59 | + | } | |||
| 60 | + | ||||
| 61 | + | func TestRulesetPolicyInvariants(t *testing.T) { | |||
| 62 | + | rs := Ruleset([]Guest{guest(t, "ertap0", 0)}) | |||
| 63 | + | for _, want := range []string{ | |||
| 64 | + | "169.254.0.0/16", "100.64.0.0/10", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", | |||
| 65 | + | `iifname "ertap0" drop`, "dport 53 accept", "masquerade", `iifname "ertap*" meta nfproto ipv6 drop`, | |||
| 66 | + | } { | |||
| 67 | + | if !strings.Contains(rs, want) { | |||
| 68 | + | t.Errorf("ruleset lacks %q", want) | |||
| 69 | + | } | |||
| 70 | + | } | |||
| 71 | + | // The drop for the host must come after the DNS accepts and the guest | |||
| 72 | + | // table must never use a drop policy (it would break other tables' traffic). | |||
| 73 | + | if strings.Contains(rs, "policy drop") { | |||
| 74 | + | t.Error("base chains must keep policy accept") | |||
| 75 | + | } | |||
| 76 | + | if strings.Index(rs, "dport 53 accept") > strings.Index(rs, `iifname "ertap0" drop`) { | |||
| 77 | + | t.Error("DNS accept must precede the catch-all drop") | |||
| 78 | + | } | |||
| 79 | + | } | |||
| 80 | + | ||||
| 81 | + | func TestSubnet(t *testing.T) { | |||
| 82 | + | base := netip.MustParsePrefix("10.200.0.0/16") | |||
| 83 | + | n, err := Subnet(base, 3) | |||
| 84 | + | if err != nil { | |||
| 85 | + | t.Fatal(err) | |||
| 86 | + | } | |||
| 87 | + | if n.Prefix.String() != "10.200.0.12/30" || n.Gateway.String() != "10.200.0.13" || n.Guest.String() != "10.200.0.14" { | |||
| 88 | + | t.Fatalf("got %+v", n) | |||
| 89 | + | } | |||
| 90 | + | n, _ = Subnet(base, 64) | |||
| 91 | + | if n.Gateway.String() != "10.200.1.1" { | |||
| 92 | + | t.Fatalf("carry: %+v", n) | |||
| 93 | + | } | |||
| 94 | + | if _, err := Subnet(base, 1<<14); err == nil { | |||
| 95 | + | t.Fatal("slot beyond the prefix accepted") | |||
| 96 | + | } | |||
| 97 | + | if _, err := Subnet(netip.MustParsePrefix("10.0.0.0/31"), 0); err == nil { | |||
| 98 | + | t.Fatal("prefix longer than /30 accepted") | |||
| 99 | + | } | |||
| 100 | + | if _, err := Subnet(base, -1); err == nil { | |||
| 101 | + | t.Fatal("negative slot accepted") | |||
| 102 | + | } | |||
| 103 | + | } | |||
| 104 | + | ||||
| 105 | + | func TestGuestValidate(t *testing.T) { | |||
| 106 | + | g := guest(t, "ertap0", 0) | |||
| 107 | + | if err := g.Validate(); err != nil { | |||
| 108 | + | t.Fatal(err) | |||
| 109 | + | } | |||
| 110 | + | for _, bad := range []string{"", "a b", `x"y`, "a;b", "waytoolonginterfacename", "x\ny"} { | |||
| 111 | + | g.Iface = bad | |||
| 112 | + | if g.Validate() == nil { | |||
| 113 | + | t.Errorf("iface %q accepted", bad) | |||
| 114 | + | } | |||
| 115 | + | } | |||
| 116 | + | g.Iface = "ok" | |||
| 117 | + | g.Addr = netip.MustParseAddr("::1") | |||
| 118 | + | if g.Validate() == nil { | |||
| 119 | + | t.Error("IPv6 guest accepted") | |||
| 120 | + | } | |||
| 121 | + | } | |||
| 122 | + | ||||
| 123 | + | func TestManager(t *testing.T) { | |||
| 124 | + | f := &cmdrun.Fake{} | |||
| 125 | + | m := NewManager(f, "nft") | |||
| 126 | + | ctx := context.Background() | |||
| 127 | + | if err := m.Add(ctx, guest(t, "ertap0", 0)); err != nil { | |||
| 128 | + | t.Fatal(err) | |||
| 129 | + | } | |||
| 130 | + | if err := m.Add(ctx, guest(t, "ertap1", 1)); err != nil { | |||
| 131 | + | t.Fatal(err) | |||
| 132 | + | } | |||
| 133 | + | if err := m.Remove(ctx, "ertap0"); err != nil { | |||
| 134 | + | t.Fatal(err) | |||
| 135 | + | } | |||
| 136 | + | if err := m.Remove(ctx, "unknown"); err != nil { | |||
| 137 | + | t.Fatal(err) | |||
| 138 | + | } | |||
| 139 | + | if len(f.Calls) != 3 { | |||
| 140 | + | t.Fatalf("want 3 nft calls (unknown iface is a no-op), got %v", f.Lines()) | |||
| 141 | + | } | |||
| 142 | + | last := f.Calls[2] | |||
| 143 | + | if last.Name != "nft" || strings.Join(last.Args, " ") != "-f -" { | |||
| 144 | + | t.Fatalf("bad command %v", last) | |||
| 145 | + | } | |||
| 146 | + | if strings.Contains(last.Stdin, "ertap0") || !strings.Contains(last.Stdin, "ertap1") { | |||
| 147 | + | t.Fatalf("after removing ertap0 only ertap1 must remain:\n%s", last.Stdin) | |||
| 148 | + | } | |||
| 149 | + | if err := m.Remove(ctx, "ertap1"); err != nil { | |||
| 150 | + | t.Fatal(err) | |||
| 151 | + | } | |||
| 152 | + | if got := f.Calls[3].Stdin; got != Ruleset(nil) { | |||
| 153 | + | t.Fatalf("last removal must delete the table, got:\n%s", got) | |||
| 154 | + | } | |||
| 155 | + | if err := m.Reset(ctx); err != nil { | |||
| 156 | + | t.Fatal(err) | |||
| 157 | + | } | |||
| 158 | + | } | |||
| 159 | + | ||||
| 160 | + | func TestManagerRollsBackOnError(t *testing.T) { | |||
| 161 | + | boom := errors.New("boom") | |||
| 162 | + | f := &cmdrun.Fake{Handle: func(cmdrun.Call) ([]byte, error) { return nil, boom }} | |||
| 163 | + | m := NewManager(f, "") | |||
| 164 | + | if err := m.Add(context.Background(), guest(t, "ertap0", 0)); !errors.Is(err, boom) { | |||
| 165 | + | t.Fatalf("got %v", err) | |||
| 166 | + | } | |||
| 167 | + | f.Handle = nil | |||
| 168 | + | if err := m.Remove(context.Background(), "ertap0"); err != nil { | |||
| 169 | + | t.Fatal(err) | |||
| 170 | + | } | |||
| 171 | + | if len(f.Calls) != 1 { | |||
| 172 | + | t.Fatalf("a guest whose Add failed must not be tracked: %v", f.Lines()) | |||
| 173 | + | } | |||
| 174 | + | } | |||
| 175 | + | ||||
| 176 | + | var _ io.Reader = strings.NewReader("") | |||
A internal/netpolicy/testdata/empty.nft +2 -0
| @@ -0,0 +1,2 @@ | |||
| 1 | + | table inet ermite_runner | |
| 2 | + | delete table inet ermite_runner | |
| @@ -0,0 +1,2 @@ | |||||
| 1 | + | table inet ermite_runner | |||
| 2 | + | delete table inet ermite_runner | |||
A internal/netpolicy/testdata/one.nft +32 -0
| @@ -0,0 +1,32 @@ | |||
| 1 | + | table inet ermite_runner | |
| 2 | + | delete table inet ermite_runner | |
| 3 | + | table inet ermite_runner { | |
| 4 | + | set blocked_v4 { | |
| 5 | + | type ipv4_addr | |
| 6 | + | flags interval | |
| 7 | + | elements = { 0.0.0.0/8, 10.0.0.0/8, 100.64.0.0/10, 127.0.0.0/8, 169.254.0.0/16, 172.16.0.0/12, 192.168.0.0/16, 224.0.0.0/4, 240.0.0.0/4 } | |
| 8 | + | } | |
| 9 | + | chain input { | |
| 10 | + | type filter hook input priority filter; policy accept; | |
| 11 | + | iifname "ertap*" meta nfproto ipv6 drop | |
| 12 | + | iifname "erbr*" meta nfproto ipv6 drop | |
| 13 | + | iifname "ertap0" ip saddr 10.200.0.2 ip daddr 10.200.0.1 udp dport 53 accept | |
| 14 | + | iifname "ertap0" ip saddr 10.200.0.2 ip daddr 10.200.0.1 tcp dport 53 accept | |
| 15 | + | iifname "ertap0" drop | |
| 16 | + | } | |
| 17 | + | chain forward { | |
| 18 | + | type filter hook forward priority filter; policy accept; | |
| 19 | + | oifname "ertap*" ct state established,related accept | |
| 20 | + | oifname "ertap*" drop | |
| 21 | + | oifname "erbr*" ct state established,related accept | |
| 22 | + | oifname "erbr*" drop | |
| 23 | + | iifname "ertap*" meta nfproto ipv6 drop | |
| 24 | + | iifname "erbr*" meta nfproto ipv6 drop | |
| 25 | + | iifname "ertap0" ip saddr != 10.200.0.2 drop | |
| 26 | + | iifname "ertap0" ip daddr @blocked_v4 drop | |
| 27 | + | } | |
| 28 | + | chain postrouting { | |
| 29 | + | type nat hook postrouting priority srcnat; policy accept; | |
| 30 | + | ip saddr 10.200.0.2 oifname != "ertap0" masquerade | |
| 31 | + | } | |
| 32 | + | } | |
| @@ -0,0 +1,32 @@ | |||||
| 1 | + | table inet ermite_runner | |||
| 2 | + | delete table inet ermite_runner | |||
| 3 | + | table inet ermite_runner { | |||
| 4 | + | set blocked_v4 { | |||
| 5 | + | type ipv4_addr | |||
| 6 | + | flags interval | |||
| 7 | + | elements = { 0.0.0.0/8, 10.0.0.0/8, 100.64.0.0/10, 127.0.0.0/8, 169.254.0.0/16, 172.16.0.0/12, 192.168.0.0/16, 224.0.0.0/4, 240.0.0.0/4 } | |||
| 8 | + | } | |||
| 9 | + | chain input { | |||
| 10 | + | type filter hook input priority filter; policy accept; | |||
| 11 | + | iifname "ertap*" meta nfproto ipv6 drop | |||
| 12 | + | iifname "erbr*" meta nfproto ipv6 drop | |||
| 13 | + | iifname "ertap0" ip saddr 10.200.0.2 ip daddr 10.200.0.1 udp dport 53 accept | |||
| 14 | + | iifname "ertap0" ip saddr 10.200.0.2 ip daddr 10.200.0.1 tcp dport 53 accept | |||
| 15 | + | iifname "ertap0" drop | |||
| 16 | + | } | |||
| 17 | + | chain forward { | |||
| 18 | + | type filter hook forward priority filter; policy accept; | |||
| 19 | + | oifname "ertap*" ct state established,related accept | |||
| 20 | + | oifname "ertap*" drop | |||
| 21 | + | oifname "erbr*" ct state established,related accept | |||
| 22 | + | oifname "erbr*" drop | |||
| 23 | + | iifname "ertap*" meta nfproto ipv6 drop | |||
| 24 | + | iifname "erbr*" meta nfproto ipv6 drop | |||
| 25 | + | iifname "ertap0" ip saddr != 10.200.0.2 drop | |||
| 26 | + | iifname "ertap0" ip daddr @blocked_v4 drop | |||
| 27 | + | } | |||
| 28 | + | chain postrouting { | |||
| 29 | + | type nat hook postrouting priority srcnat; policy accept; | |||
| 30 | + | ip saddr 10.200.0.2 oifname != "ertap0" masquerade | |||
| 31 | + | } | |||
| 32 | + | } | |||
A internal/netpolicy/testdata/two.nft +38 -0
| @@ -0,0 +1,38 @@ | |||
| 1 | + | table inet ermite_runner | |
| 2 | + | delete table inet ermite_runner | |
| 3 | + | table inet ermite_runner { | |
| 4 | + | set blocked_v4 { | |
| 5 | + | type ipv4_addr | |
| 6 | + | flags interval | |
| 7 | + | elements = { 0.0.0.0/8, 10.0.0.0/8, 100.64.0.0/10, 127.0.0.0/8, 169.254.0.0/16, 172.16.0.0/12, 192.168.0.0/16, 224.0.0.0/4, 240.0.0.0/4 } | |
| 8 | + | } | |
| 9 | + | chain input { | |
| 10 | + | type filter hook input priority filter; policy accept; | |
| 11 | + | iifname "ertap*" meta nfproto ipv6 drop | |
| 12 | + | iifname "erbr*" meta nfproto ipv6 drop | |
| 13 | + | iifname "erbr1" ip saddr 10.200.0.6 ip daddr 10.200.0.5 udp dport 53 accept | |
| 14 | + | iifname "erbr1" ip saddr 10.200.0.6 ip daddr 10.200.0.5 tcp dport 53 accept | |
| 15 | + | iifname "erbr1" drop | |
| 16 | + | iifname "ertap0" ip saddr 10.200.0.2 ip daddr 10.200.0.1 udp dport 53 accept | |
| 17 | + | iifname "ertap0" ip saddr 10.200.0.2 ip daddr 10.200.0.1 tcp dport 53 accept | |
| 18 | + | iifname "ertap0" drop | |
| 19 | + | } | |
| 20 | + | chain forward { | |
| 21 | + | type filter hook forward priority filter; policy accept; | |
| 22 | + | oifname "ertap*" ct state established,related accept | |
| 23 | + | oifname "ertap*" drop | |
| 24 | + | oifname "erbr*" ct state established,related accept | |
| 25 | + | oifname "erbr*" drop | |
| 26 | + | iifname "ertap*" meta nfproto ipv6 drop | |
| 27 | + | iifname "erbr*" meta nfproto ipv6 drop | |
| 28 | + | iifname "erbr1" ip saddr != 10.200.0.6 drop | |
| 29 | + | iifname "erbr1" ip daddr @blocked_v4 drop | |
| 30 | + | iifname "ertap0" ip saddr != 10.200.0.2 drop | |
| 31 | + | iifname "ertap0" ip daddr @blocked_v4 drop | |
| 32 | + | } | |
| 33 | + | chain postrouting { | |
| 34 | + | type nat hook postrouting priority srcnat; policy accept; | |
| 35 | + | ip saddr 10.200.0.6 oifname != "erbr1" masquerade | |
| 36 | + | ip saddr 10.200.0.2 oifname != "ertap0" masquerade | |
| 37 | + | } | |
| 38 | + | } | |
| @@ -0,0 +1,38 @@ | |||||
| 1 | + | table inet ermite_runner | |||
| 2 | + | delete table inet ermite_runner | |||
| 3 | + | table inet ermite_runner { | |||
| 4 | + | set blocked_v4 { | |||
| 5 | + | type ipv4_addr | |||
| 6 | + | flags interval | |||
| 7 | + | elements = { 0.0.0.0/8, 10.0.0.0/8, 100.64.0.0/10, 127.0.0.0/8, 169.254.0.0/16, 172.16.0.0/12, 192.168.0.0/16, 224.0.0.0/4, 240.0.0.0/4 } | |||
| 8 | + | } | |||
| 9 | + | chain input { | |||
| 10 | + | type filter hook input priority filter; policy accept; | |||
| 11 | + | iifname "ertap*" meta nfproto ipv6 drop | |||
| 12 | + | iifname "erbr*" meta nfproto ipv6 drop | |||
| 13 | + | iifname "erbr1" ip saddr 10.200.0.6 ip daddr 10.200.0.5 udp dport 53 accept | |||
| 14 | + | iifname "erbr1" ip saddr 10.200.0.6 ip daddr 10.200.0.5 tcp dport 53 accept | |||
| 15 | + | iifname "erbr1" drop | |||
| 16 | + | iifname "ertap0" ip saddr 10.200.0.2 ip daddr 10.200.0.1 udp dport 53 accept | |||
| 17 | + | iifname "ertap0" ip saddr 10.200.0.2 ip daddr 10.200.0.1 tcp dport 53 accept | |||
| 18 | + | iifname "ertap0" drop | |||
| 19 | + | } | |||
| 20 | + | chain forward { | |||
| 21 | + | type filter hook forward priority filter; policy accept; | |||
| 22 | + | oifname "ertap*" ct state established,related accept | |||
| 23 | + | oifname "ertap*" drop | |||
| 24 | + | oifname "erbr*" ct state established,related accept | |||
| 25 | + | oifname "erbr*" drop | |||
| 26 | + | iifname "ertap*" meta nfproto ipv6 drop | |||
| 27 | + | iifname "erbr*" meta nfproto ipv6 drop | |||
| 28 | + | iifname "erbr1" ip saddr != 10.200.0.6 drop | |||
| 29 | + | iifname "erbr1" ip daddr @blocked_v4 drop | |||
| 30 | + | iifname "ertap0" ip saddr != 10.200.0.2 drop | |||
| 31 | + | iifname "ertap0" ip daddr @blocked_v4 drop | |||
| 32 | + | } | |||
| 33 | + | chain postrouting { | |||
| 34 | + | type nat hook postrouting priority srcnat; policy accept; | |||
| 35 | + | ip saddr 10.200.0.6 oifname != "erbr1" masquerade | |||
| 36 | + | ip saddr 10.200.0.2 oifname != "ertap0" masquerade | |||
| 37 | + | } | |||
| 38 | + | } | |||