1// SPDX-License-Identifier: MIT
2
3// Package dnsfwd is a small DNS forwarder for job sandboxes (stdlib only). It
4// listens on the gateway address of each sandbox (UDP and TCP, port 53 in
5// production) and relays queries unchanged to the host's resolvers; the network
6// policy lets guests reach nothing else on the host. It does not parse queries
7// beyond what framing needs, so it supports every record type and EDNS.
8//
9// Abuse limits, per listener (one listener per guest, so one guest cannot
10// starve another): at most MaxInflight concurrent upstream exchanges (UDP
11// queries beyond that are dropped, never queued), at most MaxTCPConns open TCP
12// connections, an absolute lifetime of TCPLifetime per TCP connection (a client
13// cannot hold a slot by trickling bytes), a deadline on every upstream
14// exchange, and a cap on queries per TCP connection.
15//
16// One name can be overridden (Forwarder.Override, the opt-in guest forge
17// access): queries for it are answered locally with the address of the
18// listener, see override.go. Every other name is forwarded.
19package dnsfwd
20
21import (
22 "context"
23 "encoding/binary"
24 "errors"
25 "fmt"
26 "io"
27 "net"
28 "net/netip"
29 "strconv"
30 "sync"
31 "sync/atomic"
32 "syscall"
33 "time"
34
35 "github.com/bnema/zerowrap"
36 "golang.org/x/sys/unix"
37)
38
39const (
40 // TCPLifetime is the absolute lifetime of a TCP connection, set at accept.
41 TCPLifetime = 30 * time.Second
42 // UpstreamTimeout is the deadline of one upstream attempt.
43 UpstreamTimeout = 3 * time.Second
44 // TCPIdleTimeout closes TCP client connections that stay silent.
45 TCPIdleTimeout = 10 * time.Second
46
47 // MaxInflight bounds concurrent upstream exchanges per listener.
48 MaxInflight = 32
49 // MaxTCPConns bounds open TCP connections per listener.
50 MaxTCPConns = 16
51 // TCPMaxQueries caps queries per TCP connection.
52 TCPMaxQueries = 64
53
54 maxUDP = 4096 // largest datagram relayed (EDNS0 sizes up to 4096)
55 hdrLen = 12
56)
57
58// Forwarder relays DNS queries. The zero value is not usable; use New.
59type Forwarder struct {
60 upstreams func() []string
61 port int
62 // Lifetime overrides TCPLifetime; set before Listen (tests).
63 Lifetime time.Duration
64 // FreeBind binds with IP_FREEBIND, so that the gateway address may be
65 // configured after the listener. Off by default: with Docker the address
66 // exists when the network is created and a missing one is an error to
67 // report at once, not a listener on an address nobody owns. Set before Listen.
68 FreeBind bool
69 // Override, when set, is a host name that every listener answers itself:
70 // A with the address the listener is bound to, and NOERROR with no data for
71 // every other type, whatever the host's resolvers say. Case-insensitive; a
72 // trailing dot is ignored. Set before Listen.
73 Override string
74
75 mu sync.Mutex
76 listeners map[netip.Addr]*listener
77}
78
79type listener struct {
80 addr netip.Addr
81 udp net.PacketConn
82 tcp net.Listener
83 cancel context.CancelFunc
84 done sync.WaitGroup
85 // inflight and conns are the per-listener limits (counting semaphores).
86 inflight chan struct{}
87 conns chan struct{}
88 // Dropped counts UDP queries dropped because the listener was saturated.
89 dropped atomic.Int64
90}
91
92// New returns a Forwarder. upstreams returns the "host:port" resolvers to try
93// in order (see Resolvers.Addrs); port is the port to listen on (53, or another
94// port in tests).
95func New(upstreams func() []string, port int) *Forwarder {
96 return &Forwarder{
97 upstreams: upstreams,
98 port: port,
99 Lifetime: TCPLifetime,
100 listeners: map[netip.Addr]*listener{},
101 }
102}
103
104// Listen starts serving on addr (UDP and TCP). It returns once both sockets are
105// bound. Listening twice on the same address is an error.
106func (f *Forwarder) Listen(ctx context.Context, addr netip.Addr) error {
107 f.mu.Lock()
108 defer f.mu.Unlock()
109 if _, ok := f.listeners[addr]; ok {
110 return fmt.Errorf("dnsfwd: already listening on %s", addr)
111 }
112 hp := net.JoinHostPort(addr.String(), strconv.Itoa(f.port))
113 var lc net.ListenConfig
114 if f.FreeBind {
115 // The gateway address may not exist yet (podman's netavark configures the
116 // bridge address when the first container starts, after this listener).
117 lc.Control = func(_, _ string, c syscall.RawConn) error {
118 var serr error
119 if err := c.Control(func(fd uintptr) { serr = unix.SetsockoptInt(int(fd), unix.IPPROTO_IP, unix.IP_FREEBIND, 1) }); err != nil {
120 return err
121 }
122 return serr
123 }
124 }
125 var (
126 pc net.PacketConn
127 tl net.Listener
128 err error
129 )
130 // With port 0 (tests) UDP and TCP must share the port the kernel picked for
131 // UDP, which a TCP socket may already hold: pick again a few times.
132 for attempt := 0; ; attempt++ {
133 bind := hp
134 if pc, err = lc.ListenPacket(ctx, "udp", bind); err != nil {
135 return fmt.Errorf("dnsfwd: listen udp %s: %w", bind, err)
136 }
137 if f.port == 0 {
138 bind = pc.LocalAddr().String()
139 }
140 if tl, err = lc.Listen(ctx, "tcp", bind); err == nil {
141 break
142 }
143 _ = pc.Close()
144 if f.port != 0 || attempt == 9 || !errors.Is(err, syscall.EADDRINUSE) {
145 return fmt.Errorf("dnsfwd: listen tcp %s: %w", bind, err)
146 }
147 }
148 lctx, cancel := context.WithCancel(context.WithoutCancel(ctx))
149 l := &listener{addr: addr, udp: pc, tcp: tl, cancel: cancel, inflight: make(chan struct{}, MaxInflight), conns: make(chan struct{}, MaxTCPConns)}
150 f.listeners[addr] = l
151 log := zerowrap.FromCtx(ctx)
152 l.done.Add(2)
153 go func() { defer l.done.Done(); f.serveUDP(lctx, l, log) }()
154 go func() { defer l.done.Done(); f.serveTCP(lctx, l, log) }()
155 return nil
156}
157
158// Addr returns the bound UDP address for addr, useful when port is 0.
159func (f *Forwarder) Addr(addr netip.Addr) (net.Addr, bool) {
160 f.mu.Lock()
161 defer f.mu.Unlock()
162 l, ok := f.listeners[addr]
163 if !ok {
164 return nil, false
165 }
166 return l.udp.LocalAddr(), true
167}
168
169// Close stops listening on addr and waits for its goroutines. Unknown
170// addresses are ignored.
171func (f *Forwarder) Close(addr netip.Addr) {
172 f.mu.Lock()
173 l, ok := f.listeners[addr]
174 delete(f.listeners, addr)
175 f.mu.Unlock()
176 if !ok {
177 return
178 }
179 l.cancel()
180 _ = l.udp.Close()
181 _ = l.tcp.Close()
182 l.done.Wait()
183}
184
185// CloseAll stops every listener.
186func (f *Forwarder) CloseAll() {
187 f.mu.Lock()
188 addrs := make([]netip.Addr, 0, len(f.listeners))
189 for a := range f.listeners {
190 addrs = append(addrs, a)
191 }
192 f.mu.Unlock()
193 for _, a := range addrs {
194 f.Close(a)
195 }
196}
197
198// Dropped returns how many UDP queries the listener on addr dropped because it
199// was saturated.
200func (f *Forwarder) Dropped(addr netip.Addr) int64 {
201 f.mu.Lock()
202 defer f.mu.Unlock()
203 if l, ok := f.listeners[addr]; ok {
204 return l.dropped.Load()
205 }
206 return 0
207}
208
209func (f *Forwarder) serveUDP(ctx context.Context, l *listener, log zerowrap.Logger) {
210 pc := l.udp
211 buf := make([]byte, maxUDP)
212 for {
213 n, from, err := pc.ReadFrom(buf)
214 if err != nil {
215 if ctx.Err() == nil && !errors.Is(err, net.ErrClosed) {
216 log.Warn().Err(err).Msg("dns forwarder: udp read")
217 }
218 return
219 }
220 if n < hdrLen {
221 continue
222 }
223 select {
224 case l.inflight <- struct{}{}:
225 default:
226 // Saturated: drop. The client retries; blocking here would let one
227 // guest's flood stall the read loop.
228 l.dropped.Add(1)
229 continue
230 }
231 q := append([]byte(nil), buf[:n]...)
232 go func() {
233 defer func() { <-l.inflight }()
234 resp := f.exchange(ctx, l, q, false)
235 if len(resp) > 0 {
236 _, _ = pc.WriteTo(resp, from)
237 }
238 }()
239 }
240}
241
242func (f *Forwarder) serveTCP(ctx context.Context, l *listener, log zerowrap.Logger) {
243 var wg sync.WaitGroup
244 defer wg.Wait()
245 for {
246 c, err := l.tcp.Accept()
247 if err != nil {
248 if ctx.Err() == nil && !errors.Is(err, net.ErrClosed) {
249 log.Warn().Err(err).Msg("dns forwarder: tcp accept")
250 }
251 return
252 }
253 select {
254 case l.conns <- struct{}{}:
255 default:
256 _ = c.Close() // too many open connections on this listener
257 continue
258 }
259 wg.Add(1)
260 go func() {
261 defer wg.Done()
262 defer func() { <-l.conns }()
263 defer func() { _ = c.Close() }()
264 stop := context.AfterFunc(ctx, func() { _ = c.Close() })
265 defer stop()
266 // Absolute deadline from accept: neither idle nor slow-loris
267 // connections keep their slot past TCPLifetime.
268 deadline := time.Now().Add(f.Lifetime)
269 _ = c.SetDeadline(deadline)
270 f.serveConn(ctx, l, c, deadline)
271 }()
272 }
273}
274
275func (f *Forwarder) serveConn(ctx context.Context, l *listener, c net.Conn, deadline time.Time) {
276 for range TCPMaxQueries {
277 // The idle timeout can only shorten the absolute deadline.
278 dl := time.Now().Add(TCPIdleTimeout)
279 if deadline.Before(dl) {
280 dl = deadline
281 }
282 _ = c.SetReadDeadline(dl)
283 var lb [2]byte
284 if _, err := io.ReadFull(c, lb[:]); err != nil {
285 return
286 }
287 n := int(binary.BigEndian.Uint16(lb[:]))
288 if n < hdrLen {
289 return
290 }
291 q := make([]byte, n)
292 if _, err := io.ReadFull(c, q); err != nil {
293 return
294 }
295 select {
296 case l.inflight <- struct{}{}:
297 default:
298 return // saturated: close, the client retries
299 }
300 resp := f.exchange(ctx, l, q, true)
301 <-l.inflight
302 if len(resp) == 0 || len(resp) > 0xffff {
303 return
304 }
305 out := make([]byte, 2+len(resp))
306 binary.BigEndian.PutUint16(out, uint16(len(resp)))
307 copy(out[2:], resp)
308 if _, err := c.Write(out); err != nil {
309 return
310 }
311 }
312}
313
314// exchange relays q to the upstreams in order and returns the first answer, or
315// a SERVFAIL when none answers. The transport towards the upstream matches the
316// client's: a client that fell back to TCP gets a complete answer. The
317// overridden name, if any, is answered without asking the upstreams.
318func (f *Forwarder) exchange(ctx context.Context, l *listener, q []byte, overTCP bool) []byte {
319 if resp := f.overrideAnswer(l.addr, q); resp != nil {
320 return resp
321 }
322 for _, up := range f.upstreams() {
323 var resp []byte
324 var err error
325 if overTCP {
326 resp, err = exchangeTCP(ctx, up, q)
327 } else {
328 resp, err = exchangeUDP(ctx, up, q)
329 }
330 if err == nil && len(resp) >= hdrLen && resp[0] == q[0] && resp[1] == q[1] {
331 return resp
332 }
333 }
334 return servfail(q)
335}
336
337func exchangeUDP(ctx context.Context, up string, q []byte) ([]byte, error) {
338 ctx, cancel := context.WithTimeout(ctx, UpstreamTimeout)
339 defer cancel()
340 var d net.Dialer
341 c, err := d.DialContext(ctx, "udp", up)
342 if err != nil {
343 return nil, err
344 }
345 defer func() { _ = c.Close() }()
346 dl, _ := ctx.Deadline()
347 _ = c.SetDeadline(dl)
348 if _, err := c.Write(q); err != nil {
349 return nil, err
350 }
351 buf := make([]byte, maxUDP)
352 for {
353 n, err := c.Read(buf)
354 if err != nil {
355 return nil, err
356 }
357 // Ignore datagrams that do not answer this query (ID mismatch).
358 if n >= hdrLen && buf[0] == q[0] && buf[1] == q[1] {
359 return append([]byte(nil), buf[:n]...), nil
360 }
361 }
362}
363
364func exchangeTCP(ctx context.Context, up string, q []byte) ([]byte, error) {
365 ctx, cancel := context.WithTimeout(ctx, UpstreamTimeout)
366 defer cancel()
367 var d net.Dialer
368 c, err := d.DialContext(ctx, "tcp", up)
369 if err != nil {
370 return nil, err
371 }
372 defer func() { _ = c.Close() }()
373 dl, _ := ctx.Deadline()
374 _ = c.SetDeadline(dl)
375 out := make([]byte, 2+len(q))
376 binary.BigEndian.PutUint16(out, uint16(len(q)))
377 copy(out[2:], q)
378 if _, err := c.Write(out); err != nil {
379 return nil, err
380 }
381 var l [2]byte
382 if _, err := io.ReadFull(c, l[:]); err != nil {
383 return nil, err
384 }
385 resp := make([]byte, binary.BigEndian.Uint16(l[:]))
386 if _, err := io.ReadFull(c, resp); err != nil {
387 return nil, err
388 }
389 return resp, nil
390}
391
392// servfail builds a SERVFAIL reply for q: same ID and opcode, QR and RA set,
393// RD copied, no records.
394func servfail(q []byte) []byte {
395 if len(q) < hdrLen {
396 return nil
397 }
398 r := make([]byte, hdrLen)
399 r[0], r[1] = q[0], q[1]
400 r[2] = 0x80 | (q[2] & 0x79) // QR, opcode, RD kept; AA/TC cleared
401 r[3] = 0x80 | 0x02 // RA, RCODE=SERVFAIL
402 return r
403}